[2.7] bpo-30450: Pull Windows dependencies from GitHub rather than SVN (GH-1783) (GH-3306)

The Windows build now depends on Python 3.6 to fetch externals, but it
will be downloaded via NuGet (which is downloaded via PowerShell) if it
is not available via `py -3.6`.  This means the only thing that must be
installed on a modern Windows box to do a full build of CPython with all
extensions is Visual Studio.

Cherry-picked from 51599e2, parts of 40a23e8, parts of 68d663c, d5cd21d, and possibly others that I've missed.

Also:

* Rename db -> bsddb for disambiguity

* Update sqlite3 to 3.14.2.0 since it's the version we use on 3.x, and it's simpler to just use it than to also upload the old version to cpython-source-deps

* Add PCbuild/*.ilk to .gitignore

Author: zware

.github/appveyor.yml

@@ -10,6 +10,8 @@ build_script:
 - cmd: PCbuild\python.exe -m test.pythoninfo
 test_script:
 - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest -j2
+environment:
+  HOST_PYTHON: C:\Python36\python.exe
 
 # Only trigger AppVeyor if actual code or its configuration changes
 only_commits:

.gitignore

@@ -44,6 +44,7 @@ PC/*/*.suo
 PC/*/Win32-temp-*
 PC/*/x64-temp-*
 PC/*/amd64
+PCbuild/*.ilk
 PCbuild/*.user
 PCbuild/*.suo
 PCbuild/*.*sdf

Doc/make.bat

@@ -5,8 +5,19 @@ pushd %~dp0
 
 set this=%~n0
 
-if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
-if "%PYTHON%" EQU "" set PYTHON=py
+call ..\PCBuild\find_python.bat %PYTHON%
+if not defined SPHINXBUILD if defined PYTHON (
+    %PYTHON% -c "import sphinx" > nul 2> nul
+    if errorlevel 1 (
+        echo Installing sphinx with %PYTHON%
+        %PYTHON% -m pip install sphinx
+        if errorlevel 1 exit /B
+    )
+    set SPHINXBUILD=%PYTHON% -c "import sphinx, sys; sys.argv[0] = 'sphinx-build'; sphinx.main()"
+)
+
+if not defined PYTHON set PYTHON=py
+if not defined SPHINXBUILD set SPHINXBUILD=sphinx-build
 
 if DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles(x86)%
 if NOT DEFINED ProgramFiles(x86) set _PRGMFLS=%ProgramFiles%
@@ -73,7 +84,7 @@ goto end
 if NOT "%PAPER%" == "" (
     set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS%
 )
-cmd /C %SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*
+cmd /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . %BUILDDIR%\%*"
 
 if "%1" EQU "htmlhelp" (
     if  not exist "%HTMLHELP%" (

Misc/NEWS.d/next/Windows/2017-09-04-14-00-37.bpo-30450.YwitaJ.rst

@@ -0,0 +1 @@
+Pull build dependencies from GitHub rather than svn.python.org.

PC/VS9.0/pyproject.vsprops

@@ -50,7 +50,7 @@
 	/>
 	<UserMacro
 		Name="bsddb47Dir"
-		Value="$(externalsDir)\db-4.7.25.0\build_windows"
+		Value="$(externalsDir)\bsddb-4.7.25.0\build_windows"
 	/>
 	<UserMacro
 		Name="bsddb47DepLibs"
@@ -74,7 +74,7 @@
 	/>
 	<UserMacro
 		Name="sqlite3Dir"
-		Value="$(externalsDir)\sqlite-3.8.11.0"
+		Value="$(externalsDir)\sqlite-3.14.2.0"
 	/>
 	<UserMacro
 		Name="bz2Dir"

PCbuild/build.bat

@@ -5,12 +5,15 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
 echo.
 echo.Build CPython from the command line.  Requires the appropriate
 echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
-echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
-echo.given.
 echo.
 echo.After the flags recognized by this script, up to 9 arguments to be passed
 echo.directly to MSBuild may be passed.  If the argument contains an '=', the
-echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`)
+echo.entire argument must be quoted (e.g. `%~nx0 "/p:PlatformToolset=v100"`).
+echo.Alternatively you can put extra flags for MSBuild in a file named 
+echo.`msbuild.rsp` in the `PCbuild` directory, one flag per line. This file
+echo.will be picked automatically by MSBuild. Flags put in this file does not
+echo.need to be quoted. You can still use environment variables inside the 
+echo.response file.
 echo.
 echo.Available flags:
 echo.  -h  Display this help message
@@ -47,7 +50,6 @@ exit /b 127
 :Run
 setlocal
 set platf=Win32
-set vs_platf=x86
 set conf=Release
 set target=Build
 set dir=%~dp0
@@ -56,10 +58,6 @@ set verbose=/nologo /v:m
 set kill=
 set do_pgo=
 set pgo_job=-m test.regrtest --pgo
-set on_64_bit=true
-
-rem This may not be 100% accurate, but close enough.
-if "%ProgramFiles(x86)%"=="" (set on_64_bit=false)
 
 :CheckOpts
 if "%~1"=="-h" goto Usage
@@ -89,18 +87,12 @@ if "%IncludeBsddb%"=="" set IncludeBsddb=true
 
 if "%IncludeExternals%"=="true" call "%dir%get_externals.bat"
 
-if "%platf%"=="x64" (
-    if "%on_64_bit%"=="true" (
-        rem This ought to always be correct these days...
-        set vs_platf=amd64
-    ) else (
-        if "%do_pgo%"=="true" (
-            echo.ERROR: Cannot cross-compile with PGO
-            echo.    32bit operating system detected, if this is incorrect,
-            echo.    make sure the ProgramFiles(x86^) environment variable is set
-            exit /b 1
-        )
-        set vs_platf=x86_amd64
+if "%do_pgo%" EQU "true" if "%platf%" EQU "x64" (
+    if "%PROCESSOR_ARCHITEW6432%" NEQ "AMD64" if "%PROCESSOR_ARCHITECTURE%" NEQ "AMD64" (
+        echo.ERROR: Cannot cross-compile with PGO 
+        echo.       32bit operating system detected. Ensure your PROCESSOR_ARCHITECTURE
+        echo.       and PROCESSOR_ARCHITEW6432 environment variables are correct.
+        exit /b 1 
     )
 )
 
@@ -109,26 +101,28 @@ if exist "%GIT%" set GITProperty=/p:GIT="%GIT%"
 if not exist "%GIT%" echo Cannot find Git on PATH & set GITProperty=
 
 rem Setup the environment
-call "%dir%env.bat" %vs_platf% >nul
+call "%dir%find_msbuild.bat" %MSBUILD%
+if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2)
 
 if "%kill%"=="true" call :Kill
 
 if "%do_pgo%"=="true" (
     set conf=PGInstrument
-    call :Build
+    call :Build %1 %2 %3 %4 %5 %6 %7 %8 %9
     del /s "%dir%\*.pgc"
     del /s "%dir%\..\Lib\*.pyc"
     echo on
     call "%dir%\..\python.bat" %pgo_job%
     @echo off
     call :Kill
     set conf=PGUpdate
+    set target=Build
 )
 goto Build
 
 :Kill
 echo on
-msbuild "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
+%MSBUILD% "%dir%\pythoncore.vcxproj" /t:KillPython %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:KillPython=true
 
@@ -140,7 +134,7 @@ rem Call on MSBuild to do the work, echo the command.
 rem Passing %1-9 is not the preferred option, but argument parsing in
 rem batch is, shall we say, "lackluster"
 echo on
-msbuild "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
+%MSBUILD% "%dir%pcbuild.proj" /t:%target% %parallel% %verbose%^
  /p:Configuration=%conf% /p:Platform=%platf%^
  /p:IncludeExternals=%IncludeExternals%^
  /p:IncludeSSL=%IncludeSSL% /p:IncludeTkinter=%IncludeTkinter%^

PCbuild/find_msbuild.bat

@@ -0,0 +1,52 @@
+@rem
+@rem Searches for MSBuild.exe. This is the only tool we need to initiate
+@rem a build, so we no longer search for the full VC toolset.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set MSBUILD="%*") & (set _Py_MSBuild_Source=environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set MSBUILD="%~1") & (set _Py_MSBuild_Source=environment) & goto :found
+
+:begin_search
+@set MSBUILD=
+
+@rem If msbuild.exe is on the PATH, assume that the user wants that one.
+@where msbuild > "%TEMP%\msbuild.loc" 2> nul && set /P MSBUILD= < "%TEMP%\msbuild.loc" & del "%TEMP%\msbuild.loc"
+@if exist "%MSBUILD%" set MSBUILD="%MSBUILD%" & (set _Py_MSBuild_Source=PATH) & goto :found
+
+@rem VS 2015 and earlier register MSBuild separately, so we can find it.
+@rem Prefer MSBuild 14.0 over MSBuild 15.0, since the latter may not be able to find a VC14 install.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\MSBuild\ToolsVersions\14.0" /v MSBuildToolsPath /reg:32') DO @(
+    @if "%%i"=="MSBuildToolsPath" @if exist "%%k\msbuild.exe" @(set MSBUILD="%%k\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=registry) & goto :found
+
+@rem VS 2017 sets exactly one install as the "main" install, so we may find MSBuild in there.
+@reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32 >nul 2>nul
+@if NOT ERRORLEVEL 1 @for /F "tokens=1,2*" %%i in ('reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\VisualStudio\SxS\VS7" /v 15.0 /reg:32') DO @(
+    @if "%%i"=="15.0" @if exist "%%k\MSBuild\15.0\Bin\msbuild.exe" @(set MSBUILD="%%k\MSBuild\15.0\Bin\msbuild.exe")
+)
+@if exist %MSBUILD% (set _Py_MSBuild_Source=Visual Studio 2017 registry) & goto :found
+
+
+@exit /b 1
+
+:found
+@echo Using %MSBUILD% (found in the %_Py_MSBuild_Source%)
+@set _Py_MSBuild_Source=

PCbuild/find_python.bat

@@ -0,0 +1,75 @@
+@rem
+@rem Searches for python.exe and may download a private copy from nuget.
+@rem
+@rem This file is supposed to modify the state of the caller (specifically
+@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
+@rem changing any other persistent state.
+@rem
+
+@rem No arguments provided means do full search
+@if '%1' EQU '' goto :begin_search
+
+@rem One argument may be the full path. Use a goto so we don't try to
+@rem parse the next if statement - incorrect quoting in the multi-arg
+@rem case can cause us to break immediately.
+@if '%2' EQU '' goto :one_arg
+
+@rem Entire command line may represent the full path if quoting failed.
+@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
+@goto :begin_search
+
+:one_arg
+@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
+
+:begin_search
+@set PYTHON=
+
+@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
+@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
+
+@rem If we have Python in externals, use that one
+@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
+
+@rem If HOST_PYTHON is recent enough, use that
+@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
+
+@rem If py.exe finds a recent enough version, use that one
+@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
+
+@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
+@set _Py_NUGET=%NUGET%
+@set _Py_NUGET_URL=%NUGET_URL%
+@set _Py_HOST_PYTHON=%HOST_PYTHON%
+@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
+@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
+@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
+@if NOT exist "%_Py_NUGET%" (
+    @echo Downloading nuget...
+    @rem NB: Must use single quotes around NUGET here, NOT double!
+    @rem Otherwise, a space in the path would break things
+    @rem If it fails, retry with any available copy of Python
+    @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
+    @if errorlevel 1 (
+        @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
+    )
+)
+@echo Installing Python via nuget...
+@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
+@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
+@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
+
+
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=
+@exit /b 1
+
+:found
+@echo Using %PYTHON% (%_Py_Python_Source%)
+@set _Py_Python_Source=
+@set _Py_EXTERNALS_DIR=
+@set _Py_NUGET=
+@set _Py_NUGET_URL=
+@set _Py_HOST_PYTHON=

PCbuild/get_external.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import zipfile
+from urllib.request import urlretrieve
+
+
+def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose):
+    repo = f'cpython-{"bin" if binary else "source"}-deps'
+    url = f'https://github.com/{org}/{repo}/archive/{commit_hash}.zip'
+    reporthook = None
+    if verbose:
+        reporthook = print
+    zip_dir.mkdir(parents=True, exist_ok=True)
+    filename, headers = urlretrieve(
+        url,
+        zip_dir / f'{commit_hash}.zip',
+        reporthook=reporthook,
+    )
+    return filename
+
+
+def extract_zip(externals_dir, zip_path):
+    with zipfile.ZipFile(os.fspath(zip_path)) as zf:
+        zf.extractall(os.fspath(externals_dir))
+        return externals_dir / zf.namelist()[0].split('/')[0]
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument('-v', '--verbose', action='store_true')
+    p.add_argument('-b', '--binary', action='store_true',
+                   help='Is the dependency in the binary repo?')
+    p.add_argument('-O', '--organization',
+                   help='Organization owning the deps repos', default='python')
+    p.add_argument('-e', '--externals-dir', type=pathlib.Path,
+                   help='Directory in which to store dependencies',
+                   default=pathlib.Path(__file__).parent.parent / 'externals')
+    p.add_argument('tag',
+                   help='tag of the dependency')
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+    zip_path = fetch_zip(
+        args.tag,
+        args.externals_dir / 'zips',
+        org=args.organization,
+        binary=args.binary,
+        verbose=args.verbose,
+    )
+    final_name = args.externals_dir / args.tag
+    extract_zip(args.externals_dir, zip_path).replace(final_name)
+
+
+if __name__ == '__main__':
+    main()