changeset: 91589:c3022e2606df user: Zachary Ware date: Mon Jul 07 13:39:59 2014 -0500 files: Misc/NEWS PCbuild/build.bat PCbuild/build_pgo.bat PCbuild/env.bat PCbuild/get_externals.bat PCbuild/readme.txt Tools/buildbot/build-amd64.bat Tools/buildbot/build.bat Tools/buildbot/buildmsi.bat Tools/buildbot/clean-amd64.bat Tools/buildbot/clean.bat Tools/buildbot/external-amd64.bat Tools/buildbot/external-common.bat Tools/buildbot/external.bat Tools/buildbot/test-amd64.bat Tools/buildbot/test.bat Tools/scripts/run_tests.py description: Issue #21907: Improved the batch scripts provided for building Python. The user-facing scripts in PCbuild have been updated to be easier to use and the buildbot scripts in Tools\buildbot have been updated to use the user-facing scripts in PCbuild wherever possible. diff -r e66c387da81b -r c3022e2606df Misc/NEWS --- a/Misc/NEWS Mon Jul 07 21:30:54 2014 +0300 +++ b/Misc/NEWS Mon Jul 07 13:39:59 2014 -0500 @@ -746,6 +746,8 @@ Windows ------- +- Issue #21907: Improved the batch scripts provided for building Python. + - Issue #21671, CVE-2014-0224: The bundled version of OpenSSL has been updated to 1.0.1h. diff -r e66c387da81b -r c3022e2606df PCbuild/build.bat --- a/PCbuild/build.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/PCbuild/build.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,19 +1,34 @@ @echo off -rem A batch program to build or rebuild a particular configuration. +rem A batch program to build or rebuild a particular configuration, rem just for convenience. +rem Arguments: +rem -c Set the configuration (default: Release) +rem -p Set the platform (x64 or Win32, default: Win32) +rem -r Target Rebuild instead of Build +rem -d Set the configuration to Debug +rem -e Pull in external libraries using get_externals.bat + setlocal set platf=Win32 set conf=Release -set target=build +set target=Build set dir=%~dp0 :CheckOpts if "%1"=="-c" (set conf=%2) & shift & shift & goto CheckOpts if "%1"=="-p" (set platf=%2) & shift & shift & goto CheckOpts -if "%1"=="-r" (set target=rebuild) & shift & goto CheckOpts +if "%1"=="-r" (set target=Rebuild) & shift & goto CheckOpts if "%1"=="-d" (set conf=Debug) & shift & goto CheckOpts +if "%1"=="-e" call "%dir%get_externals.bat" & shift & goto CheckOpts + +if "%platf%"=="x64" (set vs_platf=x86_amd64) -set cmd=msbuild /p:useenv=true %dir%pcbuild.sln /t:%target% /p:Configuration=%conf% /p:Platform=%platf% -echo %cmd% -%cmd% +rem Setup the environment +call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %vs_platf% + +rem Call on MSBuild to do the work, echo the command. +rem Passing %1-9 is not the preferred option, but argument parsing in +rem batch is, shall we say, "lackluster" +echo on +msbuild "%dir%pcbuild.sln" /t:%target% /p:Configuration=%conf% /p:Platform=%platf% %1 %2 %3 %4 %5 %6 %7 %8 %9 diff -r e66c387da81b -r c3022e2606df PCbuild/build_pgo.bat --- a/PCbuild/build_pgo.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/PCbuild/build_pgo.bat Mon Jul 07 13:39:59 2014 -0500 @@ -9,12 +9,12 @@ set platf=Win32 rem use the performance testsuite. This is quick and simple -set job1=..\tools\pybench\pybench.py -n 1 -C 1 --with-gc -set path1=..\tools\pybench +set job1=%~dp0..\tools\pybench\pybench.py -n 1 -C 1 --with-gc +set path1=%~dp0..\tools\pybench rem or the whole testsuite for more thorough testing -set job2=..\lib\test\regrtest.py -set path2=..\lib +set job2=%~dp0..\lib\test\regrtest.py +set path2=%~dp0..\lib set job=%job1% set clrpath=%path1% @@ -31,9 +31,9 @@ call build -p %platf% -c PGInstrument rem remove .pyc files, .pgc files and execute the job -%PGI%\python.exe rmpyc.py %clrpath% +%PGI%\python.exe rmpyc.py "%clrpath%" del %PGI%\*.pgc -%PGI%\python.exe %job% +%PGI%\python.exe "%job%" rem finally build the optimized version if exist %PGO% del /s /q %PGO% diff -r e66c387da81b -r c3022e2606df PCbuild/env.bat --- a/PCbuild/env.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/PCbuild/env.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,9 +1,5 @@ @echo off -set VS10=%ProgramFiles(x86)%\Microsoft Visual Studio 10.0 -IF EXIST "%VS10%" GOTO ok -set VS10=%ProgramFiles%\Microsoft Visual Studio 10.0 -:ok echo Build environments: x86, ia64, amd64, x86_amd64, x86_ia64 echo. -call "%VS10%\VC\vcvarsall.bat" %1 +call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" %1 diff -r e66c387da81b -r c3022e2606df PCbuild/get_externals.bat --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PCbuild/get_externals.bat Mon Jul 07 13:39:59 2014 -0500 @@ -0,0 +1,100 @@ +@echo off +setlocal +rem Simple script to fetch source for external libraries + +pushd "%~dp0..\.." + +if "%SVNROOT%"=="" set SVNROOT=http://svn.python.org/projects/external/ + +rem Optionally clean up first. Be warned that this can be very destructive! +if not "%1"=="" ( + for %%c in (-c --clean --clean-only) do ( + if "%1"=="%%c" goto clean + ) + goto usage +) +goto fetch + +:clean +echo.Cleaning up external libraries. +for /D %%d in ( + bzip2-* + db-* + openssl-* + tcl-* + tcltk* + tk-* + tix-* + sqlite-* + xz-* + ) do ( + echo.Removing %%d + rmdir /s /q %%d +) +if "%1"=="--clean-only" ( + goto end +) + +:fetch +rem Fetch current versions + +svn --version > nul 2>&1 +if ERRORLEVEL 9009 ( + echo.svn.exe must be on your PATH. + echo.Try TortoiseSVN (http://tortoisesvn.net/^) and be sure to check the + echo.command line tools option. + popd + exit /b 1 +) + +echo.Fetching external libraries... + +for %%e in ( + bzip2-1.0.6 + openssl-1.0.1h + tcl-8.6.1.0 + tk-8.6.1.0 + tix-8.4.3.4 + sqlite-3.8.3.1 + xz-5.0.5 + ) do ( + if exist %%e ( + echo.%%e already exists, skipping. + ) else ( + echo.Fetching %%e... + svn export %SVNROOT%%%e + ) +) + +goto end + +:usage +echo.invalid argument: %1 +echo.usage: %~n0 [[ -c ^| --clean ] ^| --clean-only ] +echo. +echo.Pull all sources necessary for compiling optional extension modules +echo.that rely on external libraries. Requires svn.exe to be on your PATH +echo.and pulls sources from %SVNROOT%. +echo. +echo.Use the -c or --clean option to clean up all external library sources +echo.before pulling in the current versions. +echo. +echo.Use the --clean-only option to do the same cleaning, without pulling in +echo.anything new. +echo. +echo.Only the first argument is checked, all others are ignored. +echo. +echo.**WARNING**: the cleaning options unconditionally remove any directory +echo.that is a child of +echo. %CD% +echo.and matches wildcard patterns beginning with bzip2-, db-, openssl-, tcl-, +echo.tcltk, tk-, tix-, sqlite-, or xz-, and as such has the potential to be +echo.very destructive if you are not aware of what it is doing. Use with +echo.caution! +popd +exit /b -1 + + +:end +echo Finished. +popd diff -r e66c387da81b -r c3022e2606df PCbuild/readme.txt --- a/PCbuild/readme.txt Mon Jul 07 21:30:54 2014 +0300 +++ b/PCbuild/readme.txt Mon Jul 07 13:39:59 2014 -0500 @@ -1,3 +1,13 @@ +Quick Start Guide +----------------- + +1. Install Microsoft Visual C++ 2010 SP1, any edition. +2. Install Subversion, and make sure 'svn.exe' is on your PATH. +3. Install NASM, and make sure 'nasm.exe' is on your PATH. +4. Run "build.bat -e" to build Python in 32-bit Release configuration. +5. (Optional, but recommended) Run the test suite with "rt.bat -q". + + Building Python using Microsoft Visual C++ ------------------------------------------ @@ -24,8 +34,8 @@ Studio, select the desired combination of configuration and platform, then build with "Build Solution" or the F7 keyboard shortcut. You can also build from the command line using the "build.bat" script in this -directory. The solution is configured to build the projects in the -correct order. +directory; see below for details. The solution is configured to build +the projects in the correct order. The solution currently supports two platforms. The Win32 platform is used to build standard x86-compatible 32-bit binaries, output into this @@ -56,6 +66,26 @@ settings, though without PGO. +Building Python using the build.bat script +---------------------------------------------- + +In this directory you can find build.bat, a script designed to make +building Python on Windows simpler. The only absolute requirement for +using this script is for the VS100COMNTOOLS environment variable to be +properly set, which should be done by Microsoft Visual C++ 2010 +installation. + +By default, build.bat will build Python in Release configuration for +the 32-bit Win32 platform. It accepts several arguments to change +this behavior: + + -c Set the configuration (see above) + -d Shortcut for "-c Debug" + -p Set the platform to build for ("Win32" or "x64") + -r Rebuild instead of just building + -e Use get_externals.bat to fetch external sources + + Legacy support -------------- @@ -227,25 +257,18 @@ The last category of sub-projects listed above wrap external projects Python doesn't control, and as such a little more work is required in order to download the relevant source files for each project before they -can be built. The buildbots must ensure that all libraries are present -before building, so the easiest approach is to run either external.bat -or external-amd64.bat (depending on platform) in the ..\Tools\buildbot -directory from ..\, i.e.: - - C:\python\cpython\PCbuild>cd .. - C:\python\cpython>Tools\buildbot\external.bat - -This extracts all the external sub-projects from +can be built. However, a simple script is provided to make this as +painless as possible, called "get_externals.bat" and located in this +directory. This script extracts all the external sub-projects from http://svn.python.org/projects/external -via Subversion (so you'll need an svn.exe on your PATH) and places them +via Subversion (so you'll need svn.exe on your PATH) and places them in ..\.. (relative to this directory). It is also possible to download sources from each project's homepage, -though you may have to change the names of some folders in order to make -things work. For instance, if you were to download a version 5.0.7 of -XZ Utils, you would need to extract the archive into ..\..\xz-5.0.5 -anyway, since that is where the solution is set to look for xz. The -same is true for all other external projects. +though you may have to change folder names or pass the names to MSBuild +as the values of certain properties in order for the build solution to +find them. This is an advanced topic and not necessarily fully +supported. Building for AMD64 diff -r e66c387da81b -r c3022e2606df Tools/buildbot/build-amd64.bat --- a/Tools/buildbot/build-amd64.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/build-amd64.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,6 +1,2 @@ @rem Used by the buildbot "compile" step. -cmd /c Tools\buildbot\external-amd64.bat -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 -cmd /c Tools\buildbot\clean-amd64.bat - -msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=x64 +call "%~dp0build.bat" -p x64 %* diff -r e66c387da81b -r c3022e2606df Tools/buildbot/build.bat --- a/Tools/buildbot/build.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/build.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,7 +1,17 @@ @rem Used by the buildbot "compile" step. -cmd /c Tools\buildbot\external.bat -call "%VS100COMNTOOLS%vsvars32.bat" -cmd /c Tools\buildbot\clean.bat + +@rem Clean up +call "%~dp0clean.bat" + +@rem If you need the buildbots to start fresh (such as when upgrading to +@rem a new version of an external library, especially Tcl/Tk): +@rem 1) uncomment the following line: -msbuild PCbuild\pcbuild.sln /p:Configuration=Debug /p:Platform=Win32 +@rem call "%~dp0..\..\PCbuild\get_externals.bat" --clean-only +@rem 2) commit and push +@rem 3) wait for all Windows bots to start a build with that changeset +@rem 4) re-comment, commit and push again + +@rem Do the build +call "%~dp0..\..\PCbuild\build.bat" -e -d %* diff -r e66c387da81b -r c3022e2606df Tools/buildbot/buildmsi.bat --- a/Tools/buildbot/buildmsi.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/buildmsi.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,21 +1,20 @@ @rem Used by the buildbot "buildmsi" step. +setlocal -cmd /c Tools\buildbot\external.bat +set cwd=%CD% @rem build release versions of things -call "%VS100COMNTOOLS%vsvars32.bat" - -@rem build Python -msbuild /p:useenv=true PCbuild\pcbuild.sln /p:Configuration=Release /p:Platform=Win32 +call "%~dp0build.bat" -c Release @rem build the documentation -bash.exe -c 'cd Doc;make PYTHON=python2.5 update htmlhelp' -"%ProgramFiles%\HTML Help Workshop\hhc.exe" Doc\build\htmlhelp\python26a3.hhp +call "%~dp0..\..\Doc\make.bat" htmlhelp @rem build the MSI file -cd PC +call "%VS100COMNTOOLS%..\..\VC\vcvarsall.bat" x86 +cd "%~dp0..\..\PC" nmake /f icons.mak cd ..\Tools\msi del *.msi nmake /f msisupport.mak %HOST_PYTHON% msi.py +cd "%cwd%" diff -r e66c387da81b -r c3022e2606df Tools/buildbot/clean-amd64.bat --- a/Tools/buildbot/clean-amd64.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/clean-amd64.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,10 +1,2 @@ @rem Used by the buildbot "clean" step. -call "%VS100COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 -@echo Deleting .pyc/.pyo files ... -del /s Lib\*.pyc Lib\*.pyo -@echo Deleting test leftovers ... -rmdir /s /q build -cd PCbuild -msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x64 -msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x64 -cd .. +call "%~dp0clean.bat" diff -r e66c387da81b -r c3022e2606df Tools/buildbot/clean.bat --- a/Tools/buildbot/clean.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/clean.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,8 +1,22 @@ -@rem Used by the buildbot "clean" step. -call "%VS100COMNTOOLS%vsvars32.bat" -@echo Deleting test leftovers ... -rmdir /s /q build -cd PCbuild -msbuild /target:clean pcbuild.sln /p:Configuration=Release /p:PlatformTarget=x86 -msbuild /target:clean pcbuild.sln /p:Configuration=Debug /p:PlatformTarget=x86 -cd .. +@echo off +rem Used by the buildbot "clean" step. + +setlocal +set root=%~dp0..\.. +set pcbuild=%root%\PCbuild + +echo.Attempting to kill Pythons... +for %%k in (kill_python.exe + kill_python_d.exe + amd64\kill_python.exe + amd64\kill_python_d.exe + ) do ( + if exist "%pcbuild%\%%k" ( + echo.Calling %pcbuild%\%%k... + "%pcbuild%\%%k" + ) +) + +echo Purging all non-tracked files with `hg purge` +echo on +hg -R "%root%" --config extensions.purge= purge --all -X "%root%\Lib\test\data" diff -r e66c387da81b -r c3022e2606df Tools/buildbot/external-amd64.bat --- a/Tools/buildbot/external-amd64.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/external-amd64.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,6 +1,2 @@ -@rem Fetches (and builds if necessary) external dependencies - -@rem Assume we start inside the Python source directory -call "Tools\buildbot\external-common.bat" - - +@echo Please use PCbuild\get_externals.bat instead. +@"%~dp0..\..\PCbuild\get_externals.bat" %* diff -r e66c387da81b -r c3022e2606df Tools/buildbot/external-common.bat --- a/Tools/buildbot/external-common.bat Mon Jul 07 21:30:54 2014 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,47 +0,0 @@ -@rem Common file shared between external.bat and external-amd64.bat. Responsible for -@rem fetching external components into the root\.. buildbot directories. - -cd .. -@rem XXX: If you need to force the buildbots to start from a fresh environment, uncomment -@rem the following, check it in, then check it out, comment it out, then check it back in. -@rem if exist bzip2-1.0.6 rd /s/q bzip2-1.0.6 -@rem if exist tcltk rd /s/q tcltk -@rem if exist tcltk64 rd /s/q tcltk64 -@rem if exist tcl-8.6.1.0 rd /s/q tcl-8.6.1.0 -@rem if exist tk-8.6.1.0 rd /s/q tk-8.6.1.0 -@rem if exist tix-8.4.3.4 rd /s/q tix-8.4.3.4 -@rem if exist db-4.4.20 rd /s/q db-4.4.20 -@rem if exist openssl-1.0.1h rd /s/q openssl-1.0.1h -@rem if exist sqlite-3.7.12 rd /s/q sqlite-3.7.12 - -@rem bzip -if not exist bzip2-1.0.6 ( - rd /s/q bzip2-1.0.5 - svn export http://svn.python.org/projects/external/bzip2-1.0.6 -) - -@rem OpenSSL -if not exist openssl-1.0.1h ( - rd /s/q openssl-1.0.1g - svn export http://svn.python.org/projects/external/openssl-1.0.1h -) - -@rem tcl/tk/tix -if not exist tcl-8.6.1.0 ( - rd /s/q tcltk tcltk64 tcl-8.5.11.0 tk-8.5.11.0 tix-8.4.3.3 - svn export http://svn.python.org/projects/external/tcl-8.6.1.0 -) -if not exist tk-8.6.1.0 svn export http://svn.python.org/projects/external/tk-8.6.1.0 -if not exist tix-8.4.3.4 svn export http://svn.python.org/projects/external/tix-8.4.3.4 - -@rem sqlite3 -if not exist sqlite-3.8.3.1 ( - rd /s/q sqlite-source-3.8.1 - svn export http://svn.python.org/projects/external/sqlite-3.8.3.1 -) - -@rem lzma -if not exist xz-5.0.5 ( - rd /s/q xz-5.0.3 - svn export http://svn.python.org/projects/external/xz-5.0.5 -) diff -r e66c387da81b -r c3022e2606df Tools/buildbot/external.bat --- a/Tools/buildbot/external.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/external.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,5 +1,2 @@ -@rem Fetches (and builds if necessary) external dependencies - -@rem Assume we start inside the Python source directory -call "Tools\buildbot\external-common.bat" - +@echo Please use PCbuild\get_externals.bat instead. +@"%~dp0..\..\PCbuild\get_externals.bat" %* diff -r e66c387da81b -r c3022e2606df Tools/buildbot/test-amd64.bat --- a/Tools/buildbot/test-amd64.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/test-amd64.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,6 +1,6 @@ @rem Used by the buildbot "test" step. rem The following line should be removed before #20035 is closed -set TCL_LIBRARY=%CD%\..\tcltk64\lib\tcl8.6 -cd PCbuild -call rt.bat -d -q -x64 -uall -rwW -n --timeout=3600 %1 %2 %3 %4 %5 %6 %7 %8 %9 +set TCL_LIBRARY=%~dp0..\..\..\tcltk64\lib\tcl8.6 + +"%~dp0..\..\PCbuild\amd64\python_d.exe" "%~dp0..\scripts\run_tests.py" -j 1 -u all -W --timeout=3600 %* diff -r e66c387da81b -r c3022e2606df Tools/buildbot/test.bat --- a/Tools/buildbot/test.bat Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/buildbot/test.bat Mon Jul 07 13:39:59 2014 -0500 @@ -1,6 +1,6 @@ @rem Used by the buildbot "test" step. rem The following line should be removed before #20035 is closed -set TCL_LIBRARY=%CD%\..\tcltk\lib\tcl8.6 -cd PCbuild -call rt.bat -d -q -uall -rwW -n --timeout=3600 %1 %2 %3 %4 %5 %6 %7 %8 %9 +set TCL_LIBRARY=%~dp0..\..\..\tcltk\lib\tcl8.6 + +"%~dp0..\..\PCbuild\python_d.exe" "%~dp0..\scripts\run_tests.py" -j 1 -u all -W --timeout=3600 %* diff -r e66c387da81b -r c3022e2606df Tools/scripts/run_tests.py --- a/Tools/scripts/run_tests.py Mon Jul 07 21:30:54 2014 +0300 +++ b/Tools/scripts/run_tests.py Mon Jul 07 13:39:59 2014 -0500 @@ -48,7 +48,11 @@ args.extend(['-u', 'all,-largefile,-audio,-gui']) args.extend(regrtest_args) print(' '.join(args)) - os.execv(sys.executable, args) + if sys.platform == 'win32': + from subprocess import call + call(args) + else: + os.execv(sys.executable, args) if __name__ == '__main__':