bpo-32726: Build macOS 10.9+ installer with private copy of Tcl/Tk 8.6

Author: ned-deily

Mac/BuildScript/build-installer.py

@@ -189,6 +189,11 @@ def getTargetCompilers():
 #                       '/Library/Frameworks/Tk.framework/Versions/8.5/Tk']
 EXPECTED_SHARED_LIBS = {}
 
+# Are we building and linking with our own copy of Tcl/TK?
+#   For now, do so if deployment target is 10.9+.
+def internalTk():
+    return getDeptargetTuple() >= (10, 9)
+
 # List of names of third party software built with this installer.
 # The names will be inserted into the rtf version of the License.
 THIRD_PARTY_LIBS = []
@@ -217,13 +222,12 @@ def library_recipes():
           ),
     ])
 
-#   Disable for now
-    if False:   # if getDeptargetTuple() > (10, 5):
+    if internalTk():
         result.extend([
           dict(
-              name="Tcl 8.5.15",
-              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tcl8.5.15-src.tar.gz",
-              checksum='f3df162f92c69b254079c4d0af7a690f',
+              name="Tcl 8.6.7",
+              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.7-src.tar.gz",
+              checksum='5673aaf45b5de5d8dd80bb3daaeb8838',
               buildDir="unix",
               configure_pre=[
                     '--enable-shared',
@@ -233,16 +237,13 @@ def library_recipes():
               useLDFlags=False,
               install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
                   "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
-                  "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())),
+                  "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
                   },
               ),
           dict(
-              name="Tk 8.5.15",
-              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_5/tk8.5.15-src.tar.gz",
-              checksum='55b8e33f903210a4e1c8bce0f820657f',
-              patches=[
-                  "issue19373_tk_8_5_15_source.patch",
-                   ],
+              name="Tk 8.6.7",
+              url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.7-src.tar.gz",
+              checksum='46ea9c0165c515d87393700f4891ab6f',
               buildDir="unix",
               configure_pre=[
                     '--enable-aqua',
@@ -253,8 +254,8 @@ def library_recipes():
               useLDFlags=False,
               install='make TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s TK_LIBRARY=%(TK_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
                   "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
-                  "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.5'%(getVersion())),
-                  "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.5'%(getVersion())),
+                  "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
+                  "TK_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tk8.6'%(getVersion())),
                   },
                 ),
         ])
@@ -547,42 +548,45 @@ def checkEnvironment():
         fatal("This script should be run on a macOS 10.5 (or later) system")
 
     # Because we only support dynamic load of only one major/minor version of
+    # Tcl/Tk, if we are not using building and using our own private copy of
     # Tcl/Tk, ensure:
     # 1. there is a user-installed framework (usually ActiveTcl) in (or linked
     #       in) SDKROOT/Library/Frameworks.  As of Python 3.7.0, we no longer
     #       enforce that the version of the user-installed framework also
     #       exists in the system-supplied Tcl/Tk frameworks.  Time to support
     #       Tcl/Tk 8.6 even if Apple does not.
-
-    frameworks = {}
-    for framework in ['Tcl', 'Tk']:
-        fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework
-        libfw = os.path.join('/', fwpth)
-        usrfw = os.path.join(os.getenv('HOME'), fwpth)
-        frameworks[framework] = os.readlink(libfw)
-        if not os.path.exists(libfw):
-            fatal("Please install a link to a current %s %s as %s so "
-                    "the user can override the system framework."
-                    % (framework, frameworks[framework], libfw))
-        if os.path.exists(usrfw):
-            fatal("Please rename %s to avoid possible dynamic load issues."
-                    % usrfw)
-
-    if frameworks['Tcl'] != frameworks['Tk']:
-        fatal("The Tcl and Tk frameworks are not the same version.")
-
-    print(" -- Building with Tcl/Tk %s frameworks"
-                % frameworks['Tk'])
+    if not internalTk():
+        frameworks = {}
+        for framework in ['Tcl', 'Tk']:
+            fwpth = 'Library/Frameworks/%s.framework/Versions/Current' % framework
+            libfw = os.path.join('/', fwpth)
+            usrfw = os.path.join(os.getenv('HOME'), fwpth)
+            frameworks[framework] = os.readlink(libfw)
+            if not os.path.exists(libfw):
+                fatal("Please install a link to a current %s %s as %s so "
+                        "the user can override the system framework."
+                        % (framework, frameworks[framework], libfw))
+            if os.path.exists(usrfw):
+                fatal("Please rename %s to avoid possible dynamic load issues."
+                        % usrfw)
+
+        if frameworks['Tcl'] != frameworks['Tk']:
+            fatal("The Tcl and Tk frameworks are not the same version.")
+
+        print(" -- Building with external Tcl/Tk %s frameworks"
+                    % frameworks['Tk'])
+
+        # add files to check after build
+        EXPECTED_SHARED_LIBS['_tkinter.so'] = [
+                "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl"
+                    % frameworks['Tcl'],
+                "/Library/Frameworks/Tk.framework/Versions/%s/Tk"
+                    % frameworks['Tk'],
+                ]
+    else:
+        print(" -- Building private copy of Tcl/Tk")
     print("")
 
-    # add files to check after build
-    EXPECTED_SHARED_LIBS['_tkinter.so'] = [
-            "/Library/Frameworks/Tcl.framework/Versions/%s/Tcl"
-                % frameworks['Tcl'],
-            "/Library/Frameworks/Tk.framework/Versions/%s/Tk"
-                % frameworks['Tk'],
-            ]
-
     # Remove inherited environment variables which might influence build
     environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_',
                             'LD_', 'LIBRARY_', 'PATH', 'PYTHON']
@@ -1086,12 +1090,18 @@ def buildPython():
                "--with-universal-archs=%s "
                "%s "
                "%s "
+               "%s "
+               "%s "
                "LDFLAGS='-g -L%s/libraries/usr/local/lib' "
                "CFLAGS='-g -I%s/libraries/usr/local/include' 2>&1"%(
         shellQuote(os.path.join(SRCDIR, 'configure')),
         UNIVERSALARCHS,
         (' ', '--with-computed-gotos ')[PYTHON_3],
         (' ', '--without-ensurepip ')[PYTHON_3],
+        (' ', "--with-tcltk-includes='-I%s/libraries/usr/local/include'"%(
+                            shellQuote(WORKDIR)[1:-1],))[internalTk()],
+        (' ', "--with-tcltk-libs='-L%s/libraries/usr/local/lib -ltcl8.6 -ltk8.6'"%(
+                            shellQuote(WORKDIR)[1:-1],))[internalTk()],
         shellQuote(WORKDIR)[1:-1],
         shellQuote(WORKDIR)[1:-1]))
 
@@ -1126,14 +1136,22 @@ def buildPython():
     del os.environ['DYLD_LIBRARY_PATH']
     print("Copying required shared libraries")
     if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')):
-        runCommand("mv %s/* %s"%(
-            shellQuote(os.path.join(
+        build_lib_dir = os.path.join(
                 WORKDIR, 'libraries', 'Library', 'Frameworks',
-                'Python.framework', 'Versions', getVersion(),
-                'lib')),
-            shellQuote(os.path.join(WORKDIR, '_root', 'Library', 'Frameworks',
-                'Python.framework', 'Versions', getVersion(),
-                'lib'))))
+                'Python.framework', 'Versions', getVersion(), 'lib')
+        fw_lib_dir = os.path.join(
+                WORKDIR, '_root', 'Library', 'Frameworks',
+                'Python.framework', 'Versions', getVersion(), 'lib')
+        if internalTk():
+            # move Tcl and Tk pkgconfig files
+            runCommand("mv %s/pkgconfig/* %s/pkgconfig"%(
+                        shellQuote(build_lib_dir),
+                        shellQuote(fw_lib_dir) ))
+            runCommand("rm -r %s/pkgconfig"%(
+                        shellQuote(build_lib_dir), ))
+        runCommand("mv %s/* %s"%(
+                    shellQuote(build_lib_dir),
+                    shellQuote(fw_lib_dir) ))
 
     frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework')
     frmDirVersioned = os.path.join(frmDir, 'Versions', version)

Mac/BuildScript/issue19373_tk_8_5_15_source.patch

@@ -31,6 +31,20 @@ For Python.3.7, python.org currently provides two installer variants for downloa
 \i0  variant.  Use the 10.6 variant if you are running on macOS 10.6 through 10.8 or if you want to produce standalone applications that can run on systems from 10.6.  If you are running on macOS 10.9 or later and if you have no need for compatibility with older systems, use the 10.9 variant.  The Pythons installed by these installers are built with private copies of some third-party libraries not included with or newer than those in macOS itself.  The list of these libraries varies by installer variant and is included at the end of the License.rtf file.
 \b \ul \
 \
+Using IDLE or other Tk applications [NEW/CHANGED in 3.7.0b1] 
+\b0 \ulnone \
+\
+The 10.9+ installer variant comes with its own private version of Tcl/Tk 8.6. It does not use system-supplied or third-party supplied versions of Tcl/Tk.\
+\
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
+\cf0 For the 10.6+ variant, you continue to need to install a newer third-party version of the 
+\i Tcl/Tk
+\i0  8.5 (not 8.6) frameworks to use IDLE or other programs that use the Tkinter graphical user interface toolkit.  Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of 
+\i Tcl/Tk
+\i0  for this version of Python and of macOS.\
+\pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0
+
+\b \cf0 \ul \ulc0 \
 Certificate verification and OpenSSL \{UPDATED in 3.7.0b1]\
 
 \b0 \ulnone \
@@ -53,17 +67,7 @@ The bundled
 \f0  included with this installer has its own default certificate store for verifying download connections.\
 \
 
-\b \ul Update your version of Tcl/Tk to use IDLE or other Tk applications [CHANGED in 3.7.0b1] 
-\b0 \ulnone \
-\
-To use IDLE or other programs that use the Tkinter graphical user interface toolkit, you need to install a newer third-party version of the 
-\i Tcl/Tk
-\i0  frameworks.  Visit {\field{\*\fldinst{HYPERLINK "https://www.python.org/download/mac/tcltk/"}}{\fldrslt https://www.python.org/download/mac/tcltk/}} for current information about supported and recommended versions of 
-\i Tcl/Tk
-\i0  for this version of Python and of Mac OS X.  For 3.7.0b1, the 10.9 installer variant is linked with Tcl/Tk 8.6 which you must install separately.   The 10.6 installer links /with Tcl/Tk 8.5 which you should also download and install.\
-
-\b \ul \
-Other changes\
+\b \ul Other changes\
 
 \b0 \ulnone \
 For other changes in this release, see the 

Mac/BuildScript/resources/ReadMe.rtf

@@ -23,7 +23,7 @@
 \
 
 \b NEW in 3.7.0b1: 
-\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), Tcl/Tk 8.6 support in the 10.9+ variant, OpenSSL 1.1.0g, and more!\
+\b0 two installer variants (10.9+ 64-bit-only, 10.6+ 64-/32-bit), built-in Tcl/Tk 8.6 support in the 10.9+ variant (no additional third-party downloads!), OpenSSL 1.1.0g, and more!\
 \
 
 \b IMPORTANT:

Mac/BuildScript/resources/Welcome.rtf

@@ -1,4 +1,4 @@
 Provide an additional, more modern macOS installer variant that supports
 macOS 10.9+ systems in 64-bit mode only.  Upgrade the supplied third-party
 libraries to OpenSSL 1.1.0g and to SQLite 3.22.0.  The 10.9+ installer now
-links with Tcl/Tk 8.6.
+links with and supplies its own copy of Tcl/Tk 8.6.