bpo-40280: Add --with-emscripten-target to build for browser or node (GH-30552)

Co-authored-by: Ethan Smith <[email protected]>

Author: tiran

Makefile.pre.in

@@ -246,6 +246,10 @@ SRCDIRS= 	@SRCDIRS@
 # Other subdirectories
 SUBDIRSTOO=	Include Lib Misc
 
+# assets for Emscripten browser builds
+WASM_ASSETS_DIR=".$(prefix)"
+WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
+
 # Files and directories to be distributed
 CONFIGFILES=	configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in
 DISTFILES=	README.rst ChangeLog $(CONFIGFILES)
@@ -601,6 +605,7 @@ LIBEXPAT_HEADERS= \
 all:		@DEF_MAKE_ALL_RULE@
 build_all:	check-clean-src $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks \
 		Programs/_testembed python-config
+build_platform: check-clean-src $(BUILDPYTHON) platform
 
 # Check that the source is clean when building out of source.
 check-clean-src:
@@ -833,19 +838,12 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS)
 # wasm32-emscripten build
 # wasm assets directory is relative to current build dir, e.g. "./usr/local".
 # --preload-file turns a relative asset path into an absolute path.
-WASM_ASSETS_DIR=".$(prefix)"
-WASM_STDLIB="$(WASM_ASSETS_DIR)/local/lib/python$(VERSION)/os.py"
 
 $(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \
                 pybuilddir.txt $(srcdir)/Tools/wasm/wasm_assets.py
 	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \
 	    --builddir . --prefix $(prefix)
 
-python.html: Programs/python.o $(LIBRARY_DEPS) $(WASM_STDLIB)
-	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o \
-	    $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) \
-	    -s ASSERTIONS=1 --preload-file $(WASM_ASSETS_DIR)
-
 ##########################################################################
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=$(PY_STDMODULE_CFLAGS) $(CCSHARED) @LIBMPDEC_CFLAGS@
@@ -2396,7 +2394,7 @@ clean-retain-profile: pycremoval
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
 	-rm -f _bootstrap_python
-	-rm -f python.html python.js python.data
+	-rm -f python.html python*.js python.data
 	-rm -f Programs/_testembed Programs/_freeze_module
 	-rm -f Python/deepfreeze/*.[co]
 	-rm -f Python/frozen_modules/*.h

Misc/NEWS.d/next/Build/2022-01-12-10-22-23.bpo-40280.5maBz8.rst

@@ -0,0 +1,2 @@
+The ``configure`` script has a new option ``--with-emscripten-target`` to
+select browser or node as Emscripten build target.

Modules/socketmodule.c

@@ -7931,7 +7931,7 @@ PyInit__socket(void)
 #ifdef  IPPROTO_VRRP
     PyModule_AddIntMacro(m, IPPROTO_VRRP);
 #endif
-#ifdef  IPPROTO_SCTP
+#if defined(IPPROTO_SCTP) && !defined(__EMSCRIPTEN__)
     PyModule_AddIntMacro(m, IPPROTO_SCTP);
 #endif
 #ifdef  IPPROTO_BIP

Tools/wasm/README.md

@@ -27,6 +27,8 @@ embuilder build zlib
 
 ### Cross compile to wasm32-emscripten
 
+For browser:
+
 ```shell
 mkdir -p builddir/emscripten
 pushd builddir/emscripten
@@ -35,9 +37,23 @@ CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
   emconfigure ../../configure -C \
     --host=wasm32-unknown-emscripten \
     --build=$(../../config.guess) \
+    --with-emscripten-target=browser \
+    --with-build-python=$(pwd)/../build/python
+
+emmake make -j$(nproc)
+```
+
+For node:
+
+```
+CONFIG_SITE=../../Tools/wasm/config.site-wasm32-emscripten \
+  emconfigure ../../configure -C \
+    --host=wasm32-unknown-emscripten \
+    --build=$(../../config.guess) \
+    --with-emscripten-target=node \
     --with-build-python=$(pwd)/../build/python
 
-emmake make -j$(nproc) python.html
+emmake make -j$(nproc)
 ```
 
 ### Test in browser

Tools/wasm/config.site-wasm32-emscripten

@@ -30,6 +30,11 @@ ac_cv_func_shutdown=no
 # breaks build, see https://github.com/ethanhs/python-wasm/issues/16
 ac_cv_lib_bz2_BZ2_bzCompress=no
 
+# clock_nanosleep() causes time.sleep() to sleep forever.
+# nanosleep() works correctly
+ac_cv_func_clock_nanosleep=no
+ac_cv_lib_rt_clock_nanosleep=no
+
 # The rest is based on pyodide
 # https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h
 

configure

@@ -846,6 +846,8 @@ SHLIB_SUFFIX
 LIBTOOL_CRUFT
 OTHER_LIBTOOL_OPT
 UNIVERSAL_ARCH_FLAGS
+WASM_STDLIB
+WASM_ASSETS_DIR
 LDFLAGS_NOLTO
 LDFLAGS_NODIST
 CFLAGS_NODIST
@@ -1002,6 +1004,7 @@ with_universal_archs
 with_framework_name
 enable_framework
 with_cxx_main
+with_emscripten_target
 with_suffix
 enable_shared
 enable_profiling
@@ -1751,6 +1754,8 @@ Optional Packages:
   --with-cxx-main[=COMPILER]
                           compile main() and link Python executable with C++
                           compiler specified in COMPILER (default is $CXX)
+  --with-emscripten-target=[browser|node]
+                          Emscripten platform
   --with-suffix=SUFFIX    set executable suffix to SUFFIX (default is empty,
                           yes is mapped to '.exe')
   --with-pydebug          build with Py_DEBUG defined (default is no)
@@ -6205,6 +6210,41 @@ case $ac_sys_system/$ac_sys_release in #(
      ;;
 esac
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-emscripten-target" >&5
+$as_echo_n "checking for --with-emscripten-target... " >&6; }
+
+# Check whether --with-emscripten-target was given.
+if test "${with_emscripten_target+set}" = set; then :
+  withval=$with_emscripten_target;
+	if test "x$ac_sys_system" = xEmscripten; then :
+
+    case $with_emscripten_target in #(
+  browser) :
+    ac_sys_emscripten_target=browser ;; #(
+  node) :
+    ac_sys_emscripten_target=node ;; #(
+  *) :
+    as_fn_error $? "Invalid argument: --with-emscripten-target=browser|node" "$LINENO" 5
+     ;;
+esac
+
+else
+
+    as_fn_error $? "--with-emscripten-target only applies to Emscripten" "$LINENO" 5
+
+fi
+
+else
+
+  if test "x$ac_sys_system" = xEmscripten; then :
+  ac_sys_emscripten_target=browser
+fi
+
+fi
+
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_sys_emscripten_target" >&5
+$as_echo "$ac_sys_emscripten_target" >&6; }
+
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for --with-suffix" >&5
 $as_echo_n "checking for --with-suffix... " >&6; }
 
@@ -6223,8 +6263,12 @@ esac
 
 else
 
-  case $ac_sys_system in #(
-  Emscripten) :
+  case $ac_sys_system/$ac_sys_emscripten_target in #(
+  Emscripten/browser) :
+    EXEEXT=.html ;; #(
+  Emscripten/node) :
+    EXEEXT=.js ;; #(
+  wasi/*) :
     EXEEXT=.wasm ;; #(
   *) :
     EXEEXT=
@@ -7003,6 +7047,7 @@ else
 $as_echo "no" >&6; }
 fi
 
+
 if test "$Py_OPT" = 'true' ; then
   # Intentionally not forcing Py_LTO='true' here.  Too many toolchains do not
   # compile working code using it and both test_distutils and test_gdb are
@@ -7053,8 +7098,10 @@ fi
 
       ;;
   esac
-
-
+elif test "$ac_sys_system" = "Emscripten"; then
+      DEF_MAKE_ALL_RULE="build_platform"
+  REQUIRE_PGO="no"
+  DEF_MAKE_RULE="all"
 else
   DEF_MAKE_ALL_RULE="build_all"
   REQUIRE_PGO="no"
@@ -7567,6 +7614,25 @@ then
     esac
 fi
 
+# WASM flags
+case $ac_sys_system/$ac_sys_emscripten_target in #(
+  Emscripten/browser) :
+
+    LDFLAGS_NODIST="$(LDFLAGS_NODIST) -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 --preload-file \$(WASM_ASSETS_DIR)"
+    WASM_ASSETS_DIR=".\$(prefix)"
+    WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
+   ;; #(
+  Emscripten/node) :
+
+    LDFLAGS_NODIST="$(LDFLAGS_NODIST) -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s EXIT_RUNTIME=1 -s USE_PTHREADS -s PROXY_TO_PTHREAD"
+    CFLAGS_NODIST="$(CFLAGS_NODIST) -pthread"
+   ;; #(
+  *) :
+     ;;
+esac
+
+
+
 
 
 
@@ -21170,6 +21236,14 @@ else
     LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
 fi
 
+case $ac_sys_system/$ac_sys_emscripten_target in #(
+  Emscripten/browser) :
+    LIBRARY_DEPS="$LIBRARY_DEPS \$(WASM_STDLIB)" ;; #(
+  *) :
+     ;;
+esac
+
+
 
 
 # Check whether to disable test modules. Once set, setup.py will not build
@@ -23458,7 +23532,7 @@ $as_echo_n "checking for stdlib extension module xxlimited... " >&6; }
   *xxlimited*) :
     py_cv_module_xxlimited=n/a ;; #(
   *) :
-    if test "$with_trace_refs" = "no"; then :
+    if test "$with_trace_refs" = "no" -a "$ac_sys_system" != "Emscripten"; then :
   if true; then :
   py_cv_module_xxlimited=yes
 else
@@ -23494,7 +23568,7 @@ $as_echo_n "checking for stdlib extension module xxlimited_35... " >&6; }
   *xxlimited_35*) :
     py_cv_module_xxlimited_35=n/a ;; #(
   *) :
-    if test "$with_trace_refs" = "no"; then :
+    if test "$with_trace_refs" = "no" -a "$ac_sys_system" != "Emscripten"; then :
   if true; then :
   py_cv_module_xxlimited_35=yes
 else

configure.ac

@@ -1062,6 +1062,24 @@ AS_CASE([$ac_sys_system/$ac_sys_release],
  ]
 )
 
+AC_MSG_CHECKING([for --with-emscripten-target])
+AC_ARG_WITH([emscripten-target],
+  [AS_HELP_STRING([--with-emscripten-target=@<:@browser|node@:>@], [Emscripten platform])],
+[
+	AS_VAR_IF([ac_sys_system], [Emscripten], [
+    AS_CASE([$with_emscripten_target],
+      [browser], [ac_sys_emscripten_target=browser],
+      [node], [ac_sys_emscripten_target=node],
+      [AC_MSG_ERROR([Invalid argument: --with-emscripten-target=browser|node])]
+    )
+  ], [
+    AC_MSG_ERROR([--with-emscripten-target only applies to Emscripten])
+  ])
+], [
+  AS_VAR_IF([ac_sys_system], [Emscripten], [ac_sys_emscripten_target=browser])
+])
+AC_MSG_RESULT([$ac_sys_emscripten_target])
+
 AC_MSG_CHECKING([for --with-suffix])
 AC_ARG_WITH([suffix],
             [AS_HELP_STRING([--with-suffix=SUFFIX], [set executable suffix to SUFFIX (default is empty, yes is mapped to '.exe')])],
@@ -1072,8 +1090,10 @@ AC_ARG_WITH([suffix],
     [EXEEXT=$with_suffix]
   )
 ], [
-  AS_CASE([$ac_sys_system],
-    [Emscripten], [EXEEXT=.wasm],
+  AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
+    [Emscripten/browser], [EXEEXT=.html],
+    [Emscripten/node], [EXEEXT=.js],
+    [wasi/*], [EXEEXT=.wasm],
     [EXEEXT=]
   )
 ])
@@ -1446,6 +1466,7 @@ else
   AC_MSG_RESULT(no);
 fi],
 [AC_MSG_RESULT(no)])
+
 if test "$Py_OPT" = 'true' ; then
   # Intentionally not forcing Py_LTO='true' here.  Too many toolchains do not
   # compile working code using it and both test_distutils and test_gdb are
@@ -1462,8 +1483,12 @@ if test "$Py_OPT" = 'true' ; then
       ])
       ;;
   esac
-
-
+elif test "$ac_sys_system" = "Emscripten"; then
+  dnl Emscripten does not support shared extensions yet. Build
+  dnl "python.[js,html,wasm]", "pybuilddir.txt", and "platform" files.
+  DEF_MAKE_ALL_RULE="build_platform"
+  REQUIRE_PGO="no"
+  DEF_MAKE_RULE="all"
 else
   DEF_MAKE_ALL_RULE="build_all"
   REQUIRE_PGO="no"
@@ -1769,10 +1794,25 @@ then
     esac
 fi
 
+# WASM flags
+AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
+  [Emscripten/browser], [
+    LDFLAGS_NODIST="$(LDFLAGS_NODIST) -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 --preload-file \$(WASM_ASSETS_DIR)"
+    WASM_ASSETS_DIR=".\$(prefix)"
+    WASM_STDLIB="\$(WASM_ASSETS_DIR)/local/lib/python\$(VERSION)/os.py"
+  ],
+  [Emscripten/node], [
+    LDFLAGS_NODIST="$(LDFLAGS_NODIST) -s ASSERTIONS=1 -s ALLOW_MEMORY_GROWTH=1 -s NODERAWFS=1 -s EXIT_RUNTIME=1 -s USE_PTHREADS -s PROXY_TO_PTHREAD"
+    CFLAGS_NODIST="$(CFLAGS_NODIST) -pthread"
+  ],
+)
+
 AC_SUBST(BASECFLAGS)
 AC_SUBST(CFLAGS_NODIST)
 AC_SUBST(LDFLAGS_NODIST)
 AC_SUBST(LDFLAGS_NOLTO)
+AC_SUBST([WASM_ASSETS_DIR])
+AC_SUBST([WASM_STDLIB])
 
 # The -arch flags for universal builds on macOS
 UNIVERSAL_ARCH_FLAGS=
@@ -6252,6 +6292,12 @@ if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
 else
     LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
 fi
+
+dnl browser needs a WASM assets stdlib bundle
+AS_CASE([$ac_sys_system/$ac_sys_emscripten_target],
+  [Emscripten/browser], [LIBRARY_DEPS="$LIBRARY_DEPS \$(WASM_STDLIB)"],
+)
+
 AC_SUBST(STATIC_LIBPYTHON)
 AC_SUBST(LIBRARY_DEPS)
 
@@ -6520,8 +6566,9 @@ PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [], [], [-lm])
 
 dnl Limited API template modules.
 dnl The limited C API is not compatible with the Py_TRACE_REFS macro.
-PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no"])
-PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no"])
+dnl Emscripten does not support shared libraries yet.
+PY_STDLIB_MOD([xxlimited], [test "$with_trace_refs" = "no" -a "$ac_sys_system" != "Emscripten"])
+PY_STDLIB_MOD([xxlimited_35], [test "$with_trace_refs" = "no" -a "$ac_sys_system" != "Emscripten"])
 
 # substitute multiline block, must come after last PY_STDLIB_MOD()
 AC_SUBST([MODULE_BLOCK])