Fix pyodide#71: Upgrade to Python 3.7

Author: mdboom

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2
 defaults: &defaults
   working_directory: ~/repo
   docker:
-    - image: circleci/python:3.6.5-stretch-browsers
+    - image: circleci/python:3.7.0-stretch-browsers
 
 jobs:
   build:
@@ -27,11 +27,11 @@ jobs:
             sudo ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format
 
             sudo pip install virtualenv
-            
+
             virtualenv pyodide-env
 
             source pyodide-env/bin/activate
-            
+
             pip install pytest pytest-xdist pytest-instafail selenium PyYAML
 
             # Get recent version of Firefox and geckodriver

.editorconfig

@@ -16,3 +16,6 @@ indent_size = 4
 
 [Makefile]
 indent_style = tab
+
+[**.patch]
+trim_trailing_whitespace = false

Makefile

@@ -56,7 +56,7 @@ all: build/pyodide.asm.js \
 
 build/pyodide.asm.js: src/main.bc src/jsimport.bc src/jsproxy.bc src/js2python.bc \
 											src/pyimport.bc src/pyproxy.bc src/python2js.bc \
-											src/runpython.bc src/dummy_thread.bc src/hiwire.bc
+											src/runpython.bc src/hiwire.bc
 	[ -d build ] || mkdir build
 	$(CXX) -s EXPORT_NAME="'pyodide'" -o build/pyodide.asm.html $(filter %.bc,$^) \
 	  $(LDFLAGS) -s FORCE_FILESYSTEM=1
@@ -143,7 +143,7 @@ build/test.data: $(CPYTHONLIB)
 	)
 	( \
 		cd build; \
-		python $(FILEPACKAGER) test.data --preload ../$(CPYTHONLIB)/test@/lib/python3.6/test --js-output=test.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__ \
+		python $(FILEPACKAGER) test.data --preload ../$(CPYTHONLIB)/test@/lib/python3.7/test --js-output=test.js --export-name=pyodide --exclude \*.wasm.pre --exclude __pycache__ \
   )
 	uglifyjs build/test.js -o build/test.js
 

Makefile.envs

@@ -6,7 +6,7 @@ export EM_CACHE = $(PYODIDE_ROOT)/emsdk/emsdk/.emscripten_cache
 export EMSCRIPTEN = $(PYODIDE_ROOT)/emsdk/emsdk/emscripten/tag-1.38.10
 export BINARYEN_ROOT = $(PYODIDE_ROOT)/emsdk/emsdk/binaryen/tag-1.38.10_64bit_binaryen
 
-export PYVERSION=3.6.4
+export PYVERSION=3.7.0
 export PYMINOR=$(basename $(PYVERSION))
 export HOSTPYTHONROOT=$(PYODIDE_ROOT)/cpython/build/$(PYVERSION)/host
 export HOSTPYTHON=$(HOSTPYTHONROOT)/bin/python3

README.md

@@ -24,7 +24,7 @@ need to build it yourself prior.
 Additional build prerequisites are:
 
 - A working native compiler toolchain, enough to build CPython.
-- A native Python 3.6 or later to run the build scripts.
+- A native Python 3.7 to run the build scripts.
 - PyYAML
 - [lessc](https://lesscss.org/) to compile less to css.
 - [uglifyjs](https://github.com/mishoo/UglifyJS) to minify Javascript builds.

cpython/Makefile

@@ -15,6 +15,12 @@ URL=https://www.python.org/ftp/python/$(PYVERSION)/Python-$(PYVERSION).tgz
 LIB=libpython$(PYMINOR).a
 
 
+ZLIBVERSION = 1.2.11
+ZLIBTARBALL=$(ROOT)/downloads/zlib-$(ZLIBVERSION).tar.gz
+ZLIBBUILD=$(ROOT)/build/zlib-$(ZLIBVERSION)
+ZLIBURL=https://zlib.net/zlib-1.2.11.tar.gz
+
+
 all: $(INSTALL)/lib/$(LIB)
 
 
@@ -42,6 +48,11 @@ $(TARBALL):
 	md5sum --quiet --check checksums || (rm $@; false)
 
 
+$(ZLIBTARBALL):
+	[ -d $(ROOT)/downloads ] || mkdir $(ROOT)/downloads
+	wget -q -O $@ $(ZLIBURL)
+
+
 $(HOSTPYTHON) $(HOSTPGEN): $(TARBALL)
 	mkdir -p $(HOSTINSTALL)
 	[ -d $(HOSTBUILD) ] || tar -C $(HOSTINSTALL) -xf $(TARBALL)
@@ -61,7 +72,14 @@ $(BUILD)/.patched: $(TARBALL)
 	touch $@
 
 
-$(BUILD)/Makefile: $(BUILD)/.patched
+$(ZLIBBUILD)/.patched: $(ZLIBTARBALL)
+	[ -d $(ROOT)/build ] || (mkdir $(ROOT)/build)
+	tar -C $(ROOT)/build/ -xf $(ROOT)/downloads/zlib-1.2.11.tar.gz
+	cat patches/zlib/*.patch | (cd $(ZLIBBUILD) ; patch -p1)
+	touch $@
+
+
+$(BUILD)/Makefile: $(BUILD)/.patched $(ZLIBBUILD)/.patched
 	cp config.site $(BUILD)/
 	( \
 		cd $(BUILD); \
@@ -71,10 +89,6 @@ $(BUILD)/Makefile: $(BUILD)/.patched
 
 $(BUILD)/$(LIB): $(BUILD)/Makefile $(HOSTPYTHON) $(HOSTPGEN) Setup.local
 	cp Setup.local $(BUILD)/Modules/
-	( \
-		cd $(BUILD)/Modules/zlib; \
-		emconfigure ./configure --static \
-	)
 	( \
 		cp build/$(PYVERSION)/host/lib/python$(PYMINOR)/`$(HOSTPYTHON) -c "import sysconfig; print(sysconfig._get_sysconfigdata_name())"`.py build/$(PYVERSION)/Python-$(PYVERSION)/Lib/_sysconfigdata__emscripten_.py; \
 		cd $(BUILD); \

cpython/Setup.local

@@ -7,6 +7,7 @@
 array arraymodule.c	# array objects
 math mathmodule.c _math.c # -lm # math library functions, e.g. sin()
 cmath cmathmodule.c # complex math functions
+_contextvars _contextvarsmodule.c
 _struct _struct.c	# binary structure packing/unpacking
 _random _randommodule.c	# Random number generator
 _bisect _bisectmodule.c	# Bisection algorithms
@@ -21,10 +22,9 @@ parser parsermodule.c
 _socket socketmodule.c
 select selectmodule.c
 _posixsubprocess _posixsubprocess.c
-
 binascii binascii.c
 
-zlib zlibmodule.c -IModules/zlib
+zlib zlibmodule.c -I../../zlib-1.2.11
 
 pyexpat expat/xmlparse.c expat/xmlrole.c expat/xmltok.c pyexpat.c -I$(srcdir)/Modules/expat -DHAVE_EXPAT_CONFIG_H -DUSE_PYEXPAT_CAPI -DXML_POOR_ENTROPY
 
@@ -33,7 +33,9 @@ _sha256 sha256module.c
 _sha512 sha512module.c
 _sha3 _sha3/sha3module.c
 _md5 md5module.c
-_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.6.4/Modules/_blake2/blake2s_impl.c
+_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c ../../host/Python-3.7.0/Modules/_blake2/blake2s_impl.c
+
+_queue _queuemodule.c
 
 #future_builtins future_builtins.c
 

cpython/checksums

@@ -1 +1 @@
-9de6494314ea199e3633211696735f65 ./downloads/Python-3.6.4.tgz
+41b6595deb4147a1ed517a7d9a580271  downloads/Python-3.7.0.tgz

cpython/config.site

@@ -2,4 +2,5 @@ ac_cv_file__dev_ptmx=no
 ac_cv_file__dev_ptc=no
 ac_cv_func_dlopen=yes
 ac_cv_lib_socket_socket=no
-ac_cv_little_endian_double=yes
+ac_cv_header_linux_vm_sockets_h=yes
+ac_cv_little_endian_double=yes

cpython/patches/add-emscripten-host.patch

@@ -1,8 +1,39 @@
+diff --git a/config.sub b/config.sub
+index ba37cf99e2..d772f33065 100755
+--- a/config.sub
++++ b/config.sub
+@@ -118,7 +118,8 @@ case $maybe_os in
+   linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
+   knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \
+   kopensolaris*-gnu* | cloudabi*-eabi* | \
+-  storm-chaos* | os2-emx* | rtmk-nova*)
++  storm-chaos* | os2-emx* | rtmk-nova* | \
++  emscripten)
+     os=-$maybe_os
+     basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
+     ;;
+@@ -377,6 +378,7 @@ case $basic_machine in
+ 	| alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \
+ 	| alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \
+ 	| arm-*  | armbe-* | armle-* | armeb-* | armv*-* \
++  | asmjs \
+ 	| avr-* | avr32-* \
+ 	| ba-* \
+ 	| be32-* | be64-* \
+@@ -1522,6 +1524,8 @@ case $os in
+ 			;;
+ 		esac
+ 		;;
++	-emscripten)
++		;;
+ 	-nacl*)
+ 		;;
+ 	-ios)
 diff --git a/configure b/configure
-index 8cf777e..263719f 100755
+index e0389649ae..e6ab1613cf 100755
 --- a/configure
 +++ b/configure
-@@ -3202,6 +3202,9 @@ then
+@@ -3257,6 +3257,9 @@ then
  	*-*-cygwin*)
  		ac_sys_system=Cygwin
  		;;
@@ -12,7 +43,7 @@ index 8cf777e..263719f 100755
  	*)
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
-@@ -3248,6 +3251,9 @@ if test "$cross_compiling" = yes; then
+@@ -3301,6 +3304,9 @@ if test "$cross_compiling" = yes; then
  	*-*-cygwin*)
  		_host_cpu=
  		;;
@@ -23,10 +54,10 @@ index 8cf777e..263719f 100755
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
 diff --git a/configure.ac b/configure.ac
-index 78fe3c7..d7665b4 100644
+index b5beb08574..8051e5a8ed 100644
 --- a/configure.ac
 +++ b/configure.ac
-@@ -322,6 +322,9 @@ then
+@@ -379,6 +379,9 @@ then
  	*-*-cygwin*)
  		ac_sys_system=Cygwin
  		;;
@@ -36,7 +67,7 @@ index 78fe3c7..d7665b4 100644
  	*)
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
-@@ -368,6 +371,9 @@ if test "$cross_compiling" = yes; then
+@@ -423,6 +426,9 @@ if test "$cross_compiling" = yes; then
  	*-*-cygwin*)
  		_host_cpu=
  		;;
@@ -46,34 +77,3 @@ index 78fe3c7..d7665b4 100644
  	*)
  		# for now, limit cross builds to known configurations
  		MACHDEP="unknown"
-diff --git a/config.sub b/config.sub
-index d654d03..0d8236f 100755
---- a/config.sub
-+++ b/config.sub
-@@ -119,7 +119,8 @@ case $maybe_os in
-   linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \
-   knetbsd*-gnu* | netbsd*-gnu* | \
-   kopensolaris*-gnu* | cloudabi*-eabi* | \
--  storm-chaos* | os2-emx* | rtmk-nova*)
-+  storm-chaos* | os2-emx* | rtmk-nova* | \
-+  emscripten)
-     os=-$maybe_os
-     basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`
-     ;;
-@@ -254,6 +255,7 @@ case $basic_machine in
- 	| am33_2.0 \
- 	| arc | arceb \
- 	| arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \
-+	| asmjs \
- 	| avr | avr32 \
- 	| be32 | be64 \
- 	| bfin \
-@@ -1510,6 +1512,8 @@ case $os in
- 	-dicos*)
- 		os=-dicos
- 		;;
-+	-emscripten)
-+		;;
- 	-nacl*)
- 		;;
- 	-none)

cpython/patches/disable-set_inheritable.patch

@@ -1,24 +1,22 @@
-*** a/Python/fileutils.c	2016-10-25 02:01:16.270148214 +0100
---- b/Python/fileutils.c	2016-10-25 02:01:27.086148626 +0100
-***************
-*** 785,790 ****
---- 785,793 ----
-  static int
-  set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
-  {
-+ #ifdef EMSCRIPTEN
-+     return 0;
-+ #else
-  #ifdef MS_WINDOWS
-      HANDLE handle;
-      DWORD flags;
-***************
-*** 897,902 ****
---- 900,906 ----
-      }
-      return 0;
-  #endif
-+ #endif
-  }
-  
-  /* Make the file descriptor non-inheritable.
+diff --git a/Python/fileutils.c b/Python/fileutils.c
+index 35869c81ac..1af4ead7f5 100644
+--- a/Python/fileutils.c
++++ b/Python/fileutils.c
+@@ -964,6 +964,9 @@ _Py_get_inheritable(int fd)
+ static int
+ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
+ {
++#ifdef EMSCRIPTEN
++    return 0;
++#else
+ #ifdef MS_WINDOWS
+     HANDLE handle;
+     DWORD flags;
+@@ -1080,6 +1083,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works)
+     }
+     return 0;
+ #endif
++#endif
+ }
+ 
+ /* Make the file descriptor non-inheritable.

cpython/patches/no_af_vsock.patch

@@ -0,0 +1,13 @@
+diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h
+index fdb4e871ce..c990003ff3 100644
+--- a/Modules/socketmodule.h
++++ b/Modules/socketmodule.h
+@@ -144,6 +144,8 @@ typedef int socklen_t;
+ extern "C" {
+ #endif
+ 
++#undef AF_VSOCK
++
+ /* Python module and C API name */
+ #define PySocket_MODULE_NAME    "_socket"
+ #define PySocket_CAPI_NAME      "CAPI"

cpython/patches/testing.patch

@@ -1,5 +1,5 @@
 diff --git a/Lib/platform.py b/Lib/platform.py
-index cc2db9870d..ac4e3c538f 100755
+index 20f9817f4f..bcf96874c0 100755
 --- a/Lib/platform.py
 +++ b/Lib/platform.py
 @@ -748,7 +748,7 @@ def _syscmd_uname(option, default=''):
@@ -21,35 +21,35 @@ index cc2db9870d..ac4e3c538f 100755
          return default
      target = _follow_symlinks(target)
 diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
-index ca5f9c20dd..97934039ee 100644
+index 64b25aab80..7cf67fd149 100644
 --- a/Lib/test/support/script_helper.py
 +++ b/Lib/test/support/script_helper.py
-@@ -11,6 +11,7 @@ import subprocess
+@@ -9,6 +9,7 @@ import os.path
+ import subprocess
  import py_compile
- import contextlib
- import shutil
-+import unittest
  import zipfile
++import unittest
 
  from importlib.util import source_from_cache
-@@ -37,6 +38,8 @@ def interpreter_requires_environment():
+ from test.support import make_legacy_pyc, strip_python_stderr
+@@ -34,6 +35,8 @@ def interpreter_requires_environment():
      situation.  PYTHONPATH or PYTHONUSERSITE are other common environment
      variables that might impact whether or not the interpreter can start.
      """
 +    raise unittest.SkipTest('no subprocess')
 +
      global __cached_interp_requires_environment
      if __cached_interp_requires_environment is None:
-         # Try running an interpreter with -E to see if it works or not.
-@@ -165,6 +168,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
+         # If PYTHONHOME is set, assume that we need it
+@@ -172,6 +175,8 @@ def spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw):
      kw is extra keyword args to pass to subprocess.Popen. Returns a Popen
      object.
      """
 +    raise unittest.SkipTest("no subprocess")
 +
-     cmd_line = [sys.executable, '-E']
-     cmd_line.extend(args)
-     # Under Fedora (?), GNU readline can output junk on stderr when initialized,
+     cmd_line = [sys.executable]
+     if not interpreter_requires_environment():
+         cmd_line.append('-E')
 diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py
 index 55faf4c427..b2201c09e7 100644
 --- a/Lib/test/test_code.py
@@ -67,7 +67,7 @@ index 55faf4c427..b2201c09e7 100644
  import weakref
  try:
 diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
-index b73a96f757..0d1c411f9a 100644
+index 1fc4de11e1..e6c91707ae 100644
 --- a/Lib/test/test_import/__init__.py
 +++ b/Lib/test/test_import/__init__.py
 @@ -10,7 +10,10 @@ import py_compile