Integrate the stubprocess.py wrapper from Parts

CHANGES.txt

@@ -12,6 +12,9 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
 
   From Dirk Baechle:
     - Added Docker images for building and testing SCons. (issue #3585)
+    - Integrated stubprocess.py wrapper from the Parts project in order
+      to speed up large builds. It's available as an experimental option
+      '--stubprocess-wrapper' for now.
 
   From James Benton:
     - Improve Visual Studio solution/project generation code to add support

RELEASE.txt

@@ -20,6 +20,12 @@
     - Add CompilationDatabase() builder in compilation_db tool. Contributed by MongoDB.
       Setting COMPILATIONDB_USE_ABSPATH to True|False controls whether the files are absolute or relative
       paths.  Address Issue #3693 and #3694 found during development.
+    - Added a new command-line option "--stubprocess-wrapper", which will try to wrap the
+      subprocess.Popen() call when activated. Enabling this option can give a significant speedup
+      for the build of large projects that consume a lot of memory and create many files.
+      Small builds however may experience a slow-down, which is why we declared this option
+      an "experimental feature" for the moment. Please also refer to the corresponding entry
+      in the MAN page for more information.
 
 
   DEPRECATED FUNCTIONALITY

SCons/Platform/__init__.py

@@ -54,6 +54,9 @@
 import SCons.Subst
 import SCons.Tool
 
+# Gets set to True in Main._main() when the '--stubprocess-wrapper'
+# option was given on the command-line for speeding up large builds.
+stubprocess_wrapper = False
 
 def platform_default():
     r"""Return the platform string for our execution environment.

SCons/Platform/posix.py

@@ -40,10 +40,31 @@
 import select
 
 import SCons.Util
-from SCons.Platform import TempFileMunge
+from SCons.Platform import TempFileMunge, stubprocess_wrapper
 from SCons.Platform.virtualenv import ImportVirtualenv
 from SCons.Platform.virtualenv import ignore_virtualenv, enable_virtualenv
 
+if stubprocess_wrapper:
+    # Create a copy of the subprocess.Popen by subclassing
+    class PopenWrapped(subprocess.Popen):
+        pass
+
+    # We sneak our new 'wrapped' Popen into the subprocess module,
+    # such that the original subprocess.Popen() is guaranteed to stay
+    # unchanged. Otherwise, wrapping the Popen() directly might interfere
+    # with user SConstructs where subprocess is imported directly.
+    # Here we try to ensure that only SCons' spawn calls are redirected
+    # through the PopenWrapped().
+    subprocess.PopenWrapped = PopenWrapped
+
+    # Now try to import and activate Parts' stubprocess wrapper for
+    # speeding up process forks...if this isn't possible for some reason,
+    # our subprocess.PopenWrapped() will still behave exactly the same as the
+    # builtin subprocess.Popen().
+    import SCons.Platform.stubprocess
+else:
+    subprocess.PopenWrapped = subprocess.Popen
+
 exitvalmap = {
     2 : 127,
     13 : 126,
@@ -63,14 +84,14 @@ def escape(arg):
 
 
 def exec_subprocess(l, env):
-    proc = subprocess.Popen(l, env = env, close_fds = True)
+    proc = subprocess.PopenWrapped(l, env = env, close_fds = True)
     return proc.wait()
 
 def subprocess_spawn(sh, escape, cmd, args, env):
     return exec_subprocess([sh, '-c', ' '.join(args)], env)
 
 def exec_popen3(l, env, stdout, stderr):
-    proc = subprocess.Popen(l, env = env, close_fds = True,
+    proc = subprocess.PopenWrapped(l, env = env, close_fds = True,
                             stdout = stdout,
                             stderr = stderr)
     return proc.wait()