Add support for native libraries that are installed but not on path (#50

)

* Add support for native libraries that are installed but not on path

At least on macOS, installing up-to-date versions of several libraries
via a third party package manager such as Homebrew may interfere with
the rather old and tired version shipped with macOS. Consequently,
Homebrew does not add these libraries to the include and linker paths.
This change adds a configuration option to derive the correct arguments
for CPython's configure script. Adding support for other package
managers and operating systems should be trivial, since only two lines
are macOS-specific.

* Update documentation for new pkg_only configuration file entry

Leave actual value blank, so that the example, if directly reused,
does not break build for Linux and Windows. Also document actual,
working value for macOS.

AUTHORS

@@ -15,6 +15,7 @@ James Abbatiello <[email protected]>
 Jeffrey Yasskin <[email protected]>
 Maciej Fijalkowski
 Reid Klecker <[email protected]>
+Robert Grimm
 Skip Montanaro
 Stefan Behnel
 Thomas Wouters <[email protected]>

doc/benchmark.conf.sample

@@ -37,6 +37,19 @@ lto = True
 # Profiled Guided Optimization (PGO)?
 pgo = True
 
+# The space-separated list of libraries that are package-only,
+# i.e., locally installed but not on header and library paths.
+# For each such library, determine the install path and add an
+# appropriate subpath to CFLAGS and LDFLAGS declarations passed
+# to configure. As an exception, the prefix for openssl, if that
+# library is present here, is passed via the --with-openssl
+# option. Currently, this only works with Homebrew on macOS.
+# If running on macOS with Homebrew, you probably want to use:
+#     pkg_only = openssl readline sqlite3 xz zlib
+# The version of zlib shipping with macOS probably works as well,
+# as long as Apple's SDK headers are installed.
+pkg_only =
+
 # Install Python? If false, run Python from the build directory
 #
 # WARNING: Running Python from the build directory introduces subtle changes

performance/compile.py

@@ -274,6 +274,8 @@ def compile(self):
             config_args.append('--with-pydebug')
         elif self.conf.lto:
             config_args.append('--with-lto')
+        if self.conf.pkg_only:
+            config_args.extend(self.get_package_only_flags())
         if self.conf.debug:
             config_args.append('CFLAGS=-O0')
         configure = os.path.join(self.conf.repo_dir, 'configure')
@@ -298,6 +300,9 @@ def install_python(self):
 
             self.run('make', 'install')
 
+            if sys.platform == 'darwin':
+                program_ext = ''
+
             self.program = os.path.join(prefix, "bin", "python" + program_ext)
             if not os.path.exists(self.program):
                 self.program = os.path.join(prefix, "bin", "python3" + program_ext)
@@ -317,6 +322,37 @@ def get_version(self):
         self.hexversion = int(stdout)
         self.logger.error("Python hexversion: %x" % self.hexversion)
 
+    def get_package_only_flags(self):
+        arguments = []
+        extra_paths = []
+
+        for pkg in self.conf.pkg_only:
+            prefix = self.get_package_prefix(pkg)
+            if pkg == 'openssl':
+                arguments.append('--with-openssl=' + prefix)
+            else:
+                extra_paths.append(prefix)
+
+        if extra_paths:
+            # Flags are one CLI arg each and do not need quotes.
+            ps = ['-I%s/include' % p for p in extra_paths]
+            arguments.append('CFLAGS=%s' % ' '.join(ps))
+            ps = ['-L%s/lib' % p for p in extra_paths]
+            arguments.append('LDFLAGS=%s' % ' '.join(ps))
+        return arguments
+
+    def get_package_prefix(self, name):
+        if sys.platform == 'darwin':
+            cmd = ['brew', '--prefix', name]
+        else:
+            self.logger.error("ERROR: package-only libraries"
+                              " are not supported on %s" % sys.platform)
+            sys.exit(1)
+
+        stdout = self.get_output(*cmd)
+        self.logger.error("Package prefix for %s is '%s'" % (name, stdout))
+        return stdout
+
     def download(self, url, filename):
         self.logger.error("Download %s into %s" % (url, filename))
         download(url, filename)
@@ -730,6 +766,7 @@ def getboolean(section, key, default):
         conf.lto = getboolean('compile', 'lto', True)
         conf.pgo = getboolean('compile', 'pgo', True)
         conf.install = getboolean('compile', 'install', True)
+        conf.pkg_only = getstr('compile', 'pkg_only', '').split()
 
         # [run_benchmark]
         conf.system_tune = getboolean('run_benchmark', 'system_tune', True)