CI Use conda for free threaded build (scikit-learn#30312)

deveshbervar · Nov 20, 2024 · 4f2159c · 4f2159c
1 parent a54633b
commit 4f2159c
Show file tree

Hide file tree

Showing 9 changed files with 119 additions and 80 deletions.
diff --git a/.github/workflows/update-lock-files.yml b/.github/workflows/update-lock-files.yml
@@ -22,6 +22,9 @@ jobs:
           - name: scipy-dev
             update_script_args: "--select-tag scipy-dev"
             additional_commit_message: "[scipy-dev]"
+          - name: free-threaded
+            update_script_args: "--select-tag free-threaded"
+            additional_commit_message: "[free-threaded]"
           - name: cirrus-arm
             update_script_args: "--select-tag arm"
             additional_commit_message: "[cirrus arm]"

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -83,10 +83,10 @@ jobs:
         )
       )
     matrix:
-      pylatest_pip_free_threaded:
+      pylatest_free_threaded:
         PYTHON_GIL: '0'
-        DISTRIB: 'pip-free-threaded'
-        LOCK_FILE: './build_tools/azure/cpython_free_threaded_lock.txt'
+        DISTRIB: 'conda-free-threaded'
+        LOCK_FILE: './build_tools/azure/pylatest_free_threaded_linux-64_conda.lock'
         COVERAGE: 'false'
 
 - job: Linux_Nightly_Pyodide

diff --git a/build_tools/azure/cpython_free_threaded_lock.txt b/build_tools/azure/cpython_free_threaded_lock.txt
diff --git a/build_tools/azure/cpython_free_threaded_requirements.txt b/build_tools/azure/cpython_free_threaded_requirements.txt
diff --git a/build_tools/azure/install.sh b/build_tools/azure/install.sh
@@ -41,17 +41,6 @@ pre_python_environment_install() {
         apt-get install -y python3-dev python3-numpy python3-scipy \
                 python3-matplotlib libopenblas-dev \
                 python3-virtualenv python3-pandas ccache git
-
-    # TODO for now we use CPython 3.13 from Ubuntu deadsnakes PPA. When CPython
-    # 3.13 is released (scheduled October 2024) we can use something more
-    # similar to other conda+pip based builds
-    elif [[ "$DISTRIB" == "pip-free-threaded" ]]; then
-        sudo apt-get -yq update
-        sudo apt-get install -yq ccache
-        sudo apt-get install -yq software-properties-common
-        sudo add-apt-repository --yes ppa:deadsnakes/nightly
-        sudo apt-get update -yq
-        sudo apt-get install -yq --no-install-recommends python3.13-dev python3.13-venv python3.13-nogil
     fi
 }
 
@@ -68,30 +57,27 @@ check_packages_dev_version() {
 python_environment_install_and_activate() {
     if [[ "$DISTRIB" == "conda"* ]]; then
         create_conda_environment_from_lock_file $VIRTUALENV $LOCK_FILE
-        source activate $VIRTUALENV
+        activate_environment
 
     elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" ]]; then
         python3 -m virtualenv --system-site-packages --python=python3 $VIRTUALENV
-        source $VIRTUALENV/bin/activate
+        activate_environment
         pip install -r "${LOCK_FILE}"
 
-    elif [[ "$DISTRIB" == "pip-free-threaded" ]]; then
-        python3.13t -m venv $VIRTUALENV
-        source $VIRTUALENV/bin/activate
-        pip install -r "${LOCK_FILE}"
-        # TODO you need pip>=24.1 to find free-threaded wheels. This may be
-        # removed when the underlying Ubuntu image has pip>=24.1.
-        pip install 'pip>=24.1'
-        # TODO When there are CPython 3.13 free-threaded wheels for numpy,
-        # scipy and cython move them to
-        # build_tools/azure/cpython_free_threaded_requirements.txt. For now we
-        # install them from scientific-python-nightly-wheels
+    fi
+
+    # Install additional packages on top of the lock-file in specific cases
+    if [[ "$DISTRIB" == "conda-free-threaded" ]]; then
+        # TODO We install scipy and cython from
+        # scientific-python-nightly-wheels. When there are conda-forge packages
+        # for scipy and cython, we can update
+        # build_tools/update_environments_and_lock_files.py and remove the
+        # lines below
         dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
-        dev_packages="numpy scipy Cython"
+        dev_packages="scipy Cython"
         pip install --pre --upgrade --timeout=60 --extra-index $dev_anaconda_url $dev_packages --only-binary :all:
-    fi
 
-    if [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
+    elif [[ "$DISTRIB" == "conda-pip-scipy-dev" ]]; then
         echo "Installing development dependency wheels"
         dev_anaconda_url=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
         dev_packages="numpy scipy pandas Cython"

diff --git a/build_tools/azure/pylatest_free_threaded_environment.yml b/build_tools/azure/pylatest_free_threaded_environment.yml
@@ -0,0 +1,16 @@
+# DO NOT EDIT: this file is generated from the specification found in the
+# following script to centralize the configuration for CI builds:
+# build_tools/update_environments_and_lock_files.py
+channels:
+  - conda-forge
+dependencies:
+  - python-freethreading
+  - numpy
+  - joblib
+  - threadpoolctl
+  - pytest
+  - pytest-xdist
+  - ninja
+  - meson-python
+  - ccache
+  - pip
diff --git a/build_tools/azure/pylatest_free_threaded_linux-64_conda.lock b/build_tools/azure/pylatest_free_threaded_linux-64_conda.lock
@@ -0,0 +1,58 @@
+# Generated by conda-lock.
+# platform: linux-64
+# input_hash: 8bf0c47c0d22842fa5a5531ad2ad62b4795b6b1cbf713816fa1101103a2e3dcc
+@EXPLICIT
+https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2#d7c89558ba9fa0495403155b64376d81
+https://conda.anaconda.org/conda-forge/linux-64/ca-certificates-2024.8.30-hbcca054_0.conda#c27d1c142233b5bc9ca570c6e2e0c244
+https://conda.anaconda.org/conda-forge/linux-64/python_abi-3.13-5_cp313t.conda#ea4c21b96e8280414d9e243da0ec3201
+https://conda.anaconda.org/conda-forge/noarch/tzdata-2024b-hc8b5060_0.conda#8ac3367aafb1cc0a068483c580af8015
+https://conda.anaconda.org/conda-forge/linux-64/ld_impl_linux-64-2.43-h712a8e2_2.conda#048b02e3962f066da18efe3a21b77672
+https://conda.anaconda.org/conda-forge/linux-64/libgomp-14.2.0-h77fa898_1.conda#cc3573974587f12dda90d96e3e55a702
+https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-2_gnu.tar.bz2#73aaf86a425cc6e73fcf236a5a46396d
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-14.2.0-h77fa898_1.conda#3cb76c3f10d3bc7f1105b2fc9db984df
+https://conda.anaconda.org/conda-forge/linux-64/libexpat-2.6.4-h5888daf_0.conda#db833e03127376d461e1e13e76f09b6c
+https://conda.anaconda.org/conda-forge/linux-64/libgcc-ng-14.2.0-h69a702a_1.conda#e39480b9ca41323497b05492a63bc35b
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran5-14.2.0-hd5240d6_1.conda#9822b874ea29af082e5d36098d25427d
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-14.2.0-hc0a3c3a_1.conda#234a5554c53625688d51062645337328
+https://conda.anaconda.org/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda#edb0dca6bc32e4f4789199455a1dbeb8
+https://conda.anaconda.org/conda-forge/linux-64/openssl-3.4.0-hb9d3cd8_0.conda#23cc74f77eb99315c0360ec3533147a9
+https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-h4bc722e_7.conda#62ee74e96c5ebb0af99386de58cf9553
+https://conda.anaconda.org/conda-forge/linux-64/libffi-3.4.2-h7f98852_5.tar.bz2#d645c6d2ac96843a2bfaccd2d62b3ac3
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran-14.2.0-h69a702a_1.conda#f1fd30127802683586f768875127a987
+https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-h4bc722e_0.conda#aeb98fdeb2e8f25d43ef71fbacbeec80
+https://conda.anaconda.org/conda-forge/linux-64/libsqlite-3.47.0-hadc24fc_1.conda#b6f02b52a174e612e89548f4663ce56a
+https://conda.anaconda.org/conda-forge/linux-64/libstdcxx-ng-14.2.0-h4852527_1.conda#8371ac6457591af2cf6159439c1fd051
+https://conda.anaconda.org/conda-forge/linux-64/libuuid-2.38.1-h0b41bf4_0.conda#40b61aab5c7ba9ff276c41cfffe6b80b
+https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.5-he02047a_1.conda#70caf8bb6cf39a0b6b7efc885f51c0fe
+https://conda.anaconda.org/conda-forge/linux-64/tk-8.6.13-noxft_h4845f30_101.conda#d453b98d9c83e71da0741bb0ff4d76bc
+https://conda.anaconda.org/conda-forge/linux-64/xz-5.2.6-h166bdaf_0.tar.bz2#2161070d867d1b1204ea749c8eec4ef0
+https://conda.anaconda.org/conda-forge/linux-64/libgfortran-ng-14.2.0-h69a702a_1.conda#0a7f4cd238267c88e5d69f7826a407eb
+https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.28-pthreads_h94d23a6_1.conda#62857b389e42b36b686331bec0922050
+https://conda.anaconda.org/conda-forge/linux-64/ninja-1.12.1-h297d8ca_0.conda#3aa1c7e292afeff25a0091ddd7c69b72
+https://conda.anaconda.org/conda-forge/linux-64/readline-8.2-h8228510_1.conda#47d31b792659ce70f470b5c82fdfb7a4
+https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.6-ha6fb4c9_0.conda#4d056880988120e29d75bfff282e0f45
+https://conda.anaconda.org/conda-forge/linux-64/libblas-3.9.0-25_linux64_openblas.conda#8ea26d42ca88ec5258802715fe1ee10b
+https://conda.anaconda.org/conda-forge/linux-64/libhiredis-1.0.2-h2cc385e_0.tar.bz2#b34907d3a81a3cd8095ee83d174c074a
+https://conda.anaconda.org/conda-forge/linux-64/python-3.13.0-h6355ac2_0_cp313t.conda#10b52576e09161c4e744cbd95d35e648
+https://conda.anaconda.org/conda-forge/linux-64/ccache-4.10.1-h065aff2_0.conda#d6b48c138e0c8170a6fe9c136e063540
+https://conda.anaconda.org/conda-forge/noarch/colorama-0.4.6-pyhd8ed1ab_0.tar.bz2#3faab06a954c2a04039983f2c4a50d99
+https://conda.anaconda.org/conda-forge/noarch/cpython-3.13.0-py313hd8ed1ab_0.conda#efdede3c85221d80346fadb903a97bf6
+https://conda.anaconda.org/conda-forge/noarch/exceptiongroup-1.2.2-pyhd8ed1ab_0.conda#d02ae936e42063ca46af6cdad2dbd1e0
+https://conda.anaconda.org/conda-forge/noarch/execnet-2.1.1-pyhd8ed1ab_0.conda#15dda3cdbf330abfe9f555d22f66db46
+https://conda.anaconda.org/conda-forge/noarch/iniconfig-2.0.0-pyhd8ed1ab_0.conda#f800d2da156d08e289b14e87e43c1ae5
+https://conda.anaconda.org/conda-forge/linux-64/libcblas-3.9.0-25_linux64_openblas.conda#5dbd1b0fc0d01ec5e0e1fbe667281a11
+https://conda.anaconda.org/conda-forge/linux-64/liblapack-3.9.0-25_linux64_openblas.conda#4dc03a53fc69371a6158d0ed37214cd3
+https://conda.anaconda.org/conda-forge/noarch/packaging-24.2-pyhff2d567_1.conda#8508b703977f4c4ada34d657d051972c
+https://conda.anaconda.org/conda-forge/noarch/pip-24.3.1-pyh145f28c_0.conda#ca3afe2d7b893a8c8cdf489d30a2b1a3
+https://conda.anaconda.org/conda-forge/noarch/pluggy-1.5.0-pyhd8ed1ab_0.conda#d3483c8fc2dc2cc3f5cf43e26d60cabf
+https://conda.anaconda.org/conda-forge/noarch/setuptools-75.5.0-pyhff2d567_0.conda#ade63405adb52eeff89d506cd55908c0
+https://conda.anaconda.org/conda-forge/noarch/threadpoolctl-3.5.0-pyhc1e730c_0.conda#df68d78237980a159bd7149f33c0e8fd
+https://conda.anaconda.org/conda-forge/noarch/tomli-2.1.0-pyhff2d567_0.conda#3fa1089b4722df3a900135925f4519d9
+https://conda.anaconda.org/conda-forge/noarch/joblib-1.4.2-pyhd8ed1ab_0.conda#25df261d4523d9f9783bcdb7208d872f
+https://conda.anaconda.org/conda-forge/noarch/meson-1.6.0-pyhd8ed1ab_0.conda#380ba6a3eddd8e7649bfe8e6812611aa
+https://conda.anaconda.org/conda-forge/linux-64/numpy-2.1.3-py313hb01392b_0.conda#edd0335b8d3c81f0a91aa68cb8749929
+https://conda.anaconda.org/conda-forge/noarch/pyproject-metadata-0.9.0-pyh2cfa8aa_0.conda#10906a130eeb4a68645bf97c28333141
+https://conda.anaconda.org/conda-forge/noarch/pytest-8.3.3-pyhd8ed1ab_0.conda#c03d61f31f38fdb9facf70c29958bf7a
+https://conda.anaconda.org/conda-forge/noarch/python-freethreading-3.13.0-h92d6c8b_0.conda#4c3f45e4597606f5b0e2770743bbcd7e
+https://conda.anaconda.org/conda-forge/noarch/meson-python-0.17.1-pyh70fd9c4_0.conda#722b649da38842068d83b6e6770f11a1
+https://conda.anaconda.org/conda-forge/noarch/pytest-xdist-3.6.1-pyhd8ed1ab_0.conda#b39568655c127a9c4a44d178ac99b6d0
diff --git a/build_tools/shared.sh b/build_tools/shared.sh
@@ -29,7 +29,7 @@ show_installed_libraries(){
 activate_environment() {
     if [[ "$DISTRIB" =~ ^conda.* ]]; then
         source activate $VIRTUALENV
-    elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" || "$DISTRIB" == "pip-free-threaded" ]]; then
+    elif [[ "$DISTRIB" == "ubuntu" || "$DISTRIB" == "debian-32" ]]; then
         source $VIRTUALENV/bin/activate
     fi
 }

diff --git a/build_tools/update_environments_and_lock_files.py b/build_tools/update_environments_and_lock_files.py
@@ -266,6 +266,31 @@ def remove_from(alist, to_remove):
             + ["python-dateutil"]
         ),
     },
+    {
+        "name": "pylatest_free_threaded",
+        "type": "conda",
+        "tag": "free-threaded",
+        "folder": "build_tools/azure",
+        "platform": "linux-64",
+        "channels": ["conda-forge"],
+        "conda_dependencies": [
+            "python-freethreading",
+            "numpy",
+            # TODO add cython and scipy when there are conda-forge packages for
+            # them and remove dev version install in
+            # build_tools/azure/install.sh. Note that for now conda-lock does
+            # not deal with free-threaded wheels correctly, see
+            # https://github.com/conda/conda-lock/issues/754.
+            "joblib",
+            "threadpoolctl",
+            "pytest",
+            "pytest-xdist",
+            "ninja",
+            "meson-python",
+            "ccache",
+            "pip",
+        ],
+    },
     {
         "name": "pymin_conda_forge_mkl",
         "type": "conda",