Add experimental igzip (#182) #669
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - 'releases/**' | |
| pull_request: | |
| release: | |
| types: | |
| - released | |
| - prereleased | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| build-test: | |
| runs-on: ${{ matrix.conf.os }} | |
| name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }} | |
| strategy: | |
| fail-fast: ${{ !( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} | |
| matrix: | |
| python-version: | |
| - '3.8' | |
| - '3.9' | |
| - '3.10' | |
| - '3.11' | |
| - '3.12' | |
| - '3.13' | |
| conf: | |
| - { os: ubuntu-latest, target: x86_64, target-triple: x86_64-unknown-linux-gnu, manylinux: auto } | |
| - { os: ubuntu-latest, target: x86_64, target-triple: x86_64-unknown-linux-musl, manylinux: musllinux_1_1 } | |
| - { os: ubuntu-latest, target: i686, target-triple: i686-unknown-linux-gnu, manylinux: auto } | |
| - { os: ubuntu-latest, target: i686, target-triple: i686-unknown-linux-musl, manylinux: musllinux_1_1 } | |
| - { os: ubuntu-latest, target: aarch64, target-triple: aarch64-unknown-linux-gnu, manylinux: auto } | |
| - { os: ubuntu-latest, target: aarch64, target-triple: aarch64-unknown-linux-musl, manylinux: musllinux_1_1 } | |
| - { os: ubuntu-latest, target: armv7, target-triple: armv7-unknown-linux-gnueabihf, manylinux: auto } | |
| - { os: ubuntu-latest, target: armv7, target-triple: armv7-unknown-linux-musleabihf, manylinux: musllinux_1_1 } | |
| - { os: ubuntu-latest, target: s390x, target-triple: s390x-unknown-linux-gnu, manylinux: auto } | |
| # - { os: ubuntu-latest, target: s390x, target-triple: s390x-unknown-linux-musl, manylinux: musllinux_1_1 } # no target musl for s390x | |
| - { os: ubuntu-latest, target: ppc64le, target-triple: powerpc64le-unknown-linux-gnu, manylinux: auto } | |
| # - { os: ubuntu-latest, target: ppc64le, target-triple: powerpc64le-unknown-linux-musleabihf, manylinux: musllinux_1_1 } # no target musl for ppc64le | |
| - { os: macos-13, target: x86_64, target-triple: x86_64-apple-darwin } | |
| - { os: macos-13, target: aarch64, target-triple: aarch64-apple-darwin } | |
| - { os: macos-13, target: universal2, target-triple: x86_64-apple-darwin } | |
| - { os: windows-latest, target: x86_64, target-triple: x86_64-pc-windows-msvc, python-architecture: x64 } | |
| - { os: windows-latest, target: i686, target-triple: i686-pc-windows-msvc, python-architecture: x86 } | |
| include: | |
| # Windows x86_64 pypy | |
| - conf: { os: windows-latest, target: x86_64, target-triple: x86_64-pc-windows-msvc } | |
| python-version: pypy3.10 | |
| # Linux x86_64 pypy | |
| - conf: { os: ubuntu-latest, target: x86_64, target-triple: x86_64-unknown-linux-gnu, manylinux: auto } | |
| python-version: pypy3.10 | |
| # Linux arm pypy | |
| - conf: { os: ubuntu-latest, target: aarch64, target-triple: aarch64-unknown-linux-gnu, manylinux: auto } | |
| python-version: pypy3.10 | |
| # OSX x86_64 pypy | |
| - conf: { os: macos-13, target: x86_64, target-triple: x86_64-apple-darwin } | |
| python-version: pypy3.10 | |
| # OSX universal2 pypy | |
| - conf: { os: macos-13, target: universal2, target-triple: x86_64-apple-darwin } | |
| python-version: pypy3.10 | |
| # OSX arm pypy | |
| - conf: { os: macos-13, target: aarch64, target-triple: aarch64-apple-darwin } | |
| python-version: pypy3.10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| architecture: ${{ matrix.conf.python-architecture }} | |
| allow-prereleases: true | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| target: ${{ matrix.conf.target-triple }} | |
| - name: Install nasm (Windows / Linux) | |
| if: runner.os != 'macOS' | |
| uses: ilammy/setup-nasm@v1 | |
| - name: Set MSVC developer prompt | |
| if: runner.os == 'Windows' | |
| uses: ilammy/msvc-dev-cmd@v1 | |
| - name: Setup (Mac) | |
| if: runner.os == 'macOS' | |
| run: | | |
| brew install ninja automake autoconf coreutils libtool nasm | |
| echo "MACOSX_DEPLOYMENT_TARGET=10.12" >> $GITHUB_ENV | |
| - name: Setup (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| choco install ninja cmake | |
| - name: Setup (Linux) | |
| if: runner.os == 'Linux' | |
| env: | |
| TARGET_TRIPLE: ${{ matrix.conf.target-triple }} | |
| WORKSPACE: ${{ github.workspace }} | |
| BLOSC2_INSTALL_PREFIX: ${{ github.workspace }}/blosc2 | |
| ISAL_INSTALL_PREFIX: ${{ github.workspace }}/isal | |
| run: | | |
| sudo apt update | |
| sudo apt install ninja-build -y | |
| echo "BLOSC2_INSTALL_PREFIX=$BLOSC2_INSTALL_PREFIX" >> $GITHUB_ENV | |
| echo "ISAL_INSTALL_PREFIX=$ISAL_INSTALL_PREFIX" >> $GITHUB_ENV | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$BLOSC2_INSTALL_PREFIX/lib:$BLOSC2_INSTALL_PREFIX/lib64:$ISAL_INSTALL_PREFIX/lib:$ISAL_INSTALL_PREFIX/lib64" >> $GITHUB_ENV | |
| # so we'll just use 'cross' to build and pass it in for the action for all archs | |
| # nothing special for the revision pin, just deterministic install | |
| cargo install cross --git https://github.com/cross-rs/cross --rev 6d097fb | |
| # Build blosc2 | |
| cross build --release --target $TARGET_TRIPLE --package blosc2-sys --target-dir build | |
| blosc2_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep blosc2-sys) | |
| mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$blosc2_sys_dir/out $BLOSC2_INSTALL_PREFIX | |
| tree -L 2 $BLOSC2_INSTALL_PREFIX | |
| # Build isal only on 64-bit systems | |
| # At the time of this writing, it technically builds for all unix 32-bit systems | |
| # but ISA-L has explicitly stated they're dropping support. | |
| if [[ "$TARGET_TRIPLE" == armv7* || "$TARGET_TRIPLE" == i686* ]]; then | |
| echo "Not building ISA-L on 32 bit target" | |
| mkdir -p $ISAL_INSTALL_PREFIX | |
| else | |
| cross build --release --target $TARGET_TRIPLE --package isal-sys --target-dir build | |
| isal_sys_dir=$(ls build/$TARGET_TRIPLE/release/build/ | grep isal-sys) | |
| mv $WORKSPACE/build/$TARGET_TRIPLE/release/build/$isal_sys_dir/out $ISAL_INSTALL_PREFIX | |
| tree -L 2 $ISAL_INSTALL_PREFIX | |
| fi | |
| - name: Rust Tests | |
| if: matrix.conf.target == 'x86_64' && !startsWith(matrix.python-version, 'pypy') && matrix.python-version == '3.12' | |
| run: cargo test | |
| - name: Build wheel (Linux) | |
| if: runner.os == 'Linux' | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.conf.target }} | |
| manylinux: ${{ matrix.conf.manylinux }} | |
| docker-options: | | |
| -e BLOSC2_INSTALL_PREFIX=${{ github.workspace }}/blosc2 | |
| -e ISAL_INSTALL_PREFIX=${{ github.workspace }}/isal | |
| -e LD_LIBRARY_PATH=${{ github.workspace }}/blosc2/lib:${{ github.workspace }}/blosc2/lib64:${{ github.workspace }}/isal/lib:${{ github.workspace }}/isal/lib64 | |
| args: -i ${{ matrix.python-version }} --release --out dist --features use-system-blosc2-static --features use-system-isal-static | |
| before-script-linux: | | |
| ls -l $BLOSC2_INSTALL_PREFIX | |
| ls -l $ISAL_INSTALL_PREFIX | |
| - name: Build wheel (Windows) | |
| if: runner.os == 'Windows' | |
| run: | | |
| python -m pip install maturin delvewheel | |
| maturin build -i python --release --out wheels --target ${{ matrix.conf.target-triple }} | |
| $file = Get-ChildItem -Path "wheels" | Select-Object -First 1 | |
| delvewheel repair -v "wheels\$($file.Name)" -w "dist" | |
| - name: Build wheel (MacOS) | |
| if: runner.os == 'macOS' | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| target: ${{ matrix.conf.target-triple }} | |
| args: -i python --release --out dist | |
| - name: Fix wheel (MacOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| python -m pip install delocate | |
| delocate-wheel -v dist/*.whl | |
| - name: Install built wheel and Test (Native) | |
| # TODO: I'm not sure but the actual collection of tests on windows using pypy3.10 takes forever and/or fails | |
| if: | | |
| !startsWith(matrix.conf.manylinux, 'musl') && | |
| !( matrix.python-version == 'pypy3.10' && runner.os == 'Windows' ) && | |
| ( matrix.conf.target == 'x86_64' || matrix.conf.target == 'universal2' ) | |
| run: | | |
| # Second install guarantees it's going to install from local dir w/ --no-index | |
| # use first to get in dev dependencies | |
| python -m pip install cramjam[dev] --pre --find-links dist --force-reinstall | |
| python -m pip install cramjam --pre --no-index --find-links dist --force-reinstall | |
| python -m pytest -vs --benchmark-skip | |
| # Could use 'distro: alpine_latest' in 'run-on-arch-action' but seems difficult to install a specific version of python | |
| # so we'll just use existing python alpine images to test import and cli use w/o testing archs other than x86_64 | |
| - name: Install built wheel and Test (musllinux) | |
| # TODO: python:3.13-alpine doesn't exist yet | |
| if: startsWith(matrix.conf.manylinux, 'musl') && matrix.conf.target == 'x86_64' && matrix.python-version != '3.13' | |
| run: | | |
| docker run \ | |
| -v $(pwd)/dist:/wheels \ | |
| --rm python:${{ matrix.python-version }}-alpine sh \ | |
| -c "pip install cramjam --no-index --find-links /wheels && python -c 'import cramjam'" | |
| - name: Install built wheel and Test (Cross) | |
| if: | | |
| !startsWith(matrix.conf.manylinux, 'musl') && | |
| runner.os == 'Linux' && | |
| !startsWith(matrix.python-version, 'pypy') && | |
| contains(fromJson('["armv6", "armv7", "aarch64", "riscv64", "s390x", "ppc64le"]'), matrix.conf.target) | |
| uses: uraimo/[email protected] | |
| with: | |
| arch: ${{ matrix.conf.target }} | |
| distro: ubuntu20.04 | |
| githubToken: ${{ github.token }} | |
| # Mount the dist directory as /artifacts in the container | |
| dockerRunArgs: | | |
| --volume "${PWD}/dist:/artifacts" | |
| install: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3 python3-venv software-properties-common | |
| add-apt-repository ppa:deadsnakes/ppa | |
| apt-get update | |
| apt-get install -y curl python${{ matrix.python-version }}-venv | |
| run: | | |
| ls -lrth /artifacts | |
| PYTHON=python${{ matrix.python-version }} | |
| $PYTHON -m venv venv | |
| venv/bin/pip install -U pip | |
| venv/bin/pip install cramjam --pre --no-index --find-links /artifacts --force-reinstall | |
| venv/bin/python -c 'import cramjam' | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} | |
| with: | |
| name: ${{ matrix.conf.os }}-${{ matrix.python-version }}-${{ matrix.conf.target-triple }}-${{ matrix.conf.target }} | |
| path: dist | |
| build-wasm32-emscripten-pyodide: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python: | |
| - "3.12" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| target: wasm32-unknown-emscripten | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| - name: Install Emscripten | |
| uses: mymindstorm/setup-emsdk@v14 | |
| with: | |
| # This needs to match the exact expected version pyodide expects...seems a bit brittle TBH, maybe I'm missing something. | |
| # Discover by updating pyodide in package.json and re-running 'npm run test'; it'll spit out the error of | |
| # the expected vs actual versions. | |
| version: '3.1.58' | |
| - name: Build | |
| run: | | |
| pip install maturin | |
| maturin build --release -i python${{ matrix.python }} --features wasm32-compat --target wasm32-unknown-emscripten -o ./dist | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18' | |
| - run: npm install | |
| - run: npm run test | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ ( startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') ) }} | |
| with: | |
| name: wasm32-unknown-emscripten-python${{ matrix.python }} | |
| path: dist | |
| build-sdist: | |
| name: Build sdists | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| - name: Build sdist cramjam | |
| run: | | |
| python -m pip install build | |
| python -m build --sdist -o ./dist | |
| - name: Upload sdists | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist | |
| path: dist | |
| pypi-publish: | |
| name: Upload release to PyPI | |
| if: startsWith(github.ref, 'refs/tags/') | |
| runs-on: ubuntu-latest | |
| needs: [build-test, build-sdist] | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/cramjam | |
| permissions: | |
| id-token: write | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| merge-multiple: true | |
| - name: List artifacts | |
| run: ls -lhs | |
| - name: Remove wasm32 wheels # TODO: https://discuss.python.org/t/support-wasm-wheels-on-pypi/21924 | |
| run: rm ./*wasm32.whl | |
| - name: Publish package distributions to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| skip-existing: true | |
| packages-dir: ./ | |
| gh-publish: | |
| name: Publish artifacts to GH | |
| if: startsWith(github.ref, 'refs/tags/') | |
| permissions: | |
| contents: write | |
| needs: [build-test, build-sdist] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: . | |
| merge-multiple: true | |
| - name: List artifacts | |
| run: ls -lhs | |
| - name: Upload to GitHub | |
| uses: softprops/action-gh-release@v2 | |
| with: | |
| files: ./* |