WIP #377
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Pipeline | |
| on: | |
| # Run on all branches and pull requests to ensure that the code is always tested | |
| push: | |
| branches: | |
| - develop | |
| - feat/* | |
| - hotfix/* | |
| - main | |
| # Run on pull requests to ensure that the code is ready for merging | |
| pull_request: | |
| branches: | |
| - develop | |
| - feat/* | |
| - hotfix/* | |
| - main | |
| # Allows CodSpeed to trigger backtest performance analysis in order to generate initial data | |
| workflow_dispatch: | |
| jobs: | |
| init: | |
| name: Init | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| VERSION: ${{ steps.version.outputs.version }} | |
| VERSION_FULL: ${{ steps.version.outputs.version_full }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| with: | |
| # We need all Git history for "version.sh" | |
| fetch-depth: 0 | |
| # Ensure "version.sh" submodule are up-to-date | |
| submodules: recursive | |
| - name: Generate versions | |
| id: version | |
| run: | | |
| echo "version=$(bash cicd/version/version.sh -g . -c)" >> $GITHUB_OUTPUT | |
| echo "version_full=$(bash cicd/version/version.sh -g . -c -m)" >> $GITHUB_OUTPUT | |
| test: | |
| name: Test | |
| permissions: | |
| contents: read | |
| id-token: write | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - init | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Run all test suites | |
| step: | |
| - static | |
| - unit-simple | |
| - unit-codspeed | |
| # Run on all supported Python versions | |
| python-version: | |
| - "3.11" | |
| - "3.12" | |
| - "3.13" | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Setup Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup uv | |
| uses: astral-sh/[email protected] | |
| with: | |
| enable-cache: true | |
| version: "0.5.x" | |
| - name: Login to Azure | |
| uses: Azure/[email protected] | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Cache pytest | |
| uses: actions/[email protected] | |
| with: | |
| key: pytest-${{ github.head_ref || github.ref_name }}-${{ hashFiles('uv.lock') }} | |
| path: .pytest_cache | |
| - name: Cache Ruff | |
| uses: actions/[email protected] | |
| with: | |
| key: ruff-${{ github.head_ref || github.ref_name }}-${{ hashFiles('uv.lock') }} | |
| path: .ruff_cache | |
| - name: Setup dependencies | |
| run: make install-deps | |
| # See: https://github.com/ungoogled-software/ungoogled-chromium/issues/2804#issuecomment-2081611387 | |
| - name: Configure AppArmor profile | |
| run: | | |
| sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 | |
| - name: Configure environment variables | |
| run: echo "${{ secrets.DOTENV_UNIT_TESTS }}" > .env | |
| - name: Run tests (simple) | |
| if: ${{ !contains(matrix.step, 'codspeed') }} | |
| run: make test-${{ matrix.step }} version_full=${{ needs.init.outputs.VERSION_FULL }} | |
| - name: Run tests (CodSpeed) | |
| if: ${{ contains(matrix.step, 'codspeed') }} | |
| uses: CodSpeedHQ/[email protected] | |
| with: | |
| token: ${{ secrets.CODSPEED_TOKEN }} | |
| run: make test-${{ matrix.step }} version_full=${{ needs.init.outputs.VERSION_FULL }} | |
| - name: Upload artifacts | |
| uses: actions/[email protected] | |
| if: always() | |
| with: | |
| compression-level: 9 | |
| name: test-${{ matrix.step }}-${{ matrix.python-version }} | |
| path: test-reports/* | |
| build-wheels: | |
| name: Build wheels | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - init | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Setup uv | |
| uses: astral-sh/[email protected] | |
| with: | |
| enable-cache: true | |
| python-version: "3.13" | |
| version: "0.5.x" | |
| - name: Write version | |
| run: | | |
| version=$(echo "${{ needs.init.outputs.VERSION }}" | sed 's/-/\.a/' | cut -d'.' -f1-4) | |
| echo '__version__ = "'${version}'"' > src/scrape_it_now/__init__.py | |
| - name: Build wheels | |
| run: | | |
| make build | |
| - name: Upload artifacts | |
| uses: actions/[email protected] | |
| with: | |
| compression-level: 9 | |
| name: wheels | |
| path: dist/*.whl | |
| attest-binaries: | |
| name: Attest binaries | |
| permissions: | |
| attestations: write | |
| id-token: write | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-wheels | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/[email protected] | |
| with: | |
| merge-multiple: true | |
| path: dist | |
| pattern: wheels | |
| - name: Generate attestations for binaries | |
| uses: actions/[email protected] | |
| with: | |
| subject-path: dist/* | |
| attest-sbom: | |
| name: Attest SBOM | |
| permissions: | |
| attestations: write | |
| id-token: write | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-wheels | |
| - init | |
| steps: | |
| - name: Checkout | |
| uses: actions/[email protected] | |
| - name: Download artifacts | |
| uses: actions/[email protected] | |
| with: | |
| merge-multiple: true | |
| path: dist | |
| pattern: wheels | |
| - name: Setup Syft | |
| uses: anchore/sbom-action/[email protected] | |
| - name: Run scan | |
| run: make sbom version_full=${{ needs.init.outputs.VERSION_FULL }} | |
| - name: Generate attestations for SBOM | |
| uses: actions/[email protected] | |
| with: | |
| subject-path: sbom-reports/* | |
| - name: Upload results to release | |
| uses: actions/[email protected] | |
| with: | |
| compression-level: 9 | |
| name: sbom | |
| path: sbom-reports/* | |
| publish-testpypi: | |
| name: Publish to TestPyPI | |
| # The env "testpypi" has a protection rule which requires to be on the "main" or "develop" branches, which are both protected | |
| environment: | |
| name: testpypi | |
| url: https://test.pypi.org/p/scrape-it-now | |
| permissions: | |
| id-token: write | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - attest-binaries | |
| - attest-sbom | |
| - test | |
| if: (github.ref == 'refs/heads/main') || (github.ref == 'refs/heads/develop') | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/[email protected] | |
| with: | |
| merge-multiple: true | |
| path: dist | |
| pattern: wheels | |
| - name: Publish to TestPyPI | |
| uses: pypa/[email protected] | |
| with: | |
| repository-url: https://test.pypi.org/legacy/ | |
| publish-pypi: | |
| name: Publish to PyPI | |
| # The env "pypi" has a protection rule which requires to be on the "main" branch, which is protected | |
| environment: | |
| name: pypi | |
| url: https://pypi.org/p/scrape-it-now | |
| permissions: | |
| id-token: write | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| # Always publish first to TestPyPI as a safety precaution | |
| - publish-testpypi | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/[email protected] | |
| with: | |
| merge-multiple: true | |
| path: dist | |
| pattern: wheels | |
| - name: Publish to PyPI | |
| uses: pypa/[email protected] | |
| publish-release: | |
| name: Publish release | |
| permissions: | |
| contents: write | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - attest-binaries | |
| - attest-sbom | |
| - init | |
| - test | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Download artifacts | |
| id: download | |
| uses: actions/[email protected] | |
| with: | |
| merge-multiple: true | |
| path: dist | |
| - name: Publish | |
| uses: softprops/[email protected] | |
| with: | |
| files: dist/* | |
| generate_release_notes: true | |
| make_latest: true | |
| name: scrape-it-now v${{ needs.init.outputs.VERSION }} | |
| tag_name: v${{ needs.init.outputs.VERSION }} |