Improve deployment concurrency GCL management #2617
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integrations Packages Tests | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/integration-package-tests.yaml | |
| - "src/**/*.py" | |
| - "src/integrations/**/*" | |
| types: [opened, reopened, synchronize, labeled, unlabeled] | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - "src/integrations/**/*" | |
| jobs: | |
| prepare-matrix: | |
| # These tests will only run if the integration paths are affected, or someone has | |
| # added the `integrations` label to the PR | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.9" | |
| - name: Generate matrix | |
| id: set-matrix | |
| run: | | |
| if [[ "${{ contains(github.event.pull_request.labels.*.name, 'test-all-integrations') }}" == 'true' ]]; then | |
| # All of the integration packages were changed since 2.19.2, so this is a | |
| # standin for saying "all integrations" | |
| COMMIT_RANGE="2.19.2..${{ github.event.pull_request.head.sha }}" | |
| elif [[ $GITHUB_EVENT_NAME == 'pull_request' ]]; then | |
| COMMIT_RANGE="${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}" | |
| else | |
| COMMIT_RANGE="${{ github.event.before }}..${{ github.event.after }}" | |
| fi | |
| python scripts/generate_integration_package_tests_matrix.py "$COMMIT_RANGE" > matrix.json | |
| cat matrix.json | |
| echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT | |
| run-tests: | |
| timeout-minutes: 20 | |
| name: Run Tests for ${{ matrix.package }} on Python ${{ matrix.python-version }} | |
| needs: prepare-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} | |
| fail-fast: false | |
| steps: | |
| - name: Display current test matrix | |
| run: echo '${{ toJSON(matrix) }}' | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v5 | |
| id: setup_python | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: UV Cache | |
| # Manually cache the uv cache directory | |
| # until setup-python supports it: | |
| # https://github.com/actions/setup-python/issues/822 | |
| uses: actions/cache@v4 | |
| id: cache-uv | |
| with: | |
| path: ~/.cache/uv | |
| key: uvcache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ hashFiles(format('src/integrations/{0}/pyproject.toml', matrix.package)) }} | |
| - name: Install dependencies | |
| working-directory: src/integrations/${{ matrix.package }} | |
| # install uv, the package, and bleeding edge prefect | |
| run: | | |
| python -m pip install -U uv | |
| uv pip install --upgrade --system -e .[dev] | |
| uv pip install --upgrade --system -e ../../../ | |
| - name: Build dev docker image | |
| if: matrix.package == 'prefect-docker' | |
| run : prefect dev build-image | |
| - name: Start redis | |
| if: matrix.package == 'prefect-redis' | |
| run: > | |
| docker run | |
| --name "redis" | |
| --detach | |
| --publish 6379:6379 | |
| redis:latest | |
| - name: Run tests | |
| if: matrix.package != 'prefect-ray' | |
| env: | |
| PREFECT_API_DATABASE_CONNECTION_URL: "sqlite+aiosqlite:///./orion-tests.db" | |
| working-directory: src/integrations/${{ matrix.package }} | |
| run: > | |
| pytest tests | |
| --numprocesses auto | |
| --maxprocesses 4 | |
| --dist worksteal | |
| # Run prefect-ray tests sequentially to avoid Ray cluster issues | |
| - name: Run tests for prefect-ray | |
| if: matrix.package == 'prefect-ray' | |
| env: | |
| PREFECT_API_DATABASE_CONNECTION_URL: "sqlite+aiosqlite:///./orion-tests.db" | |
| working-directory: src/integrations/${{ matrix.package }} | |
| run: > | |
| pytest tests |