diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml deleted file mode 100644 index a74595511b4..00000000000 --- a/.github/actions/setup-postgres-linux/action.yml +++ /dev/null @@ -1,9 +0,0 @@ -name: "Set up postgres (linux)" -description: "Set up postgres service on linux vm for dbt integration tests" -runs: - using: "composite" - steps: - - shell: bash - run: | - echo "Running setup_db.sh" - sudo -u postgres bash ${{ github.action_path }}/setup_db.sh diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh deleted file mode 120000 index ee75ef5048c..00000000000 --- a/.github/actions/setup-postgres-linux/setup_db.sh +++ /dev/null @@ -1 +0,0 @@ -../../../test/setup_db.sh \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0bbc2fcb790..eaad9ee197c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,7 +152,7 @@ jobs: echo "include=${INCLUDE_GROUPS}" echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT - integration: + integration-postgres: name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -165,8 +165,6 @@ jobs: python-version: [ "3.9", "3.10", "3.11", "3.12" ] os: [ubuntu-latest] split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} - # this include is where we add the mac and windows os - include: ${{ fromJson(needs.integration-metadata.outputs.include) }} env: TOXENV: integration DBT_INVOCATION_ENV: github-actions @@ -188,14 +186,14 @@ jobs: env: POSTGRES_PASSWORD: password POSTGRES_USER: postgres - POSTGRES_HOST: localhost - POSTGRES_PORT: 5432 # Set health checks to wait until postgres has started options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + ports: + - 5432:5432 steps: - name: Check out the repository @@ -206,9 +204,83 @@ jobs: with: python-version: ${{ matrix.python-version }} - # - name: Set up postgres (linux) - # if: runner.os == 'Linux' - # uses: ./.github/actions/setup-postgres-linux + - name: Run postgres setup script + run: | + ./test/setup_db.sh + env: + PGHOST: localhost + PGPORT: 5432 + PGPASSWORD: password + + - name: Install python tools + run: | + python -m pip install --user --upgrade pip + python -m pip --version + python -m pip install tox + tox --version + + - name: Run integration tests + uses: nick-fields/retry@v3 + with: + timeout_minutes: 30 + max_attempts: 3 + command: tox -- --ddtrace + env: + PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }} + + - name: Get current date + if: always() + id: date + run: | + CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts + echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }} + path: ./logs + + - name: Upload Integration Test Coverage to Codecov + if: ${{ matrix.python-version == '3.11' }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + flags: integration + + integration-mac-windows: + name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }} + + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + needs: + - integration-metadata + strategy: + fail-fast: false + matrix: + split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }} + # this include is where we add the mac and windows os + include: ${{ fromJson(needs.integration-metadata.outputs.include) }} + env: + TOXENV: integration + DBT_INVOCATION_ENV: github-actions + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + + steps: + - name: Check out the repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} - name: Set up postgres (macos) if: runner.os == 'macOS' @@ -258,10 +330,10 @@ jobs: if: ${{ always() }} name: Integration Test Suite runs-on: ubuntu-latest - needs: integration + needs: [integration-mac-windows, integration-postgres] steps: - name: "Integration Tests Failed" - if: ${{ contains(needs.integration.result, 'failure') || contains(needs.integration.result, 'cancelled') }} + if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }} # when this is true the next step won't execute run: | echo "::notice title='Integration test suite failed'" diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml index b93e248cda6..da8def26229 100644 --- a/.github/workflows/structured-logging-schema-check.yml +++ b/.github/workflows/structured-logging-schema-check.yml @@ -67,6 +67,24 @@ jobs: DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 + services: + # Label used to access the service container + postgres: + # Docker Hub image + image: postgres + # Provide the password for postgres + env: + POSTGRES_PASSWORD: password + POSTGRES_USER: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: checkout dev uses: actions/checkout@v4 @@ -85,8 +103,13 @@ jobs: pip install tox tox --version - - name: Set up postgres - uses: ./.github/actions/setup-postgres-linux + - name: Run postgres setup script + run: | + ./test/setup_db.sh + env: + PGHOST: localhost + PGPORT: 5432 + PGPASSWORD: password - name: ls run: ls diff --git a/test/setup_db.sh b/test/setup_db.sh index 5b1f83bd72e..0cbc4faa598 100755 --- a/test/setup_db.sh +++ b/test/setup_db.sh @@ -1 +1,39 @@ -../../../test/setup_db.sh +#!/bin/bash +set -x +env | grep '^PG' + +# If you want to run this script for your own postgresql (run with +# docker-compose) it will look like this: +# PGHOST=127.0.0.1 PGUSER=root PGPASSWORD=password PGDATABASE=postgres \ +PGUSER="${PGUSER:-postgres}" +export PGUSER +PGPORT="${PGPORT:-5432}" +export PGPORT +PGHOST="${PGHOST:-localhost}" + +for i in {1..10}; do + if pg_isready -h "${PGHOST}" -p "${PGPORT}" -U "${PGUSER}" ; then + break + fi + + echo "Waiting for postgres to be ready..." + sleep 2; +done; + +createdb dbt +psql -c "SELECT version();" +psql -c "CREATE ROLE root WITH PASSWORD 'password';" +psql -c "ALTER ROLE root WITH LOGIN;" +psql -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;" + +psql -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;" +psql -c "ALTER ROLE noaccess WITH LOGIN;" +psql -c "GRANT CONNECT ON DATABASE dbt TO noaccess;" +psql -c "CREATE ROLE dbt_test_user_1;" +psql -c "CREATE ROLE dbt_test_user_2;" +psql -c "CREATE ROLE dbt_test_user_3;" + +psql -c 'CREATE DATABASE "dbtMixedCase";' +psql -c 'GRANT CREATE, CONNECT ON DATABASE "dbtMixedCase" TO root WITH GRANT OPTION;' + +set +x