From 7934e2dedff16ad009558b7edebf98dffa02841e Mon Sep 17 00:00:00 2001 From: Tommy Gatti Date: Fri, 30 Aug 2024 11:17:10 +1000 Subject: [PATCH] Fix Successful Repro Pipeline Errors (#56) * config-pr-1-ci.yml: Use bash arithmetic context for conditional * config-pr-1-ci.yml: When checking that merging is availible, check that source version is not empty as well * config-pr-1-ci.yml: Simplified usage of compared-config-tag in repro comment * config-pr-2-confirm.yml: Check that version is empty, not just null * test-repro.yml: Added artifact-url as a output of the workflow * config-schedule-2-start.yml: Use arithmetic context for numerical comparison * config-schedule-2-start.yml: Simplified usage of inputs.config-tag in comment --- .github/workflows/config-pr-1-ci.yml | 17 +++++++++++------ .github/workflows/config-pr-2-confirm.yml | 2 +- .github/workflows/config-schedule-2-start.yml | 14 ++++++-------- .github/workflows/test-repro.yml | 3 +++ 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/config-pr-1-ci.yml b/.github/workflows/config-pr-1-ci.yml index 7b70b2e..778b469 100644 --- a/.github/workflows/config-pr-1-ci.yml +++ b/.github/workflows/config-pr-1-ci.yml @@ -248,7 +248,8 @@ jobs: id: results run: | echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT - if [ "${{ fromJson(steps.tests.outputs.json).stats.tests_fail }}" > 0 ]; then + + if (( ${{ fromJson(steps.tests.outputs.json).stats.tests_fail }} > 0 )); then echo "result=fail" >> $GITHUB_OUTPUT else echo "result=pass" >> $GITHUB_OUTPUT @@ -275,12 +276,15 @@ jobs: path: source - name: Modification Check + # If the PR branch version is different to the target branch (but isn't null or empty) then allow merging run: | target=$(yq e '.version' ./target/metadata.yaml) source=$(yq e '.version' ./source/metadata.yaml) - if [[ "${source}" != "${target}" && "${source}" != "null" ]]; then - echo "::notice::The version has been modified to ${target}. Merging is now availible" + echo "Comparing version target '$target' against source '$source'" + + if [[ "${source}" != "${target}" && "${source}" != "null" && "${source}" != "" ]]; then + echo "::notice::The version has been modified to ${source}. Merging is now availible" else echo "::error::The version has not been modified in this PR. Merging is disallowed until an appropriate '!bump' is issued" exit 1 @@ -297,14 +301,15 @@ jobs: permissions: pull-requests: write env: - COMPARED_CHECKSUM_STRING: ${{ needs.check-checksum.outputs.compared-checksum-version != '' && format('The checksums compared against are found here {0}/{1}/tree/{2}/testing/checksum', github.server_url, github.repository, needs.check-checksum.outputs.compared-checksum-version) || '' }} + COMPARED_TAG: ${{ needs.config.outputs.compared-config-tag }} + COMPARED_CHECKSUM_STRING: ${{ needs.config.outputs.compared-config-tag != '' && format('The checksums compared against are found here {0}/{1}/tree/{2}/testing/checksum', github.server_url, github.repository, needs.config.outputs.compared-config-tag) || '' }} steps: - name: Successful Repro Comment if: needs.check-checksum.outputs.result == 'pass' uses: access-nri/actions/.github/actions/pr-comment@main with: comment: | - :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ needs.check-checksum.outputs.compared-checksum-version }}` :white_check_mark: + :white_check_mark: The Bitwise Reproducibility check succeeded when comparing against `${{ env.COMPARED_TAG }}` :white_check_mark: You must bump the minor version of this configuration - to bump the version, comment `!bump minor` or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
@@ -323,7 +328,7 @@ jobs: uses: access-nri/actions/.github/actions/pr-comment@main with: comment: | - :x: The Bitwise Reproducibility check failed ${{ needs.check-checksum.outputs.compared-checksum-version != '' && format('when comparing against `{0}`', needs.check-checksum.outputs.compared-checksum-version) || 'as there is no earlier checksum to compare against' }} :x: + :x: The Bitwise Reproducibility check failed ${{ env.COMPARED_TAG != '' && format('when comparing against `{0}`', env.COMPARED_TAG) || 'as there is no earlier checksum to compare against' }} :x: You must bump the major version of this configuration - to bump the version, comment `!bump major`or modify the `version` in `metadata.yaml`. The meaning of these version bumps is explained in the README.md, under `Config Tags`.
diff --git a/.github/workflows/config-pr-2-confirm.yml b/.github/workflows/config-pr-2-confirm.yml index 40f9834..6b1c86a 100644 --- a/.github/workflows/config-pr-2-confirm.yml +++ b/.github/workflows/config-pr-2-confirm.yml @@ -69,7 +69,7 @@ jobs: run: | version=$(yq '.version' metadata.yaml) - if [[ "${version}" == "null" ]]; then + if [[ "${version}" == "null" || "${version}" == "" ]]; then echo "before=null" >> $GITHUB_OUTPUT echo "after=1.0" >> $GITHUB_OUTPUT exit 0 diff --git a/.github/workflows/config-schedule-2-start.yml b/.github/workflows/config-schedule-2-start.yml index 99eec82..f146bd0 100644 --- a/.github/workflows/config-schedule-2-start.yml +++ b/.github/workflows/config-schedule-2-start.yml @@ -69,8 +69,6 @@ jobs: check-run-url: ${{ steps.results.outputs.check-url }} # Overall result of the checksum repro CI - `pass` (if reproducible), `fail` otherwise result: ${{ steps.results.outputs.result }} - # Version of the checksum compared against the newly generated one - compared-checksum-version: ${{ steps.results.outputs.compared-checksum-version }} steps: - name: Download Newly Created Checksum uses: actions/download-artifact@v4 @@ -93,8 +91,8 @@ jobs: id: results run: | echo "check-url=${{ fromJson(steps.tests.outputs.json).check_url }}" >> $GITHUB_OUTPUT - echo "compared-checksum-version=${{ inputs.config-tag }}" >> $GITHUB_OUTPUT - if [ "${{ fromJson(steps.tests.outputs.json).stats.tests_fail }}" -gt "0" ]; then + + if (( ${{ fromJson(steps.tests.outputs.json).stats.tests_fail }} > 0 )); then echo "result=fail" >> $GITHUB_OUTPUT else echo "result=pass" >> $GITHUB_OUTPUT @@ -120,7 +118,7 @@ jobs: model=${config%-*} echo "model=$model" >> $GITHUB_OUTPUT echo "model-url=https://github.com/ACCESS-NRI/$model" >> $GITHUB_OUTPUT - echo "tag-url=https://github.com/ACCESS-NRI/$config/releases/tag/${{ needs.check-checksum.outputs.compared-checksum-version }}" >> GITHUB_OUTPUT + echo "tag-url=https://github.com/ACCESS-NRI/$config/releases/tag/${{ inputs.config-tag }}" >> GITHUB_OUTPUT echo "run-url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT cat $GITHUB_OUTPUT @@ -134,15 +132,15 @@ jobs: Model: `${{ steps.variables.outputs.model }}`, found here: ${{ steps.variables.outputs.model-url }} Config Repo: `${{ steps.variables.outputs.config }}`, found here: ${{ steps.variables.outputs.config-url }} - Config Tag Tested for Reproducibility: `${{ needs.check-checksum.outputs.compared-checksum-version }}`, found here: ${{ steps.variables.outputs.tag-url }} + Config Tag Tested for Reproducibility: `${{ inputs.config-tag }}`, found here: ${{ steps.variables.outputs.tag-url }} Failed Run Log: ${{ steps.variables.outputs.run-url }} Experiment Location (Gadi): `${{ needs.repro-ci.outputs.experiment-location }}` Checksums created: In the `testing/checksum` directory of ${{ needs.repro-ci.outputs.artifact-url }} - Checksums compared against: ${{ format('{0}/{1}/tree/{2}/testing/checksum', github.server_url, github.repository, needs.check-checksum.outputs.compared-checksum-version) }} + Checksums compared against: ${{ format('{0}/{1}/tree/{2}/testing/checksum', github.server_url, github.repository, inputs.config-tag) }} Tagging @ACCESS-NRI/model-release run: | gh issue create \ - --title 'Scheduled Repro Check Failed for Config `${{ needs.check-checksum.outputs.compared-checksum-version }}`' \ + --title 'Scheduled Repro Check Failed for Config `${{ inputs.config-tag }}`' \ --label "type:repro-fail,priority:blocker" \ --body '${{ env.BODY }}' diff --git a/.github/workflows/test-repro.yml b/.github/workflows/test-repro.yml index 63cf7cd..612f8b7 100644 --- a/.github/workflows/test-repro.yml +++ b/.github/workflows/test-repro.yml @@ -30,6 +30,9 @@ on: artifact-name: value: ${{ jobs.repro.outputs.artifact-name }} description: Name of the artifact containing the checksums and test report for this repro run + artifact-url: + value: ${{ jobs.repro.outputs.artifact-url }} + description: URL to the artifact containing the checksums and test report for this repro run experiment-location: value: ${{ jobs.repro.outputs.experiment-location }} description: Location of the experiment on the target environment