From 727dcc9b5f0cb01dd8d174ac1db96bed5e3bbe72 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:00:33 +0800 Subject: [PATCH 01/76] comment to trigger 1 --- .../workflows/comment-to-trigger-teamcity.yml | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/comment-to-trigger-teamcity.yml diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml new file mode 100644 index 00000000000000..b61b88ba174abd --- /dev/null +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Comment to trigger teamcity + +on: + issue_comment: + types: [created, edited] + +jobs: + check-comment-if-need-to-trigger-teamcity: + + # This job only runs for pull request comments + if: ${{ github.event.issue.pull_request && contains("${COMMENT_BODY}", 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), "${COMMENT_BODY}") }} + + runs-on: ubuntu-latest + + env: + COMMENT_BODY: ${{ github.event.comment.body }} + + steps: + - name: "Parse PR comment" + id: pr_comment + run: | + echo "COMMENT_BODY: ${COMMENT_BODY}" + reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" + COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" + COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" + echo "::set-env name=COMMENT_TRIGGER_TYPE::${COMMENT_TRIGGER_TYPE}" + echo "::set-env name=COMMENT_REPEAT_TIMES::${COMMENT_REPEAT_TIMES}" + + - name: "Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )" + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + submodules: recursive + + - name: Paths filter + uses: ./.github/actions/paths-filter + id: filter + with: + filters: | + fe_ut: + - 'fe/*' + - 'fe_plugins/*' + - 'bin/start_fe.sh' + - 'bin/stop_fe.sh' + - 'run-fe-ut.sh' + - 'docs/zh-CN/docs/sql-manual/*' + - 'docs/en/docs/sql-manual/*' + + - name: Trigger fe ut + if: ${{ $COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} + run: | + echo "COMMENT_TRIGGER_TYPE: $COMMENT_TRIGGER_TYPE" + echo "COMMENT_REPEAT_TIMES: $COMMENT_REPEAT_TIMES" + echo "trigger fe ut with teamcity api" + echo "..." + + - name: Skip fe fe_ut + if: ${{ $COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} + run: | + echo "COMMENT_TRIGGER_TYPE: $COMMENT_TRIGGER_TYPE" + echo "COMMENT_REPEAT_TIMES: $COMMENT_REPEAT_TIMES" + echo "skip fe ut with teamcity api, just set to success" + echo "..." + + + + + From 3a3e8015a03f8e80a2463730ce953ba19b970a30 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:06:36 +0800 Subject: [PATCH 02/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index b61b88ba174abd..07a1305a80ea11 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -21,6 +21,9 @@ on: issue_comment: types: [created, edited] +env: + COMMENT_BODY: ${{ github.event.comment.body }} + jobs: check-comment-if-need-to-trigger-teamcity: @@ -29,9 +32,6 @@ jobs: runs-on: ubuntu-latest - env: - COMMENT_BODY: ${{ github.event.comment.body }} - steps: - name: "Parse PR comment" id: pr_comment From 2fb4a744ba6118543cace1eb40a2559d518d2fc5 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:09:25 +0800 Subject: [PATCH 03/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 07a1305a80ea11..6d0e89734be9b5 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -28,7 +28,7 @@ jobs: check-comment-if-need-to-trigger-teamcity: # This job only runs for pull request comments - if: ${{ github.event.issue.pull_request && contains("${COMMENT_BODY}", 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), "${COMMENT_BODY}") }} + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} runs-on: ubuntu-latest From bd92b1cba8078f360763505ca4865b2965c455bd Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:13:42 +0800 Subject: [PATCH 04/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 6d0e89734be9b5..c94afbcd106e3f 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -64,7 +64,7 @@ jobs: - 'docs/en/docs/sql-manual/*' - name: Trigger fe ut - if: ${{ $COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} + if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} run: | echo "COMMENT_TRIGGER_TYPE: $COMMENT_TRIGGER_TYPE" echo "COMMENT_REPEAT_TIMES: $COMMENT_REPEAT_TIMES" @@ -72,7 +72,7 @@ jobs: echo "..." - name: Skip fe fe_ut - if: ${{ $COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} + if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} run: | echo "COMMENT_TRIGGER_TYPE: $COMMENT_TRIGGER_TYPE" echo "COMMENT_REPEAT_TIMES: $COMMENT_REPEAT_TIMES" From 02c20cec74ecb2820d80ea70e37e711ab0a07018 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:16:57 +0800 Subject: [PATCH 05/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index c94afbcd106e3f..7fc090a20d21c5 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -28,12 +28,13 @@ jobs: check-comment-if-need-to-trigger-teamcity: # This job only runs for pull request comments - if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} + # if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} runs-on: ubuntu-latest steps: - name: "Parse PR comment" + if: ${{ github.event.issue.pull_request }} id: pr_comment run: | echo "COMMENT_BODY: ${COMMENT_BODY}" @@ -44,6 +45,7 @@ jobs: echo "::set-env name=COMMENT_REPEAT_TIMES::${COMMENT_REPEAT_TIMES}" - name: "Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )" + if: ${{ contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} From 83c031007e36c03799d73c0f6b659eff5a0fc3c7 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:37:53 +0800 Subject: [PATCH 06/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 7fc090a20d21c5..267f588f45c2c3 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -41,8 +41,8 @@ jobs: reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" - echo "::set-env name=COMMENT_TRIGGER_TYPE::${COMMENT_TRIGGER_TYPE}" - echo "::set-env name=COMMENT_REPEAT_TIMES::${COMMENT_REPEAT_TIMES}" + echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" >> "$GITHUB_OUTPUT" + echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" >> "$GITHUB_OUTPUT" - name: "Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )" if: ${{ contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} @@ -68,16 +68,16 @@ jobs: - name: Trigger fe ut if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} run: | - echo "COMMENT_TRIGGER_TYPE: $COMMENT_TRIGGER_TYPE" - echo "COMMENT_REPEAT_TIMES: $COMMENT_REPEAT_TIMES" + echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" + echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "trigger fe ut with teamcity api" echo "..." - name: Skip fe fe_ut if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} run: | - echo "COMMENT_TRIGGER_TYPE: $COMMENT_TRIGGER_TYPE" - echo "COMMENT_REPEAT_TIMES: $COMMENT_REPEAT_TIMES" + echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" + echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "skip fe ut with teamcity api, just set to success" echo "..." From fe4699e71344cc1caa9319195a8ee3c1c570d6ba Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 14:45:23 +0800 Subject: [PATCH 07/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 267f588f45c2c3..ae141c1a40ebad 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -44,6 +44,15 @@ jobs: echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" >> "$GITHUB_OUTPUT" echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" >> "$GITHUB_OUTPUT" + - name: test1 + if: ${{ contains(github.event.comment.body, 'run') }} + run: | + echo "contains run" + - name: test2 + if: ${{ contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} + run: | + echo "contains run type" + - name: "Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )" if: ${{ contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} uses: actions/checkout@v4 @@ -52,8 +61,8 @@ jobs: submodules: recursive - name: Paths filter - uses: ./.github/actions/paths-filter - id: filter + uses: dorny/paths-filter@v2 + id: changes with: filters: | fe_ut: @@ -66,7 +75,7 @@ jobs: - 'docs/en/docs/sql-manual/*' - name: Trigger fe ut - if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} + if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} run: | echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" @@ -74,7 +83,7 @@ jobs: echo "..." - name: Skip fe fe_ut - if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.filter.outputs.fe_ut == 'true' }} + if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} run: | echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" From 3056204982edf531d741cd37672629af801d39bc Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 15:04:09 +0800 Subject: [PATCH 08/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index ae141c1a40ebad..53ff4dbc43b174 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -49,7 +49,7 @@ jobs: run: | echo "contains run" - name: test2 - if: ${{ contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} + if: ${{ contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} run: | echo "contains run type" From bb030468e565e800a989bc0e61f5b4825ae2067d Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 15:41:55 +0800 Subject: [PATCH 09/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 53ff4dbc43b174..b4fb3db1606c2f 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -28,13 +28,12 @@ jobs: check-comment-if-need-to-trigger-teamcity: # This job only runs for pull request comments - # if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} + if: ${{ github.event.issue.pull_request && contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} runs-on: ubuntu-latest steps: - - name: "Parse PR comment" - if: ${{ github.event.issue.pull_request }} + - name: "Parse comment" id: pr_comment run: | echo "COMMENT_BODY: ${COMMENT_BODY}" @@ -44,20 +43,17 @@ jobs: echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" >> "$GITHUB_OUTPUT" echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" >> "$GITHUB_OUTPUT" - - name: test1 - if: ${{ contains(github.event.comment.body, 'run') }} + - name: 获取关联的 PR + id: extract run: | - echo "contains run" - - name: test2 - if: ${{ contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} - run: | - echo "contains run type" + pr_number=$(jq --raw-output .issue.pull_request.number "${GITHUB_EVENT_PATH}") + echo "Associated PR number is $pr_number" + echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT" - - name: "Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )" - if: ${{ contains(github.event.comment.body, 'run') && contains(fromJSON('["buildall", "compile", "beut", "feut", "p0", "p1", "external", "pipelinex_p0", "clickbench", "arm", "tpch"]'), github.event.comment.body) }} - uses: actions/checkout@v4 + - name: "Checkout ${{ steps.extract.outputs.pr_number }}" + uses: actions/checkout@v3 with: - ref: ${{ github.event.pull_request.head.sha }} + ref: pull/${{ steps.extract.outputs.pr_number }}/head submodules: recursive - name: Paths filter From e8ac52a17737981990557548163fcbf4b5d482cf Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 15:51:57 +0800 Subject: [PATCH 10/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index b4fb3db1606c2f..21849f015013d9 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -22,6 +22,8 @@ on: types: [created, edited] env: + PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') + LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" > pr_details.json && jq -r '.head.sha' pr_details.json) COMMENT_BODY: ${{ github.event.comment.body }} jobs: @@ -46,14 +48,13 @@ jobs: - name: 获取关联的 PR id: extract run: | - pr_number=$(jq --raw-output .issue.pull_request.number "${GITHUB_EVENT_PATH}") - echo "Associated PR number is $pr_number" - echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT" + echo "PULL_REQUEST_NUM is: ${PULL_REQUEST_NUM}" + echo "LATEST_COMMIT is: ${LATEST_COMMIT}" - name: "Checkout ${{ steps.extract.outputs.pr_number }}" uses: actions/checkout@v3 with: - ref: pull/${{ steps.extract.outputs.pr_number }}/head + ref: pull/${PULL_REQUEST_NUM}/head submodules: recursive - name: Paths filter From 515ef8bc5f15ac4aed1a0fc9a0abbf454a00130f Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 15:53:53 +0800 Subject: [PATCH 11/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 21849f015013d9..4050c7dd7e8df9 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -48,13 +48,13 @@ jobs: - name: 获取关联的 PR id: extract run: | - echo "PULL_REQUEST_NUM is: ${PULL_REQUEST_NUM}" - echo "LATEST_COMMIT is: ${LATEST_COMMIT}" + echo "PULL_REQUEST_NUM is: ${{ env.PULL_REQUEST_NUM }}" + echo "LATEST_COMMIT is: ${{ env.LATEST_COMMIT }}" - name: "Checkout ${{ steps.extract.outputs.pr_number }}" uses: actions/checkout@v3 with: - ref: pull/${PULL_REQUEST_NUM}/head + ref: pull/${{ env.PULL_REQUEST_NUM }}/head submodules: recursive - name: Paths filter From 33b5f5996c2554dc4ede89ea5206f398628cfc83 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 16:07:51 +0800 Subject: [PATCH 12/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 4050c7dd7e8df9..06c73a689b2297 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -23,7 +23,7 @@ on: env: PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" > pr_details.json && jq -r '.head.sha' pr_details.json) + LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) COMMENT_BODY: ${{ github.event.comment.body }} jobs: @@ -51,10 +51,10 @@ jobs: echo "PULL_REQUEST_NUM is: ${{ env.PULL_REQUEST_NUM }}" echo "LATEST_COMMIT is: ${{ env.LATEST_COMMIT }}" - - name: "Checkout ${{ steps.extract.outputs.pr_number }}" - uses: actions/checkout@v3 + - name: "Checkout PR ${{ env.PULL_REQUEST_NUM }} ${{ env.LATEST_COMMIT }}" + uses: actions/checkout@v4 with: - ref: pull/${{ env.PULL_REQUEST_NUM }}/head + ref: ${{ env.LATEST_COMMIT }} submodules: recursive - name: Paths filter From 4ab6f3905a91c9cd34a726452cde0d417f732334 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 16:15:17 +0800 Subject: [PATCH 13/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 06c73a689b2297..27dc40e3e99482 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -50,11 +50,13 @@ jobs: run: | echo "PULL_REQUEST_NUM is: ${{ env.PULL_REQUEST_NUM }}" echo "LATEST_COMMIT is: ${{ env.LATEST_COMMIT }}" + echo "PULL_REQUEST_NUM=${PULL_REQUEST_NUM}" >> "$GITHUB_OUTPUT" + echo "LATEST_COMMIT=${LATEST_COMMIT}" >> "$GITHUB_OUTPUT" - - name: "Checkout PR ${{ env.PULL_REQUEST_NUM }} ${{ env.LATEST_COMMIT }}" + - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }} ${{ steps.extract.outputs.LATEST_COMMIT }}" uses: actions/checkout@v4 with: - ref: ${{ env.LATEST_COMMIT }} + ref: ${{ steps.extract.outputs.LATEST_COMMIT }} submodules: recursive - name: Paths filter From 9c1fe5b5926fc807d8da62cff0737779c3840766 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 16:17:17 +0800 Subject: [PATCH 14/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 27dc40e3e99482..296e0d1d5064b0 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -21,10 +21,6 @@ on: issue_comment: types: [created, edited] -env: - PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) - COMMENT_BODY: ${{ github.event.comment.body }} jobs: check-comment-if-need-to-trigger-teamcity: @@ -33,6 +29,10 @@ jobs: if: ${{ github.event.issue.pull_request && contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} runs-on: ubuntu-latest + env: + PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') + LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) + COMMENT_BODY: ${{ github.event.comment.body }} steps: - name: "Parse comment" From 55c0507cad094cde7aeab3d02b200e79cf345f19 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 16:21:07 +0800 Subject: [PATCH 15/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 296e0d1d5064b0..e9dc3fe975e1b2 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -21,7 +21,6 @@ on: issue_comment: types: [created, edited] - jobs: check-comment-if-need-to-trigger-teamcity: @@ -50,8 +49,8 @@ jobs: run: | echo "PULL_REQUEST_NUM is: ${{ env.PULL_REQUEST_NUM }}" echo "LATEST_COMMIT is: ${{ env.LATEST_COMMIT }}" - echo "PULL_REQUEST_NUM=${PULL_REQUEST_NUM}" >> "$GITHUB_OUTPUT" - echo "LATEST_COMMIT=${LATEST_COMMIT}" >> "$GITHUB_OUTPUT" + echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" >> "$GITHUB_OUTPUT" + echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" >> "$GITHUB_OUTPUT" - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }} ${{ steps.extract.outputs.LATEST_COMMIT }}" uses: actions/checkout@v4 From f0c8fd9ffb1bb0f18a5ad7dae03441005bc4e3b7 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 16:43:54 +0800 Subject: [PATCH 16/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index e9dc3fe975e1b2..38001251d31201 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -52,25 +52,15 @@ jobs: echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" >> "$GITHUB_OUTPUT" echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" >> "$GITHUB_OUTPUT" - - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }} ${{ steps.extract.outputs.LATEST_COMMIT }}" + - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }}, ${{ steps.extract.outputs.LATEST_COMMIT }}" uses: actions/checkout@v4 with: ref: ${{ steps.extract.outputs.LATEST_COMMIT }} submodules: recursive - - name: Paths filter - uses: dorny/paths-filter@v2 + - name: Get changed files id: changes - with: - filters: | - fe_ut: - - 'fe/*' - - 'fe_plugins/*' - - 'bin/start_fe.sh' - - 'bin/stop_fe.sh' - - 'run-fe-ut.sh' - - 'docs/zh-CN/docs/sql-manual/*' - - 'docs/en/docs/sql-manual/*' + uses: tj-actions/changed-files@v40 - name: Trigger fe ut if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} @@ -81,7 +71,7 @@ jobs: echo "..." - name: Skip fe fe_ut - if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} + if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'false' }} run: | echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" From 48ae415d440fe22eaaa929eadf60194e15ff05de Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 16:53:43 +0800 Subject: [PATCH 17/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 38001251d31201..ef89fae5298d18 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -58,9 +58,26 @@ jobs: ref: ${{ steps.extract.outputs.LATEST_COMMIT }} submodules: recursive - - name: Get changed files + # - name: Get changed files + # id: changes + # uses: tj-actions/changed-files@v40 + # with: + # base_sha: ${{ github.event.pull_request.base.sha }} + + - name: Get Changed Files + id: pr_changed_files + uses: actions/changed-files@v1 + with: + format: 'json' + from: ${{ steps.extract.outputs.PULL_REQUEST_NUM }} + + - name: Filter Changed Files id: changes - uses: tj-actions/changed-files@v40 + uses: actions/paths-filter@v2 + with: + filters: | + fe_ut: + - 'fe/*' - name: Trigger fe ut if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} From 945d82972149326f99fd6f89e8ee084567418ee3 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 17:22:46 +0800 Subject: [PATCH 18/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index ef89fae5298d18..61750fecbe43ed 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -52,32 +52,19 @@ jobs: echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" >> "$GITHUB_OUTPUT" echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" >> "$GITHUB_OUTPUT" - - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }}, ${{ steps.extract.outputs.LATEST_COMMIT }}" + - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.extract.outputs.LATEST_COMMIT }}" uses: actions/checkout@v4 with: ref: ${{ steps.extract.outputs.LATEST_COMMIT }} - submodules: recursive - - # - name: Get changed files - # id: changes - # uses: tj-actions/changed-files@v40 - # with: - # base_sha: ${{ github.event.pull_request.base.sha }} - - - name: Get Changed Files - id: pr_changed_files - uses: actions/changed-files@v1 - with: - format: 'json' - from: ${{ steps.extract.outputs.PULL_REQUEST_NUM }} - - - name: Filter Changed Files + - name: "Check if pr need run build" id: changes - uses: actions/paths-filter@v2 - with: - filters: | - fe_ut: - - 'fe/*' + run: | + cd regression-test/pipeline/common/ + if bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then + echo "fe_ut=true" >> "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "be-ut"; then + echo "be_ut=true" >> "$GITHUB_OUTPUT" + fi - name: Trigger fe ut if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} From 2d4121cd0fd982a3e8ae86cf07141c9732db8b16 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 18:17:14 +0800 Subject: [PATCH 19/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 56 +++++++++++++++++-- .../common/check-pr-if-need-run-build.sh | 2 +- .../pipeline/common/teamcity-utils.sh | 43 ++++++++++++++ 3 files changed, 95 insertions(+), 6 deletions(-) create mode 100644 regression-test/pipeline/common/teamcity-utils.sh diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 61750fecbe43ed..d4b91c5e9bd750 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -56,6 +56,7 @@ jobs: uses: actions/checkout@v4 with: ref: ${{ steps.extract.outputs.LATEST_COMMIT }} + - name: "Check if pr need run build" id: changes run: | @@ -64,23 +65,68 @@ jobs: echo "fe_ut=true" >> "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "be-ut"; then echo "be_ut=true" >> "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "ckb"; then + echo "ckb=true" >> "$GITHUB_OUTPUT" + echo "tpch=true" >> "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then + echo "p0=true" >> "$GITHUB_OUTPUT" + echo "external=true" >> "$GITHUB_OUTPUT" + echo "pipelinex_p0"=true" >> "$GITHUB_OUTPUT" + echo "arm"=true" >> "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then + echo "p1=true" >> "$GITHUB_OUTPUT" fi - name: Trigger fe ut - if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'true' }} + if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'true' }} run: | echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + echo "fe-ut related files changed." echo "trigger fe ut with teamcity api" - echo "..." + source ./regression-test/pipeline/common/teamcity-utils.sh + if get_running_build; then + cancel_running_build + fi + if get_queue_build; then + cancel_queue_build + fi + add_build - - name: Skip fe fe_ut - if: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE == 'feut' && steps.changes.outputs.fe_ut == 'false' }} + - name: Skip fe ut + if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'false' }} run: | echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + echo "fe-ut related files not changed." echo "skip fe ut with teamcity api, just set to success" - echo "..." + source ./regression-test/pipeline/common/teamcity-utils.sh + skip_build + + - name: Trigger be ut + if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.be_ut == 'true' }} + run: | + echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" + echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + echo "be-ut related files changed." + echo "trigger be ut with teamcity api" + if get_running_build; then + cancel_running_build + fi + if get_queue_build; then + cancel_queue_build + fi + add_build + + - name: Skip be ut + if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.be_ut == 'false' }} + run: | + echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" + echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + echo "be-ut related files not changed." + echo "skip be ut with teamcity api, just set to success" + source ./regression-test/pipeline/common/teamcity-utils.sh + skip_build diff --git a/regression-test/pipeline/common/check-pr-if-need-run-build.sh b/regression-test/pipeline/common/check-pr-if-need-run-build.sh index df4cd3c5102c5a..82b39cf86119a0 100755 --- a/regression-test/pipeline/common/check-pr-if-need-run-build.sh +++ b/regression-test/pipeline/common/check-pr-if-need-run-build.sh @@ -48,7 +48,7 @@ _get_pr_changed_files() { while [[ ${try_times} -gt 0 ]]; do if curl \ -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?per_page="${per_page}" \ + https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=10\&per_page="${per_page}" \ 2>/dev/null >"${file_name}"; then break else diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh new file mode 100644 index 00000000000000..c831125b798285 --- /dev/null +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#!/bin/bash + +get_running_build() { + echo "获取pr在某条流水线上正在跑的build" +} + +get_queue_build() { + echo "获取pr在某条流水线上排队的build" +} + +add_build() { + echo "新触发一个build" +} + +cancel_running_build() { + echo "cancel_running_build" +} + +cancel_queue_build() { + echo "cancel_queue_build" +} + +skip_build() { + echo "对于不需要跑流水线的pr, 直接回复github成功" +} From 876b249ead39c08fcfb763c08c6ab4f59ac6c936 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 18:18:52 +0800 Subject: [PATCH 20/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index d4b91c5e9bd750..32abb3a8859fbb 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -84,6 +84,7 @@ jobs: echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "fe-ut related files changed." echo "trigger fe ut with teamcity api" + set -x source ./regression-test/pipeline/common/teamcity-utils.sh if get_running_build; then cancel_running_build @@ -100,6 +101,7 @@ jobs: echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "fe-ut related files not changed." echo "skip fe ut with teamcity api, just set to success" + set -x source ./regression-test/pipeline/common/teamcity-utils.sh skip_build @@ -110,6 +112,7 @@ jobs: echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "be-ut related files changed." echo "trigger be ut with teamcity api" + set -x if get_running_build; then cancel_running_build fi @@ -125,6 +128,7 @@ jobs: echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "be-ut related files not changed." echo "skip be ut with teamcity api, just set to success" + set -x source ./regression-test/pipeline/common/teamcity-utils.sh skip_build From 9651bd4b5ea35b37c0b762e7f2ab690da7e42e14 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:12:45 +0800 Subject: [PATCH 21/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 32abb3a8859fbb..0bd868039d33ff 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -62,26 +62,27 @@ jobs: run: | cd regression-test/pipeline/common/ if bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then - echo "fe_ut=true" >> "$GITHUB_OUTPUT" + echo "fe_ut=true" | tee -a "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "be-ut"; then - echo "be_ut=true" >> "$GITHUB_OUTPUT" + echo "be_ut=true" | tee -a "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "ckb"; then - echo "ckb=true" >> "$GITHUB_OUTPUT" - echo "tpch=true" >> "$GITHUB_OUTPUT" + echo "ckb=true" | tee -a "$GITHUB_OUTPUT" + echo "tpch=true" | tee -a "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then - echo "p0=true" >> "$GITHUB_OUTPUT" - echo "external=true" >> "$GITHUB_OUTPUT" - echo "pipelinex_p0"=true" >> "$GITHUB_OUTPUT" - echo "arm"=true" >> "$GITHUB_OUTPUT" + echo "p0=true" | tee -a "$GITHUB_OUTPUT" + echo "external=true" | tee -a "$GITHUB_OUTPUT" + echo "pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" + echo "arm"=true" | tee -a "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then - echo "p1=true" >> "$GITHUB_OUTPUT" + echo "p1=true" | tee -a "$GITHUB_OUTPUT" fi + echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" + echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + - name: Trigger fe ut if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'true' }} run: | - echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" - echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "fe-ut related files changed." echo "trigger fe ut with teamcity api" set -x @@ -97,8 +98,6 @@ jobs: - name: Skip fe ut if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'false' }} run: | - echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" - echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "fe-ut related files not changed." echo "skip fe ut with teamcity api, just set to success" set -x @@ -108,8 +107,6 @@ jobs: - name: Trigger be ut if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.be_ut == 'true' }} run: | - echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" - echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "be-ut related files changed." echo "trigger be ut with teamcity api" set -x @@ -124,8 +121,6 @@ jobs: - name: Skip be ut if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.be_ut == 'false' }} run: | - echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" - echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" echo "be-ut related files not changed." echo "skip be ut with teamcity api, just set to success" set -x From efc78a1eef5c14b2ffd3b36dce604b0f8acf799f Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:15:14 +0800 Subject: [PATCH 22/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 0bd868039d33ff..4168c8668bf9e5 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -79,6 +79,7 @@ jobs: echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + cd - - name: Trigger fe ut if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'true' }} From 4b2c4c68aa0851560852e2e440c162ebc8c04290 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:15:27 +0800 Subject: [PATCH 23/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 4168c8668bf9e5..4ba106f2e88081 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -87,6 +87,7 @@ jobs: echo "fe-ut related files changed." echo "trigger fe ut with teamcity api" set -x + pwd source ./regression-test/pipeline/common/teamcity-utils.sh if get_running_build; then cancel_running_build From 2eeb6f09987db6f31efbc485181d47407ff07a0b Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:23:39 +0800 Subject: [PATCH 24/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 4ba106f2e88081..1fe132a2e363fa 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -88,6 +88,7 @@ jobs: echo "trigger fe ut with teamcity api" set -x pwd + ls -al ./regression-test/pipeline/common/teamcity-utils.sh source ./regression-test/pipeline/common/teamcity-utils.sh if get_running_build; then cancel_running_build From 7a8c936c173268a3a6c3b0a3aa318a3162a0d28e Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:26:59 +0800 Subject: [PATCH 25/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 1fe132a2e363fa..eca2bcdbda9cf8 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -98,6 +98,10 @@ jobs: fi add_build + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + - name: Skip fe ut if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'false' }} run: | From 88a2fb9b6b3f55067c77c4f58dc1e7bfe99d05cb Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:32:26 +0800 Subject: [PATCH 26/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index eca2bcdbda9cf8..513653f2707e36 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -52,10 +52,13 @@ jobs: echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" >> "$GITHUB_OUTPUT" echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" >> "$GITHUB_OUTPUT" - - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.extract.outputs.LATEST_COMMIT }}" + # - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.extract.outputs.LATEST_COMMIT }}" + # uses: actions/checkout@v4 + # with: + # ref: ${{ steps.extract.outputs.LATEST_COMMIT }} + + - name: "Checkout master" uses: actions/checkout@v4 - with: - ref: ${{ steps.extract.outputs.LATEST_COMMIT }} - name: "Check if pr need run build" id: changes From bb18addaf8354fe04a435b5e92ce89b815a7e203 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:46:41 +0800 Subject: [PATCH 27/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 69 ++++++++----------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 513653f2707e36..60a6ae566c1b2e 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -34,28 +34,22 @@ jobs: COMMENT_BODY: ${{ github.event.comment.body }} steps: - - name: "Parse comment" - id: pr_comment + - name: "Parse PR comment" + id: parse run: | - echo "COMMENT_BODY: ${COMMENT_BODY}" + echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" | tee -a "$GITHUB_OUTPUT" + echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_BODY='$${{ env.COMMENT_BODY }}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" - echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" >> "$GITHUB_OUTPUT" - echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" >> "$GITHUB_OUTPUT" + echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" - - name: 获取关联的 PR - id: extract - run: | - echo "PULL_REQUEST_NUM is: ${{ env.PULL_REQUEST_NUM }}" - echo "LATEST_COMMIT is: ${{ env.LATEST_COMMIT }}" - echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" >> "$GITHUB_OUTPUT" - echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" >> "$GITHUB_OUTPUT" - - # - name: "Checkout PR ${{ steps.extract.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.extract.outputs.LATEST_COMMIT }}" + # - name: "Checkout PR ${{ steps.parse.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.parse.outputs.LATEST_COMMIT }}" # uses: actions/checkout@v4 # with: - # ref: ${{ steps.extract.outputs.LATEST_COMMIT }} + # ref: ${{ steps.parse.outputs.LATEST_COMMIT }} - name: "Checkout master" uses: actions/checkout@v4 @@ -64,34 +58,30 @@ jobs: id: changes run: | cd regression-test/pipeline/common/ - if bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then - echo "fe_ut=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "be-ut"; then - echo "be_ut=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "ckb"; then - echo "ckb=true" | tee -a "$GITHUB_OUTPUT" + if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then + echo "changes_fe_ut=true" | tee -a "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "be-ut"; then + echo "changes_be_ut=true" | tee -a "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "ckb"; then + echo "changes_ckb=true" | tee -a "$GITHUB_OUTPUT" echo "tpch=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then - echo "p0=true" | tee -a "$GITHUB_OUTPUT" - echo "external=true" | tee -a "$GITHUB_OUTPUT" - echo "pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" - echo "arm"=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.extract.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then - echo "p1=true" | tee -a "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then + echo "changes_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changes_external=true" | tee -a "$GITHUB_OUTPUT" + echo "changes_pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" + echo "changes_arm"=true" | tee -a "$GITHUB_OUTPUT" + elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then + echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" fi - - echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE }}" - echo "-- COMMENT_REPEAT_TIMES: ${{ steps.pr_comment.outputs.COMMENT_REPEAT_TIMES }}" + echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" + echo "-- COMMENT_REPEAT_TIMES: ${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" cd - - name: Trigger fe ut - if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'true' }} + if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'true' }} run: | echo "fe-ut related files changed." echo "trigger fe ut with teamcity api" - set -x - pwd - ls -al ./regression-test/pipeline/common/teamcity-utils.sh source ./regression-test/pipeline/common/teamcity-utils.sh if get_running_build; then cancel_running_build @@ -106,20 +96,18 @@ jobs: uses: mxschmitt/action-tmate@v3 - name: Skip fe ut - if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.fe_ut == 'false' }} + if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'false' }} run: | echo "fe-ut related files not changed." echo "skip fe ut with teamcity api, just set to success" - set -x source ./regression-test/pipeline/common/teamcity-utils.sh skip_build - name: Trigger be ut - if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.be_ut == 'true' }} + if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'true' }} run: | echo "be-ut related files changed." echo "trigger be ut with teamcity api" - set -x if get_running_build; then cancel_running_build fi @@ -129,11 +117,10 @@ jobs: add_build - name: Skip be ut - if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.pr_comment.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.be_ut == 'false' }} + if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'false' }} run: | echo "be-ut related files not changed." echo "skip be ut with teamcity api, just set to success" - set -x source ./regression-test/pipeline/common/teamcity-utils.sh skip_build From ffcd9fd211555c4e305711812023460b58f5365a Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:47:40 +0800 Subject: [PATCH 28/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 60a6ae566c1b2e..e72dc61aa48463 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -39,7 +39,7 @@ jobs: run: | echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" | tee -a "$GITHUB_OUTPUT" echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" | tee -a "$GITHUB_OUTPUT" - echo "COMMENT_BODY='$${{ env.COMMENT_BODY }}'" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_BODY='${{ env.COMMENT_BODY }}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" From b4dc05c1e5eee0a7951cc16b0ce9b0264939defa Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 19:58:12 +0800 Subject: [PATCH 29/76] 2 --- .github/workflows/comment-to-trigger-teamcity.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index e72dc61aa48463..8b1655f32ea393 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -58,6 +58,7 @@ jobs: id: changes run: | cd regression-test/pipeline/common/ + set -x if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then echo "changes_fe_ut=true" | tee -a "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "be-ut"; then @@ -72,6 +73,8 @@ jobs: echo "changes_arm"=true" | tee -a "$GITHUB_OUTPUT" elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" + else + exit 1 fi echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" echo "-- COMMENT_REPEAT_TIMES: ${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" From 56584e711387e8289a27b79d060228b5f549184c Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 20:07:49 +0800 Subject: [PATCH 30/76] 2 --- regression-test/pipeline/common/check-pr-if-need-run-build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/pipeline/common/check-pr-if-need-run-build.sh b/regression-test/pipeline/common/check-pr-if-need-run-build.sh index 82b39cf86119a0..6cf7eee88ce645 100755 --- a/regression-test/pipeline/common/check-pr-if-need-run-build.sh +++ b/regression-test/pipeline/common/check-pr-if-need-run-build.sh @@ -48,7 +48,7 @@ _get_pr_changed_files() { while [[ ${try_times} -gt 0 ]]; do if curl \ -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=10\&per_page="${per_page}" \ + https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=1\&per_page="${per_page}" \ 2>/dev/null >"${file_name}"; then break else @@ -61,7 +61,7 @@ _get_pr_changed_files() { added_files=$(jq -r '.[] | select(.status == "added") | .filename' "${file_name}") modified_files=$(jq -r '.[] | select(.status == "modified") | .filename' "${file_name}") removed_files=$(jq -r '.[] | select(.status == "removed") | .filename' "${file_name}") - rm "${file_name}" + # rm "${file_name}" if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi echo -e " From 473b3890c4b6789ba3e574014bd26fa866528958 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 20:21:17 +0800 Subject: [PATCH 31/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 21 +++++++++++++------ .../common/check-pr-if-need-run-build.sh | 1 + 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 8b1655f32ea393..968a2d7fec4075 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -76,11 +76,20 @@ jobs: else exit 1 fi - echo "-- COMMENT_TRIGGER_TYPE: ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" - echo "-- COMMENT_REPEAT_TIMES: ${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + echo "-- COMMENT_TRIGGER_TYPE: '${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}'" + echo "-- COMMENT_REPEAT_TIMES: '${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}'" cd - + + - name: "test" + if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + echo "contains feut or buildall" + - name: "test2" + if: ${{ bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "fe-ut" }} + run: | + echo "check-pr need run build fe-ut" - - name: Trigger fe ut + - name: "Trigger fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'true' }} run: | echo "fe-ut related files changed." @@ -94,7 +103,7 @@ jobs: fi add_build - - name: Setup tmate session + - name: "Setup tmate session" if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 @@ -106,7 +115,7 @@ jobs: source ./regression-test/pipeline/common/teamcity-utils.sh skip_build - - name: Trigger be ut + - name: "Trigger be ut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'true' }} run: | echo "be-ut related files changed." @@ -119,7 +128,7 @@ jobs: fi add_build - - name: Skip be ut + - name: "Skip be ut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'false' }} run: | echo "be-ut related files not changed." diff --git a/regression-test/pipeline/common/check-pr-if-need-run-build.sh b/regression-test/pipeline/common/check-pr-if-need-run-build.sh index 6cf7eee88ce645..daf5ee6df49763 100755 --- a/regression-test/pipeline/common/check-pr-if-need-run-build.sh +++ b/regression-test/pipeline/common/check-pr-if-need-run-build.sh @@ -46,6 +46,7 @@ _get_pr_changed_files() { per_page=100 file_name='pr_change_files' while [[ ${try_times} -gt 0 ]]; do + # TODO: 修改文件多于100个的情况未处理 if curl \ -H "Accept: application/vnd.github+json" \ https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=1\&per_page="${per_page}" \ From b8689955bae67c2bd0877e2d18f42ec73fa4ed2f Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 20:33:12 +0800 Subject: [PATCH 32/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 968a2d7fec4075..839f41fda2bc61 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -61,33 +61,32 @@ jobs: set -x if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then echo "changes_fe_ut=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "be-ut"; then + fi + if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "be-ut"; then echo "changes_be_ut=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "ckb"; then + fi + if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "ckb"; then echo "changes_ckb=true" | tee -a "$GITHUB_OUTPUT" echo "tpch=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then + fi + if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then echo "changes_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changes_external=true" | tee -a "$GITHUB_OUTPUT" echo "changes_pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" echo "changes_arm"=true" | tee -a "$GITHUB_OUTPUT" - elif bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then + fi + if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" - else - exit 1 fi echo "-- COMMENT_TRIGGER_TYPE: '${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}'" echo "-- COMMENT_REPEAT_TIMES: '${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}'" cd - - + - name: "test" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | echo "contains feut or buildall" - - name: "test2" - if: ${{ bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "fe-ut" }} - run: | - echo "check-pr need run build fe-ut" + - name: "Trigger fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'true' }} From 8a22eaa16a26a22803adbcdd9fc9a4b81b0b61fe Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 20:45:36 +0800 Subject: [PATCH 33/76] 2 --- .../workflows/comment-to-trigger-teamcity.yml | 39 ++++--------------- .../common/check-pr-if-need-run-build.sh | 2 +- .../pipeline/common/teamcity-utils.sh | 4 ++ 3 files changed, 12 insertions(+), 33 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 839f41fda2bc61..5ba7310f24b2cc 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -82,56 +82,31 @@ jobs: echo "-- COMMENT_REPEAT_TIMES: '${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}'" cd - - - name: "test" - if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} - run: | - echo "contains feut or buildall" - - - name: "Trigger fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'true' }} run: | - echo "fe-ut related files changed." - echo "trigger fe ut with teamcity api" source ./regression-test/pipeline/common/teamcity-utils.sh - if get_running_build; then - cancel_running_build - fi - if get_queue_build; then - cancel_queue_build - fi - add_build + cancel_and_add_build - - name: "Setup tmate session" - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 + # - name: "Setup tmate session" + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 - - name: Skip fe ut + - name: "Skip fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'false' }} run: | - echo "fe-ut related files not changed." - echo "skip fe ut with teamcity api, just set to success" source ./regression-test/pipeline/common/teamcity-utils.sh skip_build - name: "Trigger be ut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'true' }} run: | - echo "be-ut related files changed." - echo "trigger be ut with teamcity api" - if get_running_build; then - cancel_running_build - fi - if get_queue_build; then - cancel_queue_build - fi - add_build + source ./regression-test/pipeline/common/teamcity-utils.sh + cancel_and_add_build - name: "Skip be ut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'false' }} run: | - echo "be-ut related files not changed." - echo "skip be ut with teamcity api, just set to success" source ./regression-test/pipeline/common/teamcity-utils.sh skip_build diff --git a/regression-test/pipeline/common/check-pr-if-need-run-build.sh b/regression-test/pipeline/common/check-pr-if-need-run-build.sh index daf5ee6df49763..549e5976ed183f 100755 --- a/regression-test/pipeline/common/check-pr-if-need-run-build.sh +++ b/regression-test/pipeline/common/check-pr-if-need-run-build.sh @@ -62,7 +62,7 @@ _get_pr_changed_files() { added_files=$(jq -r '.[] | select(.status == "added") | .filename' "${file_name}") modified_files=$(jq -r '.[] | select(.status == "modified") | .filename' "${file_name}") removed_files=$(jq -r '.[] | select(.status == "removed") | .filename' "${file_name}") - # rm "${file_name}" + rm "${file_name}" if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi echo -e " diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index c831125b798285..d44ea65d47637b 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -41,3 +41,7 @@ cancel_queue_build() { skip_build() { echo "对于不需要跑流水线的pr, 直接回复github成功" } + +cancel_and_add_build() { + ehco "try to cancel queue or running build first, then add build" +} \ No newline at end of file From 58ccd3cab4880b23fce69fee36b2a75ba4f747ff Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 15 Nov 2023 20:47:17 +0800 Subject: [PATCH 34/76] 2 --- regression-test/pipeline/common/teamcity-utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index d44ea65d47637b..ff81e293833055 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -43,5 +43,5 @@ skip_build() { } cancel_and_add_build() { - ehco "try to cancel queue or running build first, then add build" -} \ No newline at end of file + echo "try to cancel queue or running build first, then add build" +} From 7659aefb3cc40449becd112f49c715614f677a08 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 19:06:39 +0800 Subject: [PATCH 35/76] 3 --- .../workflows/comment-to-trigger-teamcity.yml | 23 ++- .../pipeline/common/teamcity-utils.sh | 154 +++++++++++++++++- 2 files changed, 160 insertions(+), 17 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 5ba7310f24b2cc..958ed9dedce122 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -32,6 +32,7 @@ jobs: PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) COMMENT_BODY: ${{ github.event.comment.body }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - name: "Parse PR comment" @@ -82,21 +83,27 @@ jobs: echo "-- COMMENT_REPEAT_TIMES: '${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}'" cd - - - name: "Trigger fe ut" - if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'true' }} + - name: "Trigger or Skip fe ut" + if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | + echo "GITHUB_TOKEN: ${GITHUB_TOKEN}" source ./regression-test/pipeline/common/teamcity-utils.sh - cancel_and_add_build - - # - name: "Setup tmate session" - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 + if [[ ${{ steps.changes.outputs.changes_fe_ut }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" + fi - name: "Skip fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'false' }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - skip_build + skip_teamcity_pipeline "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" + + # - name: "Setup tmate session" + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 + - name: "Trigger be ut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'true' }} diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index ff81e293833055..d552e377b3f9d5 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -18,30 +18,166 @@ #!/bin/bash -get_running_build() { - echo "获取pr在某条流水线上正在跑的build" +# github中评论的要触发的流水线名字 +# 到 +# teamcity流水线实际的名称 +# 的映射 +declare -A comment_to_pipeline +comment_to_pipeline=( + ['feut']='Doris_Doris_FeUt' + ['beut']='Doris_DorisBeUt_BeUt' + ['compile']='Doris_DorisCompile_Compile' + ['p0']='Doris_DorisRegression_P0Regression' + ['p1']='Doris_DorisRegression_P1Regression' + ['external']='Doris_External_Regression' + ['clickbench']='Doris_Performance_Clickbench_ClickbenchNew' + ['pipelinex_p0']='Doris_DorisRegression_P0RegressionPipelineX' + ['arm']='Doris_ArmPipeline_P0Regression' + ['tpch']='Tpch_TpchSf100' +) + + +get_commit_id_of_build() { + # 获取某个build的commit id + if [[ -z "$1" ]]; then return 1; fi + build_id="$1" + if ret=$( + curl -s -X GET \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:text/plain" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/app/rest/builds/${build_id}" + ); then + commit_id=$(echo "${ret}" | jq -r '.revisions.revision[0].version') + echo "${commit_id}" + else + return 1 + fi +} + +get_running_build_of_pr() { + # "获取pr在某条流水线上正在跑的build" + if [[ -z "${PIPELINE}" || -z "${PULL_REQUEST_NUM}" ]]; then + echo "ERROR: env PIPELINE or PULL_REQUEST_NUM not set." + return 1 + fi + local running_builds_list + if ret=$( + curl -s -X GET \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:text/plain" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/app/rest/builds?locator=buildType:${PIPELINE},branch:pull/${PULL_REQUEST_NUM},running:true" + ); then + running_builds_list=$(echo "${ret}" | jq -r '.build[].id') + echo "${running_builds_list}" + else + return 1 + fi } -get_queue_build() { - echo "获取pr在某条流水线上排队的build" +get_queue_build_of_pr() { + # "获取pr在某条流水线上排队的build" + local queue_builds_list + if ret=$( + curl -s -X GET \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:text/plain" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/app/rest/buildQueue?locator=buildType:${PIPELINE}" + ); then + queue_builds_list=$(echo "${ret}" | jq ".build[] | select(.branchName == \"pull/${PULL_REQUEST_NUM}\") | .id") + echo "${queue_builds_list}" + else + return 1 + fi } +# get_queue_build_of_pr "$1" "$2" + add_build() { - echo "新触发一个build" + # 新触发一个build + if [[ -z "$2" ]]; then + echo "Usage: add_build PIPELINE PULL_REQUEST_NUM [COMMENT_REPEAT_TIMES]" + return 1 + fi + PULL_REQUEST_NUM="$1" + COMMENT_TRIGGER_TYPE="$2" + COMMENT_REPEAT_TIMES="$3" + + if [[ -z "${COMMIT_ID_FROM_TRIGGER}" ]]; then + echo "WARNINR: env COMMIT_ID_FROM_TRIGGER not set" + fi + local PIPELINE + PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + if ret=$( + curl -s -X POST \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:text/plain" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.commit_id_from_trigger\&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times&value=${COMMENT_REPEAT_TIMES:-1}" + ); then + echo "INFO: Add new build to PIPELINE ${PIPELINE} of PR ${PULL_REQUEST_NUM} with COMMENT_REPEAT_TIMES ${COMMENT_REPEAT_TIMES:-1}" + else + return 1 + fi } cancel_running_build() { - echo "cancel_running_build" + echo "TODO: cancel_running_build" } cancel_queue_build() { - echo "cancel_queue_build" + echo "TODO: cancel_queue_build" } -skip_build() { - echo "对于不需要跑流水线的pr, 直接回复github成功" +function skip_build() { + # 对于不需要跑teamcity pipeline的PR，直接调用github的接口返回成功 + if [[ -z "${GITHUB_TOKEN}" ]]; then + echo "ERROR: env GITHUB_TOKEN not set" + return 1 + fi + if [[ -z "$2" ]]; then + echo "Usage: skip_teamcity_pipeline PR_COMMIT_ID COMMENT_TRIGGER_TYPE" + return 1 + fi + PR_COMMIT_ID="$1" + COMMENT_TRIGGER_TYPE="$2" + + # github中评论的要触发的流水线名字 + # 到 + # teamcity流水线返回结果给github的名称 + # 的映射 + # 新加流水线需要修改这里 + declare -A context_map + context_map=(['compile']='COMPILE (DORIS_COMPILE)' + ['feut']='FE UT (Doris FE UT)' + ['beut']='BE UT (Doris BE UT)' + ['p0']='P0 Regression (Doris Regression)' + ['p1']='P1 Regression (Doris Regression)' + ['external']='External Regression (Doris External Regression)' + ['pipelinex_p0']='P0 Regression PipelineX (Doris Regression)' + ['clickbench']='clickbench-new (clickbench)' + ['arm']='P0 Regression (ARM pipeline)' + ['tpch']='tpch-sf100 (tpch)') + local state="${TC_BUILD_STATE:-success}" # 可选值 success failure pending + payload="{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"Skip teamCity build\",\"context\":\"${context_map[${COMMENT_TRIGGER_TYPE}]}\"}" + if curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN:-}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/apache/doris/statuses/${PR_COMMIT_ID:-}" \ + -d "${payload}"; then + echo "INFO: Skipped ${PR_COMMIT_ID} ${COMMENT_TRIGGER_TYPE}" + else + return 1 + fi } +# GITHUB_TOKEN=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 +# skip_teamcity_pipeline '9940812522228f574c5b630666189bc0aa4b1c60' clickbench + cancel_and_add_build() { echo "try to cancel queue or running build first, then add build" } From a3d8d38e2999eb4709b847db178519c4be96c994 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 20:02:27 +0800 Subject: [PATCH 36/76] 3 --- .../workflows/comment-to-trigger-teamcity.yml | 34 ++++++--------- .../pipeline/common/teamcity-utils.sh | 41 ++++++++++--------- 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 958ed9dedce122..31ec28537fa575 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest env: PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) + COMMIT_ID: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -39,7 +39,7 @@ jobs: id: parse run: | echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" | tee -a "$GITHUB_OUTPUT" - echo "LATEST_COMMIT=${{ env.LATEST_COMMIT }}" | tee -a "$GITHUB_OUTPUT" + echo "COMMIT_ID=${{ env.COMMIT_ID }}" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_BODY='${{ env.COMMENT_BODY }}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" @@ -47,10 +47,10 @@ jobs: echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" - # - name: "Checkout PR ${{ steps.parse.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.parse.outputs.LATEST_COMMIT }}" + # - name: "Checkout PR ${{ steps.parse.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.parse.outputs.COMMIT_ID }}" # uses: actions/checkout@v4 # with: - # ref: ${{ steps.parse.outputs.LATEST_COMMIT }} + # ref: ${{ steps.parse.outputs.COMMIT_ID }} - name: "Checkout master" uses: actions/checkout@v4 @@ -86,36 +86,28 @@ jobs: - name: "Trigger or Skip fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | - echo "GITHUB_TOKEN: ${GITHUB_TOKEN}" source ./regression-test/pipeline/common/teamcity-utils.sh + set -x if [[ ${{ steps.changes.outputs.changes_fe_ut }} == 'true' ]];then add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" else skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" fi - - name: "Skip fe ut" - if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_fe_ut == 'false' }} - run: | - source ./regression-test/pipeline/common/teamcity-utils.sh - skip_teamcity_pipeline "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" - # - name: "Setup tmate session" # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 - - - name: "Trigger be ut" - if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'true' }} + - name: "Trigger or Skip be ut" + if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - cancel_and_add_build - - - name: "Skip be ut" - if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) && steps.changes.outputs.changes_be_ut == 'false' }} - run: | - source ./regression-test/pipeline/common/teamcity-utils.sh - skip_build + set -x + if [[ ${{ steps.changes.outputs.changes_be_ut }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" + fi diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index d552e377b3f9d5..55e408e5d2ab94 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -19,9 +19,10 @@ #!/bin/bash # github中评论的要触发的流水线名字 -# 到 +# 到 # teamcity流水线实际的名称 # 的映射 +# 新加流水线需要修改这里 declare -A comment_to_pipeline comment_to_pipeline=( ['feut']='Doris_Doris_FeUt' @@ -36,6 +37,24 @@ comment_to_pipeline=( ['tpch']='Tpch_TpchSf100' ) +# github中评论的要触发的流水线名字 +# 到 +# teamcity流水线返回结果给github的名称 +# 的映射 +# 新加流水线需要修改这里 +declare -A conment_to_context +conment_to_context=( + ['compile']='COMPILE (DORIS_COMPILE)' + ['feut']='FE UT (Doris FE UT)' + ['beut']='BE UT (Doris BE UT)' + ['p0']='P0 Regression (Doris Regression)' + ['p1']='P1 Regression (Doris Regression)' + ['external']='External Regression (Doris External Regression)' + ['pipelinex_p0']='P0 Regression PipelineX (Doris Regression)' + ['clickbench']='clickbench-new (clickbench)' + ['arm']='P0 Regression (ARM pipeline)' + ['tpch']='tpch-sf100 (tpch)' +) get_commit_id_of_build() { # 获取某个build的commit id @@ -104,7 +123,7 @@ add_build() { PULL_REQUEST_NUM="$1" COMMENT_TRIGGER_TYPE="$2" COMMENT_REPEAT_TIMES="$3" - + if [[ -z "${COMMIT_ID_FROM_TRIGGER}" ]]; then echo "WARNINR: env COMMIT_ID_FROM_TRIGGER not set" fi @@ -144,24 +163,8 @@ function skip_build() { PR_COMMIT_ID="$1" COMMENT_TRIGGER_TYPE="$2" - # github中评论的要触发的流水线名字 - # 到 - # teamcity流水线返回结果给github的名称 - # 的映射 - # 新加流水线需要修改这里 - declare -A context_map - context_map=(['compile']='COMPILE (DORIS_COMPILE)' - ['feut']='FE UT (Doris FE UT)' - ['beut']='BE UT (Doris BE UT)' - ['p0']='P0 Regression (Doris Regression)' - ['p1']='P1 Regression (Doris Regression)' - ['external']='External Regression (Doris External Regression)' - ['pipelinex_p0']='P0 Regression PipelineX (Doris Regression)' - ['clickbench']='clickbench-new (clickbench)' - ['arm']='P0 Regression (ARM pipeline)' - ['tpch']='tpch-sf100 (tpch)') local state="${TC_BUILD_STATE:-success}" # 可选值 success failure pending - payload="{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"Skip teamCity build\",\"context\":\"${context_map[${COMMENT_TRIGGER_TYPE}]}\"}" + payload="{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"Skip teamCity build\",\"context\":\"${conment_to_context[${COMMENT_TRIGGER_TYPE}]}\"}" if curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ From f83e7efec67b5a3045d048ab2731a452d6e428ba Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 21:51:18 +0800 Subject: [PATCH 37/76] 3 --- .../workflows/comment-to-trigger-teamcity.yml | 24 ++- .../pipeline/common/github-utils.sh | 150 ++++++++++++++++++ .../pipeline/common/teamcity-utils.sh | 1 - 3 files changed, 161 insertions(+), 14 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 31ec28537fa575..cbabd2420e4ba9 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -58,30 +58,32 @@ jobs: - name: "Check if pr need run build" id: changes run: | - cd regression-test/pipeline/common/ + source regression-test/pipeline/common/github-utils.sh set -x - if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "fe-ut"; then + _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" + if change_files_fe_ut; then echo "changes_fe_ut=true" | tee -a "$GITHUB_OUTPUT" fi - if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "be-ut"; then + if change_files_be_ut; then echo "changes_be_ut=true" | tee -a "$GITHUB_OUTPUT" fi - if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "ckb"; then + if change_files_ckb; then echo "changes_ckb=true" | tee -a "$GITHUB_OUTPUT" echo "tpch=true" | tee -a "$GITHUB_OUTPUT" fi - if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p0"; then + if change_files_regression_p0; then echo "changes_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changes_external=true" | tee -a "$GITHUB_OUTPUT" echo "changes_pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" echo "changes_arm"=true" | tee -a "$GITHUB_OUTPUT" fi - if bash check-pr-if-need-run-build.sh "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "regression-p1"; then + if change_files_regression_p1; then echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" fi - echo "-- COMMENT_TRIGGER_TYPE: '${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}'" - echo "-- COMMENT_REPEAT_TIMES: '${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}'" - cd - + + - name: "Setup tmate session" + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} @@ -94,10 +96,6 @@ jobs: skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" fi - # - name: "Setup tmate session" - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 - - name: "Trigger or Skip be ut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index c8c5ba213fb35d..f7681127129f93 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -63,3 +63,153 @@ ${COMMENT_BODY} " create_an_issue_comment "${ISSUE_NUMBER}" "${COMMENT_BODY}" } + +_get_pr_changed_files() { + usage_str="Usage: + _get_pr_changed_files [OPTIONS] + note: https://github.com/apache/doris/pull/13259, PULL_NUMBER is 13259 + OPTIONS can be one of [all|added|modified|removed], default is all + " + if [[ -z "$1" ]]; then echo -e "${usage_str}" && return 1; fi + if ! curl --version >/dev/null; then echo 'error: curl required...' && return 1; fi + if ! command -v jq >/dev/null; then sudo yum install jq -y || sudo apt install -y jq; fi + + PULL_NUMBER="$1" + which_file="$2" + pr_url="https://github.com/${OWNER:=apache}/${REPO:=doris}/pull/${PULL_NUMBER}" + try_times=10 + # The number of results per page (max 100), Default 30. + per_page=100 + file_name='pr_change_files' + while [[ ${try_times} -gt 0 ]]; do + # TODO: 修改文件多于100个的情况未处理 + if curl \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=1\&per_page="${per_page}" \ + 2>/dev/null >"${file_name}"; then + break + else + try_times=$((try_times - 1)) + fi + done + if [[ ${try_times} = 0 ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi + + all_files=$(jq -r '.[] | .filename' "${file_name}") + added_files=$(jq -r '.[] | select(.status == "added") | .filename' "${file_name}") + modified_files=$(jq -r '.[] | select(.status == "modified") | .filename' "${file_name}") + removed_files=$(jq -r '.[] | select(.status == "removed") | .filename' "${file_name}") + rm "${file_name}" + if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi + + echo -e " +https://github.com/apache/doris/pull/${PULL_NUMBER}/files all change files: +---------------------------------------------------------------" + if [[ "${which_file:-all}" == "all" ]]; then + echo -e "${all_files}\n" && export all_files + elif [[ "${which_file}" == "added" ]]; then + echo -e "${added_files}\n" && export added_files + elif [[ "${which_file}" == "modified" ]]; then + echo -e "${modified_files}\n" && export modified_files + elif [[ "${which_file}" == "removed" ]]; then + echo -e "${removed_files}\n" && export removed_files + else + return 1 + fi +} + +_only_modified_regression_conf() { + if [[ -n ${added_files} || -n ${removed_files} ]]; then echo "Not only modified regression conf, find added/removed files" && return 1; fi + if [[ -z ${modified_files} ]]; then echo "modified_files is empty, return false" && return 1; fi + for f in ${modified_files}; do + if [[ "${f}" == "regression-test/pipeline/p0/conf/regression-conf.groovy" ]] || + [[ "${f}" == "regression-test/pipeline/p1/conf/regression-conf.groovy" ]]; then + continue + else + echo "Not only modified regression conf" && return 1 + fi + done + echo "only modified regression conf" && return 0 +} + +change_files_fe_ut() { + if _only_modified_regression_conf; then echo "return no need" && return 1; fi + if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi + for af in ${all_files}; do + if [[ "${af}" == 'fe'* ]] || + [[ "${af}" == 'fe_plugins'* ]] || + [[ "${af}" == 'bin/start_fe.sh' ]] || + [[ "${af}" == 'docs/zh-CN/docs/sql-manual/'* ]] || + [[ "${af}" == 'docs/en/docs/sql-manual/'* ]] || + [[ "${af}" == 'bin/stop_fe.sh' ]] || + [[ "${af}" == 'run-fe-ut.sh' ]]; then echo "fe-ut related file changed, return need" && return 0; fi + done + echo "return no need" && return 1 +} + +change_files_be_ut() { + if _only_modified_regression_conf; then echo "return no need" && return 1; fi + if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi + for af in ${all_files}; do + if [[ "${af}" == 'be'* ]] || + [[ "${af}" == 'contrib'* ]] || + [[ "${af}" == 'thirdparty'* ]] || + [[ "${af}" == 'bin/start_be.sh' ]] || + [[ "${af}" == 'bin/stop_be.sh' ]] || + [[ "${af}" == 'run-be-ut.sh' ]]; then + echo "be-ut related file changed, return need" && return 0 + fi + done + echo "return no need" && return 1 +} + +change_files_regression_p0() { + if _only_modified_regression_conf; then echo "return no need" && return 1; fi + if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi + for af in ${all_files}; do + if [[ "${af}" == 'be'* ]] || + [[ "${af}" == 'bin'* ]] || + [[ "${af}" == 'conf'* ]] || + [[ "${af}" == 'contrib'* ]] || + [[ "${af}" == 'fe'* ]] || + [[ "${af}" == 'fe_plugins'* ]] || + [[ "${af}" == 'gensrc'* ]] || + [[ "${af}" == 'regression-test'* ]] || + [[ "${af}" == 'thirdparty'* ]] || + [[ "${af}" == 'docker'* ]] || + [[ "${af}" == 'ui'* ]] || + [[ "${af}" == 'webroot'* ]] || + [[ "${af}" == 'build.sh' ]] || + [[ "${af}" == 'env.sh' ]] || + [[ "${af}" == 'run-regression-test.sh' ]]; then + echo "regression related file changed, return need" && return 0 + fi + done + echo "return no need" && return 1 +} + +change_files_regression_p1() { + change_files_regression_p0 +} + +change_files_arm_regression_p0() { + if [[ $(($1 % 2)) -eq 0 ]]; then echo "the pull request id is even, return no need" && return 1; fi + change_files_regression_p0 +} + +change_files_ckb() { + if _only_modified_regression_conf; then echo "return no need" && return 1; fi + if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi + for af in ${all_files}; do + if [[ "${af}" == 'be'* ]] || + [[ "${af}" == 'bin'* ]] || + [[ "${af}" == 'conf'* ]] || + [[ "${af}" == 'fe'* ]] || + [[ "${af}" == 'gensrc'* ]] || + [[ "${af}" == 'thirdparty'* ]] || + [[ "${af}" == 'build.sh' ]] || + [[ "${af}" == 'env.sh' ]]; then + echo "clickbench performance related file changed, return need" && return 0 + fi + done + echo "return no need" && return 1 +} diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index 55e408e5d2ab94..20711cfa537993 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -178,7 +178,6 @@ function skip_build() { fi } -# GITHUB_TOKEN=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 # skip_teamcity_pipeline '9940812522228f574c5b630666189bc0aa4b1c60' clickbench cancel_and_add_build() { From 4d55e9e1786e772ce0fea4ca440002d1d6b9b400 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 22:44:50 +0800 Subject: [PATCH 38/76] 3 --- .github/workflows/comment-to-trigger-teamcity.yml | 11 +++-------- regression-test/pipeline/common/teamcity-utils.sh | 12 +++++------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index cbabd2420e4ba9..b4af3b41a76b02 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -47,11 +47,6 @@ jobs: echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" - # - name: "Checkout PR ${{ steps.parse.outputs.PULL_REQUEST_NUM }}, commit ${{ steps.parse.outputs.COMMIT_ID }}" - # uses: actions/checkout@v4 - # with: - # ref: ${{ steps.parse.outputs.COMMIT_ID }} - - name: "Checkout master" uses: actions/checkout@v4 @@ -81,9 +76,9 @@ jobs: echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" fi - - name: "Setup tmate session" - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 + # - name: "Setup tmate session" + # if: ${{ failure() }} + # uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip fe ut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index 20711cfa537993..4037da5d54716d 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -129,13 +129,11 @@ add_build() { fi local PIPELINE PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" - if ret=$( - curl -s -X POST \ - -u OneMoreChance:OneMoreChance \ - -H "Content-Type:text/plain" \ - -H "Accept: application/json" \ - "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.commit_id_from_trigger\&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times&value=${COMMENT_REPEAT_TIMES:-1}" - ); then + if curl -s -X POST \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:text/plain" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.commit_id_from_trigger&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times_from_trigger&value=${COMMENT_REPEAT_TIMES:-1}"; then echo "INFO: Add new build to PIPELINE ${PIPELINE} of PR ${PULL_REQUEST_NUM} with COMMENT_REPEAT_TIMES ${COMMENT_REPEAT_TIMES:-1}" else return 1 From fe19aa4b5c0c6a285fdf130637930310bef7db21 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 22:48:59 +0800 Subject: [PATCH 39/76] 3 --- .github/workflows/auto_trigger_teamcity.yml | 254 +++++++++--------- .../workflows/comment-to-trigger-teamcity.yml | 4 +- 2 files changed, 129 insertions(+), 129 deletions(-) diff --git a/.github/workflows/auto_trigger_teamcity.yml b/.github/workflows/auto_trigger_teamcity.yml index ab3e85fa6b560f..05f68ef1e0771a 100644 --- a/.github/workflows/auto_trigger_teamcity.yml +++ b/.github/workflows/auto_trigger_teamcity.yml @@ -1,143 +1,143 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. +# # Licensed to the Apache Software Foundation (ASF) under one +# # or more contributor license agreements. See the NOTICE file +# # distributed with this work for additional information +# # regarding copyright ownership. The ASF licenses this file +# # to you under the Apache License, Version 2.0 (the +# # "License"); you may not use this file except in compliance +# # with the License. You may obtain a copy of the License at +# # +# # http://www.apache.org/licenses/LICENSE-2.0 +# # +# # Unless required by applicable law or agreed to in writing, +# # software distributed under the License is distributed on an +# # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# # KIND, either express or implied. See the License for the +# # specific language governing permissions and limitations +# # under the License. -name: Auto trigger teamcity +# name: Auto trigger teamcity -on: - issue_comment: - types: [created, edited] - pull_request_review_comment: - types: [created, edited] +# on: +# issue_comment: +# types: [created, edited] +# pull_request_review_comment: +# types: [created, edited] -env: - PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) - TEAMCITY_URL: '-H \"Content-Type:text/plain\" -H \"Accept: application/json\" -u OneMoreChance:OneMoreChance http://43.132.222.7:8111' +# env: +# PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') +# LATEST_COMMIT: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) +# TEAMCITY_URL: '-H \"Content-Type:text/plain\" -H \"Accept: application/json\" -u OneMoreChance:OneMoreChance http://43.132.222.7:8111' -jobs: - run_teamcity_pipeline: - if: (contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'just_for_test')) && contains(github.event.comment.body, 'run') && !contains(github.event.comment.body, 'Thanks for your contribution') +# jobs: +# run_teamcity_pipeline: +# if: (contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'just_for_test')) && contains(github.event.comment.body, 'run') && !contains(github.event.comment.body, 'Thanks for your contribution') - runs-on: ubuntu-latest +# runs-on: ubuntu-latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ISSUE_COMMENT: ${{ github.event.comment.body }} - steps: - - name: Run pipeline by teamcity restful - run: | - if [ "_xx""${{ github.event.issue.pull_request.url }}" != "_xx" ]; then - echo "Comment was made on pull request: $(echo ${{ github.event.issue.pull_request.url }} | awk -F/ '{print $NF}')" - else - echo "Comment was made on an issue, not a pull request." - fi - pull_request_num=${{ env.PULL_REQUEST_NUM }} - encoded_string=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$ISSUE_COMMENT") - comment_message="$ISSUE_COMMENT" - latest_commit_id="${{ env.LATEST_COMMIT }}" - teamcity_url="${{ env.TEAMCITY_URL }}" - trigger_pipelines="" - same_build_sign="false" - echo "pull_request_num : ${pull_request_num}" - echo "encoded_string : ${encoded_string}" - echo "latest_commit_id : ${{ env.LATEST_COMMIT }}" +# env: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# ISSUE_COMMENT: ${{ github.event.comment.body }} +# steps: +# - name: Run pipeline by teamcity restful +# run: | +# if [ "_xx""${{ github.event.issue.pull_request.url }}" != "_xx" ]; then +# echo "Comment was made on pull request: $(echo ${{ github.event.issue.pull_request.url }} | awk -F/ '{print $NF}')" +# else +# echo "Comment was made on an issue, not a pull request." +# fi +# pull_request_num=${{ env.PULL_REQUEST_NUM }} +# encoded_string=$(perl -MURI::Escape -e 'print uri_escape($ARGV[0]);' "$ISSUE_COMMENT") +# comment_message="$ISSUE_COMMENT" +# latest_commit_id="${{ env.LATEST_COMMIT }}" +# teamcity_url="${{ env.TEAMCITY_URL }}" +# trigger_pipelines="" +# same_build_sign="false" +# echo "pull_request_num : ${pull_request_num}" +# echo "encoded_string : ${encoded_string}" +# echo "latest_commit_id : ${{ env.LATEST_COMMIT }}" - set -x - reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" - comment_trigger_type="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" - comment_repeat_times="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" - if [[ "${comment_trigger_type}" == "buildall" ]]; then - trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression" - elif [[ "${comment_trigger_type}" == "compile" ]]; then - trigger_pipelines="Doris_DorisCompile_Compile" - elif [[ "${comment_trigger_type}" == "p0" ]]; then - trigger_pipelines="Doris_DorisRegression_P0Regression" - elif [[ "${comment_trigger_type}" == "p1" ]]; then - trigger_pipelines="Doris_DorisRegression_P1Regression" - elif [[ "${comment_trigger_type}" == "feut" ]]; then - trigger_pipelines="Doris_Doris_FeUt" - elif [[ "${comment_trigger_type}" == "beut" ]]; then - trigger_pipelines="Doris_DorisBeUt_BeUt" - elif [[ "${comment_trigger_type}" == "external" ]]; then - trigger_pipelines="Doris_External_Regression" - elif [[ "${comment_trigger_type}" == "clickbench" ]]; then - trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew" - elif [[ "${comment_trigger_type}" == "pipelinex_p0" ]]; then - trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX" - elif [[ "${comment_trigger_type}" == "arm" ]]; then - trigger_pipelines="Doris_ArmPipeline_P0Regression" - elif [[ "${comment_trigger_type}" == "tpch" ]]; then - trigger_pipelines="Tpch_TpchSf100" - fi - if [ -z "${trigger_pipelines}" ];then - echo "Just a general comment that doesn't match any pipeline rules" - fi - set +x +# set -x +# reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" +# comment_trigger_type="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" +# comment_repeat_times="$(echo "${comment_message}" | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" +# if [[ "${comment_trigger_type}" == "buildall" ]]; then +# trigger_pipelines="Doris_Doris_FeUt Doris_DorisBeUt_BeUt Doris_DorisCompile_Compile Doris_Performance_Clickbench_ClickbenchNew Doris_ArmPipeline_P0Regression" +# elif [[ "${comment_trigger_type}" == "compile" ]]; then +# trigger_pipelines="Doris_DorisCompile_Compile" +# elif [[ "${comment_trigger_type}" == "p0" ]]; then +# trigger_pipelines="Doris_DorisRegression_P0Regression" +# elif [[ "${comment_trigger_type}" == "p1" ]]; then +# trigger_pipelines="Doris_DorisRegression_P1Regression" +# elif [[ "${comment_trigger_type}" == "feut" ]]; then +# trigger_pipelines="Doris_Doris_FeUt" +# elif [[ "${comment_trigger_type}" == "beut" ]]; then +# trigger_pipelines="Doris_DorisBeUt_BeUt" +# elif [[ "${comment_trigger_type}" == "external" ]]; then +# trigger_pipelines="Doris_External_Regression" +# elif [[ "${comment_trigger_type}" == "clickbench" ]]; then +# trigger_pipelines="Doris_Performance_Clickbench_ClickbenchNew" +# elif [[ "${comment_trigger_type}" == "pipelinex_p0" ]]; then +# trigger_pipelines="Doris_DorisRegression_P0RegressionPipelineX" +# elif [[ "${comment_trigger_type}" == "arm" ]]; then +# trigger_pipelines="Doris_ArmPipeline_P0Regression" +# elif [[ "${comment_trigger_type}" == "tpch" ]]; then +# trigger_pipelines="Tpch_TpchSf100" +# fi +# if [ -z "${trigger_pipelines}" ];then +# echo "Just a general comment that doesn't match any pipeline rules" +# fi +# set +x - echo "need run pipelines: ${trigger_pipelines}" +# echo "need run pipelines: ${trigger_pipelines}" - for pipeline in ${trigger_pipelines} - do - same_build_sign="false" - echo "-----------------------------------------------------------------" - running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true" - echo "${running_builds_command}" - running_builds_list=$(eval "${running_builds_command}" > running_builds.json && jq -r '.build[].id' running_builds.json) - echo "running_builds_list : ${running_builds_list}" +# for pipeline in ${trigger_pipelines} +# do +# same_build_sign="false" +# echo "-----------------------------------------------------------------" +# running_builds_command="curl -s -X GET ${teamcity_url}/app/rest/builds\?locator\=buildType\:${pipeline}\,branch:pull/${pull_request_num}\,running:true" +# echo "${running_builds_command}" +# running_builds_list=$(eval "${running_builds_command}" > running_builds.json && jq -r '.build[].id' running_builds.json) +# echo "running_builds_list : ${running_builds_list}" - queue_builds_command="curl -s -X GET ${teamcity_url}/app/rest/buildQueue\?locator\=buildType\:${pipeline}" - echo "${queue_builds_command}" - eval "${queue_builds_command}" > queue_builds.json - queue_builds_list=$(cat queue_builds.json | jq ".build[] | select(.branchName == \"pull/${pull_request_num}\") | .id" ) - echo "queue builds list: ${queue_builds_list}" +# queue_builds_command="curl -s -X GET ${teamcity_url}/app/rest/buildQueue\?locator\=buildType\:${pipeline}" +# echo "${queue_builds_command}" +# eval "${queue_builds_command}" > queue_builds.json +# queue_builds_list=$(cat queue_builds.json | jq ".build[] | select(.branchName == \"pull/${pull_request_num}\") | .id" ) +# echo "queue builds list: ${queue_builds_list}" - merged_list=("${running_builds_list[@]} ${queue_builds_list[@]}") - echo "merged_list : ${merged_list}" +# merged_list=("${running_builds_list[@]} ${queue_builds_list[@]}") +# echo "merged_list : ${merged_list}" - for build in ${merged_list}; do - running_commit_command="curl -s -X GET ${teamcity_url}/app/rest/builds/${build}" - echo "${running_commit_command}" - eval "${running_commit_command}" > build_commit_info.json - running_commit_id=$(jq -r '.revisions.revision[0].version' build_commit_info.json) - running_env_commit_id=$(cat build_commit_info.json | jq ".properties.property[] | select(.name == \"env.latest_commit_id\") | .value") +# for build in ${merged_list}; do +# running_commit_command="curl -s -X GET ${teamcity_url}/app/rest/builds/${build}" +# echo "${running_commit_command}" +# eval "${running_commit_command}" > build_commit_info.json +# running_commit_id=$(jq -r '.revisions.revision[0].version' build_commit_info.json) +# running_env_commit_id=$(cat build_commit_info.json | jq ".properties.property[] | select(.name == \"env.latest_commit_id\") | .value") - if [[ "_"${latest_commit_id} == "_"${running_commit_id} || _"\"${latest_commit_id}\"" == "_"${running_env_commit_id} ]];then - echo "the same pr_commit build already exist, this build ${build} is running or in queue!" - same_build_sign="true" - break - fi - done +# if [[ "_"${latest_commit_id} == "_"${running_commit_id} || _"\"${latest_commit_id}\"" == "_"${running_env_commit_id} ]];then +# echo "the same pr_commit build already exist, this build ${build} is running or in queue!" +# same_build_sign="true" +# break +# fi +# done - if [ "_""${same_build_sign}" == "_false" ];then - sleep 10s - echo "there is no running build or queue build, so trigger a new !" - echo "comment_repeat_times: ${comment_repeat_times}" - if [[ -n "${comment_repeat_times}" ]]; then - execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${comment_repeat_times}" - else - execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}" - fi - echo "${execute_command}" - eval "${execute_command}" - echo "-----------------------------------------------------------------" - else - echo "there is same pr commit task in queue,so skip trigger !" - echo "-----------------------------------------------------------------" - fi - done +# if [ "_""${same_build_sign}" == "_false" ];then +# sleep 10s +# echo "there is no running build or queue build, so trigger a new !" +# echo "comment_repeat_times: ${comment_repeat_times}" +# if [[ -n "${comment_repeat_times}" ]]; then +# execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}\&name=env.repeat_times\&value=${comment_repeat_times}" +# else +# execute_command="curl -s -X POST ${teamcity_url}/httpAuth/action.html\?add2Queue\=${pipeline}\&branchName\=pull/${pull_request_num}\&name=env.latest_pr_comment\&value=${encoded_string}\&name=env.latest_commit_id\&value=${latest_commit_id}" +# fi +# echo "${execute_command}" +# eval "${execute_command}" +# echo "-----------------------------------------------------------------" +# else +# echo "there is same pr commit task in queue,so skip trigger !" +# echo "-----------------------------------------------------------------" +# fi +# done diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index b4af3b41a76b02..536113483d48e2 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest env: PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - COMMIT_ID: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) + COMMIT_ID_FROM_TRIGGER: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -39,7 +39,7 @@ jobs: id: parse run: | echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" | tee -a "$GITHUB_OUTPUT" - echo "COMMIT_ID=${{ env.COMMIT_ID }}" | tee -a "$GITHUB_OUTPUT" + echo "COMMIT_ID_FROM_TRIGGER=${{ env.COMMIT_ID_FROM_TRIGGER }}" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_BODY='${{ env.COMMENT_BODY }}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" From 51dc7eec968d1b077ffd20ad6b9a6de997be3e13 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 23:11:48 +0800 Subject: [PATCH 40/76] 3 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 536113483d48e2..41c58f14926597 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest env: PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - COMMIT_ID_FROM_TRIGGER: $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) + COMMIT_ID_FROM_TRIGGER: ${{ $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) }} COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From aa9d1e5cee4a5bccfd291e3fe797d01c46f8c3c8 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 23:16:29 +0800 Subject: [PATCH 41/76] 3 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 41c58f14926597..ca3db5c84c4c19 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest env: PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - COMMIT_ID_FROM_TRIGGER: ${{ $(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json) }} + COMMIT_ID_FROM_TRIGGER: ${{ curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json }} COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From adb98ec5b0c1e5f0cc84aa7aa3db76b6534ec4ea Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 23:28:24 +0800 Subject: [PATCH 42/76] 3 --- .github/workflows/comment-to-trigger-teamcity.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index ca3db5c84c4c19..c0675a8f05c876 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -29,8 +29,6 @@ jobs: runs-on: ubuntu-latest env: - PULL_REQUEST_NUM: $(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}') - COMMIT_ID_FROM_TRIGGER: ${{ curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json }} COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -38,9 +36,11 @@ jobs: - name: "Parse PR comment" id: parse run: | - echo "PULL_REQUEST_NUM=${{ env.PULL_REQUEST_NUM }}" | tee -a "$GITHUB_OUTPUT" - echo "COMMIT_ID_FROM_TRIGGER=${{ env.COMMIT_ID_FROM_TRIGGER }}" | tee -a "$GITHUB_OUTPUT" - echo "COMMENT_BODY='${{ env.COMMENT_BODY }}'" | tee -a "$GITHUB_OUTPUT" + PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')" + COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json)" + echo "PULL_REQUEST_NUM=${PULL_REQUEST_NUM}" | tee -a "$GITHUB_OUTPUT" + echo "COMMIT_ID_FROM_TRIGGER=${COMMIT_ID_FROM_TRIGGER}" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" From be1ff1919cb51524d972aa42e130c74cccfa0bde Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 16 Nov 2023 23:31:14 +0800 Subject: [PATCH 43/76] 3 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index c0675a8f05c876..1fe07fa04f8e10 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -84,6 +84,7 @@ jobs: if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x if [[ ${{ steps.changes.outputs.changes_fe_ut }} == 'true' ]];then add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" @@ -95,6 +96,7 @@ jobs: if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x if [[ ${{ steps.changes.outputs.changes_be_ut }} == 'true' ]];then add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" From 8047e2525d04aa9f916df73ddcbf498ff4ec417a Mon Sep 17 00:00:00 2001 From: stephen Date: Fri, 17 Nov 2023 18:33:44 +0800 Subject: [PATCH 44/76] 4 --- .../workflows/comment-to-trigger-teamcity.yml | 10 +-- .../pipeline/common/github-utils.sh | 69 +++++++++++++------ 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 1fe07fa04f8e10..7b67a18cf6eb47 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -56,23 +56,23 @@ jobs: source regression-test/pipeline/common/github-utils.sh set -x _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" - if change_files_fe_ut; then + if file_changed_fe_ut; then echo "changes_fe_ut=true" | tee -a "$GITHUB_OUTPUT" fi - if change_files_be_ut; then + if file_changed_be_ut; then echo "changes_be_ut=true" | tee -a "$GITHUB_OUTPUT" fi - if change_files_ckb; then + if file_changed_ckb; then echo "changes_ckb=true" | tee -a "$GITHUB_OUTPUT" echo "tpch=true" | tee -a "$GITHUB_OUTPUT" fi - if change_files_regression_p0; then + if file_changed_regression_p0; then echo "changes_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changes_external=true" | tee -a "$GITHUB_OUTPUT" echo "changes_pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" echo "changes_arm"=true" | tee -a "$GITHUB_OUTPUT" fi - if change_files_regression_p1; then + if file_changed_regression_p1; then echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" fi diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index f7681127129f93..36f5dd2f7a4717 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -64,6 +64,25 @@ ${COMMENT_BODY} create_an_issue_comment "${ISSUE_NUMBER}" "${COMMENT_BODY}" } +_get_pr_changed_files_count() { + PULL_NUMBER="${PULL_NUMBER:-$1}" + if [[ -z "${PULL_NUMBER}" ]]; then + echo "Usage: _get_pr_changed_files_count PULL_NUMBER" + return 1 + fi + + OWNER="${OWNER:=apache}" + REPO="${REPO:=doris}" + if ret=$( + curl -s -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}" | jq -e '.changed_files' + ); then + echo "${ret}" + else + return 1 + fi +} + _get_pr_changed_files() { usage_str="Usage: _get_pr_changed_files [OPTIONS] @@ -77,20 +96,30 @@ _get_pr_changed_files() { PULL_NUMBER="$1" which_file="$2" pr_url="https://github.com/${OWNER:=apache}/${REPO:=doris}/pull/${PULL_NUMBER}" - try_times=10 # The number of results per page (max 100), Default 30. per_page=100 - file_name='pr_change_files' - while [[ ${try_times} -gt 0 ]]; do - # TODO: 修改文件多于100个的情况未处理 - if curl \ - -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page=1\&per_page="${per_page}" \ - 2>/dev/null >"${file_name}"; then - break - else - try_times=$((try_times - 1)) - fi + file_name='pr_changed_files' + rm -f "${file_name}" + page=1 + changed_files_count="$(_get_pr_changed_files_count "${PULL_NUMBER}")" + while [[ ${changed_files_count} -gt 0 ]]; do + try_times=10 + while [[ ${try_times} -gt 0 ]]; do + # TODO: 修改文件多于100个的情况未处理 + set -x + if curl -s \ + -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}"/files?page="${page}"\&per_page="${per_page}" \ + >>"${file_name}"; then + set +x + break + else + set +x + try_times=$((try_times - 1)) + fi + done + page=$((page + 1)) + changed_files_count=$((changed_files_count - per_page)) done if [[ ${try_times} = 0 ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi @@ -131,7 +160,7 @@ _only_modified_regression_conf() { echo "only modified regression conf" && return 0 } -change_files_fe_ut() { +file_changed_fe_ut() { if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do @@ -146,7 +175,7 @@ change_files_fe_ut() { echo "return no need" && return 1 } -change_files_be_ut() { +file_changed_be_ut() { if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do @@ -162,7 +191,7 @@ change_files_be_ut() { echo "return no need" && return 1 } -change_files_regression_p0() { +file_changed_regression_p0() { if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do @@ -187,16 +216,16 @@ change_files_regression_p0() { echo "return no need" && return 1 } -change_files_regression_p1() { - change_files_regression_p0 +file_changed_regression_p1() { + file_changed_regression_p0 } -change_files_arm_regression_p0() { +file_changed_arm_regression_p0() { if [[ $(($1 % 2)) -eq 0 ]]; then echo "the pull request id is even, return no need" && return 1; fi - change_files_regression_p0 + file_changed_regression_p0 } -change_files_ckb() { +file_changed_ckb() { if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do From bb443b6c349bbb5368dc6da933e616668a05c02d Mon Sep 17 00:00:00 2001 From: stephen Date: Fri, 17 Nov 2023 21:45:38 +0800 Subject: [PATCH 45/76] 4 --- .../workflows/comment-to-trigger-teamcity.yml | 123 +++++++++++++++++- .../pipeline/common/github-utils.sh | 16 ++- 2 files changed, 129 insertions(+), 10 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 7b67a18cf6eb47..c96f6b943ca635 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -80,31 +80,142 @@ jobs: # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 - - name: "Trigger or Skip fe ut" + - name: "Trigger or Skip feut" if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x if [[ ${{ steps.changes.outputs.changes_fe_ut }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "feut" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "feut" fi - - name: "Trigger or Skip be ut" + - name: "Trigger or Skip beut" if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x if [[ ${{ steps.changes.outputs.changes_be_ut }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "beut" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }}" + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "beut" fi + - name: "Trigger or Skip compile" + if: ${{ contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "compile" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "compile" + fi + + - name: "Trigger or Skip p0" + if: ${{ contains(fromJSON('["p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p0" + exit + fi + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p0" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p0" + fi + + - name: "Trigger or Skip p1" + if: ${{ contains(fromJSON('["p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p1" + exit + fi + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p1" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p1" + fi + + - name: "Trigger or Skip external" + if: ${{ contains(fromJSON('["external", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger external" + exit + fi + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "external" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "external" + fi + - name: "Trigger or Skip pipelinex_p0" + if: ${{ contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger pipelinex_p0" + exit + fi + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "pipelinex_p0" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "pipelinex_p0" + fi + - name: "Trigger or Skip arm" + if: ${{ contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "arm" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "arm" + fi + - name: "Trigger or Skip clickbench" + if: ${{ contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "clickbench" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "clickbench" + fi + - name: "Trigger or Skip tpch" + if: ${{ contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} + set -x + if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then + echo "COMMENT_TRIGGER_TYPE is buildall, trigger clickbench is enough, clickbench will trigger tpch" + exit + fi + if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then + add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "tpch" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "tpch" + fi diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 36f5dd2f7a4717..1842f86281c096 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -147,17 +147,25 @@ https://github.com/apache/doris/pull/${PULL_NUMBER}/files all change files: } _only_modified_regression_conf() { - if [[ -n ${added_files} || -n ${removed_files} ]]; then echo "Not only modified regression conf, find added/removed files" && return 1; fi - if [[ -z ${modified_files} ]]; then echo "modified_files is empty, return false" && return 1; fi + if [[ -n ${added_files} || -n ${removed_files} ]]; then + # echo "Not only modified regression conf, find added/removed files" + return 1 + fi + if [[ -z ${modified_files} ]]; then + # echo "modified_files is empty, return false" + return 1 + fi for f in ${modified_files}; do if [[ "${f}" == "regression-test/pipeline/p0/conf/regression-conf.groovy" ]] || [[ "${f}" == "regression-test/pipeline/p1/conf/regression-conf.groovy" ]]; then continue else - echo "Not only modified regression conf" && return 1 + # echo "Not only modified regression conf" + return 1 fi done - echo "only modified regression conf" && return 0 + # echo "only modified regression conf" + return 0 } file_changed_fe_ut() { From a09d087b6f688eecb8db80d13e206d2fcc7eac08 Mon Sep 17 00:00:00 2001 From: stephen Date: Fri, 17 Nov 2023 21:52:14 +0800 Subject: [PATCH 46/76] 4 --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index c96f6b943ca635..96cf4267c33ad5 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -69,8 +69,8 @@ jobs: if file_changed_regression_p0; then echo "changes_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changes_external=true" | tee -a "$GITHUB_OUTPUT" - echo "changes_pipelinex_p0"=true" | tee -a "$GITHUB_OUTPUT" - echo "changes_arm"=true" | tee -a "$GITHUB_OUTPUT" + echo "changes_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changes_arm=true" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p1; then echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" From 910c294a62b6584b8d2777d8e8060a47fa4651d1 Mon Sep 17 00:00:00 2001 From: stephen Date: Fri, 17 Nov 2023 22:16:36 +0800 Subject: [PATCH 47/76] 4 --- .github/workflows/comment-to-trigger-teamcity.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 96cf4267c33ad5..2863e6190503a9 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -25,7 +25,8 @@ jobs: check-comment-if-need-to-trigger-teamcity: # This job only runs for pull request comments - if: ${{ github.event.issue.pull_request && contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} + # if: ${{ github.event.issue.pull_request && contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && ( contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'tpch') ) }} runs-on: ubuntu-latest env: From 8681dc5cddc32bca2f97a50d69b57a28655c817e Mon Sep 17 00:00:00 2001 From: stephen Date: Sun, 19 Nov 2023 23:48:19 +0800 Subject: [PATCH 48/76] 9 --- .../workflows/comment-to-trigger-teamcity.yml | 217 ++++++++++-------- .../pipeline/common/teamcity-utils.sh | 49 ++-- 2 files changed, 154 insertions(+), 112 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 2863e6190503a9..db4588ab7da523 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -24,24 +24,48 @@ on: jobs: check-comment-if-need-to-trigger-teamcity: - # This job only runs for pull request comments - # if: ${{ github.event.issue.pull_request && contains(fromJSON('["run buildall", "run compile", "run beut", "run feut", "run p0", "run p1", "run external", "run pipelinex_p0", "run clickbench", "run arm", "run tpch"]'), github.event.comment.body) }} - if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && ( contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'tpch') ) }} + # This job only runs for pull request comments, and comment body contains 'run' + # if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && ( contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'tpch') ) }} + if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') }} runs-on: ubuntu-latest env: COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TT: ${{ curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref' }} steps: - name: "Parse PR comment" id: parse run: | + if [[ "${COMMENT_BODY}" == *'run buildall'* || + "${COMMENT_BODY}" == *'run compile'* || + "${COMMENT_BODY}" == *'run beut'* || + "${COMMENT_BODY}" == *'run feut'* || + "${COMMENT_BODY}" == *'run p0'* || + "${COMMENT_BODY}" == *'run p1'* || + "${COMMENT_BODY}" == *'run external'* || + "${COMMENT_BODY}" == *'run pipelinex_p0'* || + "${COMMENT_BODY}" == *'run clickbench'* || + "${COMMENT_BODY}" == *'run arm'* || + "${COMMENT_BODY}" == *'run tpch'* ]]; then + echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT" + else + echo "find no keyword in comment body, skip this action." + exit + fi + + echo "GITHUB_EVENT_PATH: $GITHUB_EVENT_PATH" + echo "TT: $TT" + PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')" - COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/$(echo '${{ github.event.issue.pull_request.url }}' | awk -F/ '{print $NF}')" > pr_details.json && jq -r '.head.sha' pr_details.json)" + COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.head.sha')" + TARGET_BRANCH="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref')" echo "PULL_REQUEST_NUM=${PULL_REQUEST_NUM}" | tee -a "$GITHUB_OUTPUT" echo "COMMIT_ID_FROM_TRIGGER=${COMMIT_ID_FROM_TRIGGER}" | tee -a "$GITHUB_OUTPUT" + echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT" + reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" @@ -49,32 +73,34 @@ jobs: echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" - name: "Checkout master" + if: ${{ steps.parse.outputs.comment_trigger }} uses: actions/checkout@v4 - name: "Check if pr need run build" + if: ${{ steps.parse.outputs.comment_trigger }} id: changes run: | source regression-test/pipeline/common/github-utils.sh set -x _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" if file_changed_fe_ut; then - echo "changes_fe_ut=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT" fi if file_changed_be_ut; then - echo "changes_be_ut=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT" fi if file_changed_ckb; then - echo "changes_ckb=true" | tee -a "$GITHUB_OUTPUT" - echo "tpch=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p0; then - echo "changes_p0=true" | tee -a "$GITHUB_OUTPUT" - echo "changes_external=true" | tee -a "$GITHUB_OUTPUT" - echo "changes_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" - echo "changes_arm=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_external=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p1; then - echo "changes_p1=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" fi # - name: "Setup tmate session" @@ -82,141 +108,138 @@ jobs: # uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip feut" - if: ${{ contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x - if [[ ${{ steps.changes.outputs.changes_fe_ut }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "feut" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "feut" - fi + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_fe_ut }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "feut" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + - name: "Trigger or Skip beut" - if: ${{ contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x - if [[ ${{ steps.changes.outputs.changes_be_ut }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "beut" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "beut" - fi + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_be_ut }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "beut" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip compile" - if: ${{ contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "compile" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "compile" - fi + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_p0 }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "compile" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip p0" - if: ${{ contains(fromJSON('["p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} - set -x if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p0" - exit - fi - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p0" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p0" + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p0" && exit fi + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_p0 }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "p0" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip p1" - if: ${{ contains(fromJSON('["p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} - set -x if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p1" - exit - fi - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p1" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "p1" + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger p1" && exit fi + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_p1 }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "p1" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip external" - if: ${{ contains(fromJSON('["external", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["external", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} - set -x if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger external" - exit - fi - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "external" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "external" + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger external" && exit fi + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_external }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "external" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip pipelinex_p0" - if: ${{ contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} - set -x if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger pipelinex_p0" - exit - fi - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "pipelinex_p0" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "pipelinex_p0" + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger pipelinex_p0" && exit fi + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_pipelinex_p0 }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "pipelinex_p0" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip arm" - if: ${{ contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "arm" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "arm" - fi + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_arm }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "arm" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip clickbench" - if: ${{ contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} set -x - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "clickbench" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "clickbench" - fi + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_ckb }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "clickbench" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip tpch" - if: ${{ contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} - set -x if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger clickbench is enough, clickbench will trigger tpch" - exit - fi - if [[ ${{ steps.changes.outputs.changes_p0 }} == 'true' ]];then - add_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "tpch" "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - else - skip_build "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" "tpch" + echo "COMMENT_TRIGGER_TYPE is buildall, trigger clickbench is enough, clickbench will trigger tpch" && exit fi + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_tpch }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "tpch" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index 4037da5d54716d..1327ee251d2aba 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -114,19 +114,16 @@ get_queue_build_of_pr() { # get_queue_build_of_pr "$1" "$2" -add_build() { +trigger_build() { # 新触发一个build - if [[ -z "$2" ]]; then - echo "Usage: add_build PIPELINE PULL_REQUEST_NUM [COMMENT_REPEAT_TIMES]" + PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$2}" + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$3}" + COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$4}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMIT_ID_FROM_TRIGGER}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: add_build PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" return 1 fi - PULL_REQUEST_NUM="$1" - COMMENT_TRIGGER_TYPE="$2" - COMMENT_REPEAT_TIMES="$3" - - if [[ -z "${COMMIT_ID_FROM_TRIGGER}" ]]; then - echo "WARNINR: env COMMIT_ID_FROM_TRIGGER not set" - fi local PIPELINE PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" if curl -s -X POST \ @@ -148,17 +145,17 @@ cancel_queue_build() { echo "TODO: cancel_queue_build" } -function skip_build() { +skip_build() { # 对于不需要跑teamcity pipeline的PR，直接调用github的接口返回成功 if [[ -z "${GITHUB_TOKEN}" ]]; then echo "ERROR: env GITHUB_TOKEN not set" return 1 fi if [[ -z "$2" ]]; then - echo "Usage: skip_teamcity_pipeline PR_COMMIT_ID COMMENT_TRIGGER_TYPE" + echo "Usage: skip_teamcity_pipeline COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE" return 1 fi - PR_COMMIT_ID="$1" + COMMIT_ID_FROM_TRIGGER="$1" COMMENT_TRIGGER_TYPE="$2" local state="${TC_BUILD_STATE:-success}" # 可选值 success failure pending @@ -168,14 +165,36 @@ function skip_build() { -H "Accept: application/vnd.github+json" \ -H "Authorization: Bearer ${GITHUB_TOKEN:-}" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/apache/doris/statuses/${PR_COMMIT_ID:-}" \ + "https://api.github.com/repos/apache/doris/statuses/${COMMIT_ID_FROM_TRIGGER:-}" \ -d "${payload}"; then - echo "INFO: Skipped ${PR_COMMIT_ID} ${COMMENT_TRIGGER_TYPE}" + echo "INFO: Skipped ${COMMIT_ID_FROM_TRIGGER} ${COMMENT_TRIGGER_TYPE}" else return 1 fi } +trigger_or_skip_build() { + # 根据相关文件是否修改，来触发or跳过跑流水线 + FILE_CHANGED="$1" + PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$2}" + COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$3}" + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$4}" + COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$5}" + if [[ -z "${FILE_CHANGED}" || + -z "${PULL_REQUEST_NUM}" || + -z "${COMMIT_ID_FROM_TRIGGER}" || + -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: add_build FILE_CHANGED PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" + return 1 + fi + + if [[ "${FILE_CHANGED}" == "true" ]]; then + trigger_build "${PULL_REQUEST_NUM}" "${COMMIT_ID_FROM_TRIGGER}" "${COMMENT_TRIGGER_TYPE}" "${COMMENT_REPEAT_TIMES}" + else + skip_build "${COMMIT_ID_FROM_TRIGGER}" "${COMMENT_TRIGGER_TYPE}" + fi +} + # skip_teamcity_pipeline '9940812522228f574c5b630666189bc0aa4b1c60' clickbench cancel_and_add_build() { From 1aa66a8ca6483e389108e6859255d11f4e3d9077 Mon Sep 17 00:00:00 2001 From: stephen Date: Sun, 19 Nov 2023 23:49:11 +0800 Subject: [PATCH 49/76] 9 --- .github/workflows/comment-to-trigger-teamcity.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index db4588ab7da523..218f8d7ad2a4d4 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -32,7 +32,6 @@ jobs: env: COMMENT_BODY: ${{ github.event.comment.body }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TT: ${{ curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref' }} steps: - name: "Parse PR comment" @@ -56,7 +55,6 @@ jobs: fi echo "GITHUB_EVENT_PATH: $GITHUB_EVENT_PATH" - echo "TT: $TT" PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')" COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.head.sha')" From d6d035df3630a74fd3578c86dc0e3d1e85d8c12e Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 20 Nov 2023 19:01:35 +0800 Subject: [PATCH 50/76] 0 --- .../workflows/comment-to-trigger-teamcity.yml | 3 - .../pipeline/common/teamcity-utils.sh | 119 +++++++++++++----- 2 files changed, 85 insertions(+), 37 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 218f8d7ad2a4d4..f44fb55fb3f057 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -25,7 +25,6 @@ jobs: check-comment-if-need-to-trigger-teamcity: # This job only runs for pull request comments, and comment body contains 'run' - # if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') && ( contains(github.event.comment.body, 'buildall') || contains(github.event.comment.body, 'compile') || contains(github.event.comment.body, 'beut') || contains(github.event.comment.body, 'feut') || contains(github.event.comment.body, 'p0') || contains(github.event.comment.body, 'p1') || contains(github.event.comment.body, 'external') || contains(github.event.comment.body, 'pipelinex_p0') || contains(github.event.comment.body, 'clickbench') || contains(github.event.comment.body, 'arm') || contains(github.event.comment.body, 'tpch') ) }} if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'run') }} runs-on: ubuntu-latest @@ -54,8 +53,6 @@ jobs: exit fi - echo "GITHUB_EVENT_PATH: $GITHUB_EVENT_PATH" - PULL_REQUEST_NUM="$(echo "${{ github.event.issue.pull_request.url }}" | awk -F/ '{print $NF}')" COMMIT_ID_FROM_TRIGGER="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.head.sha')" TARGET_BRANCH="$(curl -s -H "Authorization:Bearer ${{ secrets.GITHUB_TOKEN }}" "https://api.github.com/repos/${{ github.repository }}/pulls/${PULL_REQUEST_NUM}" | jq -r '.base.ref')" diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index 1327ee251d2aba..fe27b407fcc650 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -76,10 +76,13 @@ get_commit_id_of_build() { get_running_build_of_pr() { # "获取pr在某条流水线上正在跑的build" - if [[ -z "${PIPELINE}" || -z "${PULL_REQUEST_NUM}" ]]; then - echo "ERROR: env PIPELINE or PULL_REQUEST_NUM not set." - return 1 + PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 fi + + PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local running_builds_list if ret=$( curl -s -X GET \ @@ -94,9 +97,17 @@ get_running_build_of_pr() { return 1 fi } +# get_running_build_of_pr "$1" "$2" get_queue_build_of_pr() { # "获取pr在某条流水线上排队的build" + PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 + fi + + PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local queue_builds_list if ret=$( curl -s -X GET \ @@ -108,42 +119,60 @@ get_queue_build_of_pr() { queue_builds_list=$(echo "${ret}" | jq ".build[] | select(.branchName == \"pull/${PULL_REQUEST_NUM}\") | .id") echo "${queue_builds_list}" else - return 1 + echo "WARNING: failed to get queue build for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" && return 1 fi } - # get_queue_build_of_pr "$1" "$2" -trigger_build() { - # 新触发一个build +cancel_running_build() { PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" - COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$2}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$3}" - COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$4}" - if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMIT_ID_FROM_TRIGGER}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then - echo "Usage: add_build PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" - return 1 + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 fi - local PIPELINE - PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" - if curl -s -X POST \ - -u OneMoreChance:OneMoreChance \ - -H "Content-Type:text/plain" \ - -H "Accept: application/json" \ - "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.commit_id_from_trigger&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times_from_trigger&value=${COMMENT_REPEAT_TIMES:-1}"; then - echo "INFO: Add new build to PIPELINE ${PIPELINE} of PR ${PULL_REQUEST_NUM} with COMMENT_REPEAT_TIMES ${COMMENT_REPEAT_TIMES:-1}" - else - return 1 - fi -} -cancel_running_build() { - echo "TODO: cancel_running_build" + PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local build_ids + if ! build_ids=$(get_running_build_of_pr "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}"); then return 1; fi + for id in ${build_ids}; do + if curl -s -X POST \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:application/json" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/app/rest/builds/id:${id}" \ + -d '{ "comment": "Canceling a running build before triggering a new one", "readdIntoQueue": false }'; then + echo "INFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" + else + echo "WARNING: failed to cancel running build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" + fi + done } +# cancel_running_build "$1" "$2" cancel_queue_build() { - echo "TODO: cancel_queue_build" + PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 + fi + + PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local build_ids + if ! build_ids=$(get_queue_build_of_pr "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}"); then return 1; fi + for id in ${build_ids}; do + if curl -s -X POST \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:application/json" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/app/rest/buildQueue/id:${id}" \ + -d '{ "comment": "Canceling a queued build before triggering a new one", "readdIntoQueue": false }'; then + echo "INFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" + else + echo "WARNING: failed to cancel queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" + fi + done } +# cancel_queue_build "$1" "$2" skip_build() { # 对于不需要跑teamcity pipeline的PR，直接调用github的接口返回成功 @@ -172,6 +201,31 @@ skip_build() { return 1 fi } +# skip_build "$1" "$2" + +trigger_build() { + # 新触发一个build + PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$2}" + COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$3}" + COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$4}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMIT_ID_FROM_TRIGGER}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then + echo "Usage: add_build PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" + return 1 + fi + local PIPELINE + PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + if curl -s -X POST \ + -u OneMoreChance:OneMoreChance \ + -H "Content-Type:text/plain" \ + -H "Accept: application/json" \ + "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.commit_id_from_trigger&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times_from_trigger&value=${COMMENT_REPEAT_TIMES:-1}"; then + echo "INFO: Add new build to PIPELINE ${PIPELINE} of PR ${PULL_REQUEST_NUM} with COMMENT_REPEAT_TIMES ${COMMENT_REPEAT_TIMES:-1}" + else + return 1 + fi +} +# trigger_build "$1" "$2" "$3" "$4" trigger_or_skip_build() { # 根据相关文件是否修改，来触发or跳过跑流水线 @@ -189,14 +243,11 @@ trigger_or_skip_build() { fi if [[ "${FILE_CHANGED}" == "true" ]]; then + cancel_running_build "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}" + cancel_queue_build "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}" trigger_build "${PULL_REQUEST_NUM}" "${COMMIT_ID_FROM_TRIGGER}" "${COMMENT_TRIGGER_TYPE}" "${COMMENT_REPEAT_TIMES}" else skip_build "${COMMIT_ID_FROM_TRIGGER}" "${COMMENT_TRIGGER_TYPE}" fi } - -# skip_teamcity_pipeline '9940812522228f574c5b630666189bc0aa4b1c60' clickbench - -cancel_and_add_build() { - echo "try to cancel queue or running build first, then add build" -} +# trigger_or_skip_build "$1" "$2" "$3" "$4" "$5" From 0836662d5aea67d62845f499d6ede8a93b305d96 Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 20 Nov 2023 20:37:36 +0800 Subject: [PATCH 51/76] 0 --- .../workflows/comment-to-trigger-teamcity.yml | 32 +++++++++++++------ .../pipeline/common/github-utils.sh | 26 ++++++++------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index f44fb55fb3f057..adb1a60f8f0aa9 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -77,24 +77,36 @@ jobs: run: | source regression-test/pipeline/common/github-utils.sh set -x - _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" - if file_changed_fe_ut; then + if _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}"; then + if file_changed_fe_ut; then + echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT" + fi + if file_changed_be_ut; then + echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT" + fi + if file_changed_ckb; then + echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT" + fi + if file_changed_regression_p0; then + echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_external=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" + fi + if file_changed_regression_p1; then + echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" + fi + else + echo "INFO: failed to _get_pr_changed_files, default trigger all" echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT" - fi - if file_changed_be_ut; then echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT" - fi - if file_changed_ckb; then echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT" echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT" - fi - if file_changed_regression_p0; then echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_external=true" | tee -a "$GITHUB_OUTPUT" echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" - fi - if file_changed_regression_p1; then echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" fi diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 1842f86281c096..3069cbd76a471a 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -73,14 +73,18 @@ _get_pr_changed_files_count() { OWNER="${OWNER:=apache}" REPO="${REPO:=doris}" - if ret=$( - curl -s -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}" | jq -e '.changed_files' - ); then - echo "${ret}" - else - return 1 - fi + try_times=10 + while [[ ${try_times} -gt 0 ]]; do + if ret=$( + curl -s -H "Accept: application/vnd.github+json" \ + https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}" | jq -e '.changed_files' + ); then + echo "${ret}" && return + fi + sleep 1s + try_times=$((try_times - 1)) + done + if [[ ${try_times} -eq 0 ]]; then echo "Failed to get pr(${PULL_NUMBER}) changed file count" && return 1; fi } _get_pr_changed_files() { @@ -101,7 +105,7 @@ _get_pr_changed_files() { file_name='pr_changed_files' rm -f "${file_name}" page=1 - changed_files_count="$(_get_pr_changed_files_count "${PULL_NUMBER}")" + if ! changed_files_count="$(_get_pr_changed_files_count "${PULL_NUMBER}")"; then return 1; fi while [[ ${changed_files_count} -gt 0 ]]; do try_times=10 while [[ ${try_times} -gt 0 ]]; do @@ -121,14 +125,14 @@ _get_pr_changed_files() { page=$((page + 1)) changed_files_count=$((changed_files_count - per_page)) done - if [[ ${try_times} = 0 ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi + if [[ ${try_times} -eq 0 ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 1; fi all_files=$(jq -r '.[] | .filename' "${file_name}") added_files=$(jq -r '.[] | select(.status == "added") | .filename' "${file_name}") modified_files=$(jq -r '.[] | select(.status == "modified") | .filename' "${file_name}") removed_files=$(jq -r '.[] | select(.status == "removed") | .filename' "${file_name}") rm "${file_name}" - if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 255; fi + if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 1; fi echo -e " https://github.com/apache/doris/pull/${PULL_NUMBER}/files all change files: From 3507dfd79aa88952e4d24cf7a34b710476a4901c Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 20 Nov 2023 21:10:47 +0800 Subject: [PATCH 52/76] 0 --- regression-test/pipeline/common/teamcity-utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index fe27b407fcc650..59f6768960e8d7 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -140,7 +140,7 @@ cancel_running_build() { -H "Content-Type:application/json" \ -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/builds/id:${id}" \ - -d '{ "comment": "Canceling a running build before triggering a new one", "readdIntoQueue": false }'; then + -d '{ "comment": "Canceling this running build before triggering a new one", "readdIntoQueue": false }'; then echo "INFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" else echo "WARNING: failed to cancel running build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" @@ -165,7 +165,7 @@ cancel_queue_build() { -H "Content-Type:application/json" \ -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/buildQueue/id:${id}" \ - -d '{ "comment": "Canceling a queued build before triggering a new one", "readdIntoQueue": false }'; then + -d '{ "comment": "Canceling this queued build before triggering a new one", "readdIntoQueue": false }'; then echo "INFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" else echo "WARNING: failed to cancel queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" From 70efbefcc0040cda51f36d3041bc87753cb98c46 Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 20 Nov 2023 21:43:47 +0800 Subject: [PATCH 53/76] 0 --- .../pipeline/common/teamcity-utils.sh | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index 59f6768960e8d7..3aa1e3d0defd97 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -60,6 +60,7 @@ get_commit_id_of_build() { # 获取某个build的commit id if [[ -z "$1" ]]; then return 1; fi build_id="$1" + set -x if ret=$( curl -s -X GET \ -u OneMoreChance:OneMoreChance \ @@ -67,9 +68,11 @@ get_commit_id_of_build() { -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/builds/${build_id}" ); then + set +x commit_id=$(echo "${ret}" | jq -r '.revisions.revision[0].version') echo "${commit_id}" else + set +x return 1 fi } @@ -84,6 +87,7 @@ get_running_build_of_pr() { PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local running_builds_list + set -x if ret=$( curl -s -X GET \ -u OneMoreChance:OneMoreChance \ @@ -91,9 +95,11 @@ get_running_build_of_pr() { -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/builds?locator=buildType:${PIPELINE},branch:pull/${PULL_REQUEST_NUM},running:true" ); then + set +x running_builds_list=$(echo "${ret}" | jq -r '.build[].id') echo "${running_builds_list}" else + set +x return 1 fi } @@ -109,6 +115,7 @@ get_queue_build_of_pr() { PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local queue_builds_list + set -x if ret=$( curl -s -X GET \ -u OneMoreChance:OneMoreChance \ @@ -116,9 +123,11 @@ get_queue_build_of_pr() { -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/buildQueue?locator=buildType:${PIPELINE}" ); then + set +x queue_builds_list=$(echo "${ret}" | jq ".build[] | select(.branchName == \"pull/${PULL_REQUEST_NUM}\") | .id") echo "${queue_builds_list}" else + set +x echo "WARNING: failed to get queue build for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" && return 1 fi } @@ -135,14 +144,17 @@ cancel_running_build() { local build_ids if ! build_ids=$(get_running_build_of_pr "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}"); then return 1; fi for id in ${build_ids}; do + set -x if curl -s -X POST \ -u OneMoreChance:OneMoreChance \ -H "Content-Type:application/json" \ -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/builds/id:${id}" \ -d '{ "comment": "Canceling this running build before triggering a new one", "readdIntoQueue": false }'; then - echo "INFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" + set +x + echo -e "\nINFO: canceled running build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" else + set +x echo "WARNING: failed to cancel running build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" fi done @@ -160,14 +172,17 @@ cancel_queue_build() { local build_ids if ! build_ids=$(get_queue_build_of_pr "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}"); then return 1; fi for id in ${build_ids}; do + set -x if curl -s -X POST \ -u OneMoreChance:OneMoreChance \ -H "Content-Type:application/json" \ -H "Accept: application/json" \ "http://43.132.222.7:8111/app/rest/buildQueue/id:${id}" \ -d '{ "comment": "Canceling this queued build before triggering a new one", "readdIntoQueue": false }'; then - echo "INFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" + set +x + echo -e "\nINFO: canceled queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" else + set +x echo "WARNING: failed to cancel queue build(id ${id}) for PR ${PULL_REQUEST_NUM} of pipeline ${PIPELINE}" fi done @@ -189,6 +204,7 @@ skip_build() { local state="${TC_BUILD_STATE:-success}" # 可选值 success failure pending payload="{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"Skip teamCity build\",\"context\":\"${conment_to_context[${COMMENT_TRIGGER_TYPE}]}\"}" + set -x if curl -L \ -X POST \ -H "Accept: application/vnd.github+json" \ @@ -196,8 +212,10 @@ skip_build() { -H "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/apache/doris/statuses/${COMMIT_ID_FROM_TRIGGER:-}" \ -d "${payload}"; then + set +x echo "INFO: Skipped ${COMMIT_ID_FROM_TRIGGER} ${COMMENT_TRIGGER_TYPE}" else + set +x return 1 fi } @@ -215,13 +233,16 @@ trigger_build() { fi local PIPELINE PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + set -x if curl -s -X POST \ -u OneMoreChance:OneMoreChance \ -H "Content-Type:text/plain" \ -H "Accept: application/json" \ "http://43.132.222.7:8111/httpAuth/action.html?add2Queue=${PIPELINE}&branchName=pull/${PULL_REQUEST_NUM}&name=env.commit_id_from_trigger&value=${COMMIT_ID_FROM_TRIGGER:-}&name=env.repeat_times_from_trigger&value=${COMMENT_REPEAT_TIMES:-1}"; then + set +x echo "INFO: Add new build to PIPELINE ${PIPELINE} of PR ${PULL_REQUEST_NUM} with COMMENT_REPEAT_TIMES ${COMMENT_REPEAT_TIMES:-1}" else + set +x return 1 fi } From 13afbd2b4b38f027f7df51d7168f8132c2653559 Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 20 Nov 2023 21:46:05 +0800 Subject: [PATCH 54/76] 0 --- regression-test/pipeline/common/github-utils.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 3069cbd76a471a..73055e4fdd7e1a 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -75,15 +75,18 @@ _get_pr_changed_files_count() { REPO="${REPO:=doris}" try_times=10 while [[ ${try_times} -gt 0 ]]; do + set -x if ret=$( curl -s -H "Accept: application/vnd.github+json" \ https://api.github.com/repos/"${OWNER}"/"${REPO}"/pulls/"${PULL_NUMBER}" | jq -e '.changed_files' ); then + set +x echo "${ret}" && return fi sleep 1s try_times=$((try_times - 1)) done + set +x if [[ ${try_times} -eq 0 ]]; then echo "Failed to get pr(${PULL_NUMBER}) changed file count" && return 1; fi } From 971ed403d0d86237fdd33b203a1408d06e5fb424 Mon Sep 17 00:00:00 2001 From: stephen Date: Mon, 20 Nov 2023 21:49:42 +0800 Subject: [PATCH 55/76] 0 --- .github/workflows/comment-to-trigger-teamcity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index adb1a60f8f0aa9..0057dc7f4d6c42 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -36,6 +36,7 @@ jobs: - name: "Parse PR comment" id: parse run: | + COMMENT_BODY=$(echo "${COMMENT_BODY}" | xargs) if [[ "${COMMENT_BODY}" == *'run buildall'* || "${COMMENT_BODY}" == *'run compile'* || "${COMMENT_BODY}" == *'run beut'* || From ad81a3dca68f55e099de9b792e01cf3b780a9db4 Mon Sep 17 00:00:00 2001 From: stephen Date: Wed, 22 Nov 2023 09:03:52 +0800 Subject: [PATCH 56/76] 2 --- regression-test/pipeline/common/github-utils.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 73055e4fdd7e1a..4a7cfd4ed2656c 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -112,7 +112,6 @@ _get_pr_changed_files() { while [[ ${changed_files_count} -gt 0 ]]; do try_times=10 while [[ ${try_times} -gt 0 ]]; do - # TODO: 修改文件多于100个的情况未处理 set -x if curl -s \ -H "Accept: application/vnd.github+json" \ From 84bd7337cc0ca7e68ada05e272c6c37f3239e549 Mon Sep 17 00:00:00 2001 From: stephen Date: Thu, 23 Nov 2023 16:13:08 +0800 Subject: [PATCH 57/76] 3 --- .../pipeline/common/github-utils.sh | 31 ++++++--- .../pipeline/common/teamcity-utils.sh | 69 +++++++++++-------- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh index 4a7cfd4ed2656c..63481d1173427c 100644 --- a/regression-test/pipeline/common/github-utils.sh +++ b/regression-test/pipeline/common/github-utils.sh @@ -135,24 +135,34 @@ _get_pr_changed_files() { removed_files=$(jq -r '.[] | select(.status == "removed") | .filename' "${file_name}") rm "${file_name}" if [[ -z "${all_files}" ]]; then echo -e "\033[31m List pull request(${pr_url}) files FAIL... \033[0m" && return 1; fi + echo "${all_files}" >all_files + echo "${added_files}" >added_files + echo "${modified_files}" >modified_files + echo "${removed_files}" >removed_files echo -e " https://github.com/apache/doris/pull/${PULL_NUMBER}/files all change files: ---------------------------------------------------------------" if [[ "${which_file:-all}" == "all" ]]; then - echo -e "${all_files}\n" && export all_files + echo -e "${all_files}\n" elif [[ "${which_file}" == "added" ]]; then - echo -e "${added_files}\n" && export added_files + echo -e "${added_files}\n" elif [[ "${which_file}" == "modified" ]]; then - echo -e "${modified_files}\n" && export modified_files + echo -e "${modified_files}\n" elif [[ "${which_file}" == "removed" ]]; then - echo -e "${removed_files}\n" && export removed_files + echo -e "${removed_files}\n" else return 1 fi } _only_modified_regression_conf() { + local added_files + local removed_files + local modified_files + added_files=$(cat added_files) + removed_files=$(cat removed_files) + modified_files=$(cat modified_files) if [[ -n ${added_files} || -n ${removed_files} ]]; then # echo "Not only modified regression conf, find added/removed files" return 1 @@ -175,6 +185,8 @@ _only_modified_regression_conf() { } file_changed_fe_ut() { + local all_files + all_files=$(cat all_files) if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do @@ -190,6 +202,8 @@ file_changed_fe_ut() { } file_changed_be_ut() { + local all_files + all_files=$(cat all_files) if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do @@ -206,6 +220,8 @@ file_changed_be_ut() { } file_changed_regression_p0() { + local all_files + all_files=$(cat all_files) if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do @@ -234,12 +250,9 @@ file_changed_regression_p1() { file_changed_regression_p0 } -file_changed_arm_regression_p0() { - if [[ $(($1 % 2)) -eq 0 ]]; then echo "the pull request id is even, return no need" && return 1; fi - file_changed_regression_p0 -} - file_changed_ckb() { + local all_files + all_files=$(cat all_files) if _only_modified_regression_conf; then echo "return no need" && return 1; fi if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi for af in ${all_files}; do diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh index 3aa1e3d0defd97..57ae0925fd5cad 100644 --- a/regression-test/pipeline/common/teamcity-utils.sh +++ b/regression-test/pipeline/common/teamcity-utils.sh @@ -59,7 +59,9 @@ conment_to_context=( get_commit_id_of_build() { # 获取某个build的commit id if [[ -z "$1" ]]; then return 1; fi - build_id="$1" + local build_id="$1" + local commit_id + local ret set -x if ret=$( curl -s -X GET \ @@ -79,14 +81,15 @@ get_commit_id_of_build() { get_running_build_of_pr() { # "获取pr在某条流水线上正在跑的build" - PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + local PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 fi - PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local running_builds_list + local ret set -x if ret=$( curl -s -X GET \ @@ -107,14 +110,15 @@ get_running_build_of_pr() { get_queue_build_of_pr() { # "获取pr在某条流水线上排队的build" - PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + local PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 fi - PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local queue_builds_list + local ret set -x if ret=$( curl -s -X GET \ @@ -134,13 +138,13 @@ get_queue_build_of_pr() { # get_queue_build_of_pr "$1" "$2" cancel_running_build() { - PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + local PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 fi - PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local build_ids if ! build_ids=$(get_running_build_of_pr "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}"); then return 1; fi for id in ${build_ids}; do @@ -162,13 +166,13 @@ cancel_running_build() { # cancel_running_build "$1" "$2" cancel_queue_build() { - PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" + local PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$2}" if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: get_queue_build_of_pr PULL_REQUEST_NUM COMMENT_TRIGGER_TYPE" && return 1 fi - PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" local build_ids if ! build_ids=$(get_queue_build_of_pr "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}"); then return 1; fi for id in ${build_ids}; do @@ -199,11 +203,11 @@ skip_build() { echo "Usage: skip_teamcity_pipeline COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE" return 1 fi - COMMIT_ID_FROM_TRIGGER="$1" - COMMENT_TRIGGER_TYPE="$2" + local COMMIT_ID_FROM_TRIGGER="$1" + local COMMENT_TRIGGER_TYPE="$2" local state="${TC_BUILD_STATE:-success}" # 可选值 success failure pending - payload="{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"Skip teamCity build\",\"context\":\"${conment_to_context[${COMMENT_TRIGGER_TYPE}]}\"}" + local payload="{\"state\":\"${state}\",\"target_url\":\"\",\"description\":\"Skip teamCity build\",\"context\":\"${conment_to_context[${COMMENT_TRIGGER_TYPE}]}\"}" set -x if curl -L \ -X POST \ @@ -223,16 +227,15 @@ skip_build() { trigger_build() { # 新触发一个build - PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" - COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$2}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$3}" - COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$4}" + local PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$1}" + local COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$2}" + local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$3}" + local COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$4}" if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMIT_ID_FROM_TRIGGER}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: add_build PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" return 1 fi - local PIPELINE - PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" + local PIPELINE="${comment_to_pipeline[${COMMENT_TRIGGER_TYPE}]}" set -x if curl -s -X POST \ -u OneMoreChance:OneMoreChance \ @@ -250,25 +253,31 @@ trigger_build() { trigger_or_skip_build() { # 根据相关文件是否修改，来触发or跳过跑流水线 - FILE_CHANGED="$1" - PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$2}" - COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$3}" - COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$4}" - COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$5}" - if [[ -z "${FILE_CHANGED}" || - -z "${PULL_REQUEST_NUM}" || + local FILE_CHANGED="$1" # 默认为"true" + local PULL_REQUEST_NUM="${PULL_REQUEST_NUM:-$2}" + local COMMIT_ID_FROM_TRIGGER="${COMMIT_ID_FROM_TRIGGER:-$3}" + local COMMENT_TRIGGER_TYPE="${COMMENT_TRIGGER_TYPE:-$4}" + local COMMENT_REPEAT_TIMES="${COMMENT_REPEAT_TIMES:-$5}" + if [[ -z "${PULL_REQUEST_NUM}" || -z "${COMMIT_ID_FROM_TRIGGER}" || -z "${COMMENT_TRIGGER_TYPE}" ]]; then echo "Usage: add_build FILE_CHANGED PULL_REQUEST_NUM COMMIT_ID_FROM_TRIGGER COMMENT_TRIGGER_TYPE [COMMENT_REPEAT_TIMES]" return 1 fi - if [[ "${FILE_CHANGED}" == "true" ]]; then + if [[ "${FILE_CHANGED:-"true"}" == "true" ]]; then cancel_running_build "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}" cancel_queue_build "${PULL_REQUEST_NUM}" "${COMMENT_TRIGGER_TYPE}" trigger_build "${PULL_REQUEST_NUM}" "${COMMIT_ID_FROM_TRIGGER}" "${COMMENT_TRIGGER_TYPE}" "${COMMENT_REPEAT_TIMES}" else skip_build "${COMMIT_ID_FROM_TRIGGER}" "${COMMENT_TRIGGER_TYPE}" + if [[ ${COMMENT_TRIGGER_TYPE} == "compile" ]]; then + # skip compile 的时候，也把 p0 p1 external pipelinex_p0 都 skip 了 + skip_build "${COMMIT_ID_FROM_TRIGGER}" "p0" + skip_build "${COMMIT_ID_FROM_TRIGGER}" "p1" + skip_build "${COMMIT_ID_FROM_TRIGGER}" "external" + skip_build "${COMMIT_ID_FROM_TRIGGER}" "pipelinex_p0" + fi fi } # trigger_or_skip_build "$1" "$2" "$3" "$4" "$5" From 7220ad0b1e1b894da290597fe188d51c2ab54e9d Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 23 Nov 2023 16:16:56 +0800 Subject: [PATCH 58/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 0057dc7f4d6c42..825949a0a45750 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -111,9 +111,9 @@ jobs: echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" fi - # - name: "Setup tmate session" - # if: ${{ failure() }} - # uses: mxschmitt/action-tmate@v3 + - name: "Setup tmate session" + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip feut" if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} From bae33385ea261f8a0db82a5ec971b4c214ca672f Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 23 Nov 2023 16:18:42 +0800 Subject: [PATCH 59/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 825949a0a45750..a471334652f299 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -112,7 +112,6 @@ jobs: fi - name: "Setup tmate session" - if: ${{ failure() }} uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip feut" From 1936ddfa6b7349a6f319db265f15eba2ceed34e1 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 11:44:00 +0800 Subject: [PATCH 60/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index a471334652f299..072f2cef8082c4 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -111,8 +111,8 @@ jobs: echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" fi - - name: "Setup tmate session" - uses: mxschmitt/action-tmate@v3 + # - name: "Setup tmate session" + # uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip feut" if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} From 0b5a803d2903bbab6aa574ca7861ee3ed5cf89a3 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 12:06:00 +0800 Subject: [PATCH 61/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 072f2cef8082c4..da2d3c00f95409 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -65,8 +65,8 @@ jobs: reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" - echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" - echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_TRIGGER_TYPE='${COMMENT_TRIGGER_TYPE}'" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_REPEAT_TIMES='${COMMENT_REPEAT_TIMES}'" | tee -a "$GITHUB_OUTPUT" - name: "Checkout master" if: ${{ steps.parse.outputs.comment_trigger }} From 89d2917a96c962ad9b36e3e13c5a64abf23d75ff Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 14:38:35 +0800 Subject: [PATCH 62/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index da2d3c00f95409..16778267e01fcb 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -35,7 +35,7 @@ jobs: steps: - name: "Parse PR comment" id: parse - run: | + run: |- COMMENT_BODY=$(echo "${COMMENT_BODY}" | xargs) if [[ "${COMMENT_BODY}" == *'run buildall'* || "${COMMENT_BODY}" == *'run compile'* || From ff7dc2c2b557bd7d945b91d87e233e3a59fa57f9 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 14:40:47 +0800 Subject: [PATCH 63/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 16778267e01fcb..3a25a87dfaa777 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -63,8 +63,8 @@ jobs: echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" - COMMENT_TRIGGER_TYPE="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" - COMMENT_REPEAT_TIMES="$(echo "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" + COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" + COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" echo "COMMENT_TRIGGER_TYPE='${COMMENT_TRIGGER_TYPE}'" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_REPEAT_TIMES='${COMMENT_REPEAT_TIMES}'" | tee -a "$GITHUB_OUTPUT" From a1cc511c87f8c0286710fdce77451db0bf068d33 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 14:44:49 +0800 Subject: [PATCH 64/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 3a25a87dfaa777..57446af23c1017 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -128,7 +128,7 @@ jobs: - name: "Trigger or Skip beut" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger == "true" && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x From 6906a9f253d0ff427a7ae882d6002153c7a38fa8 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 14:46:56 +0800 Subject: [PATCH 65/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 57446af23c1017..26e54fdbfcc7e4 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -128,7 +128,7 @@ jobs: - name: "Trigger or Skip beut" - if: ${{ steps.parse.outputs.comment_trigger == "true" && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ steps.parse.outputs.comment_trigger == true && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x From 48d1001fcd1ed0773f02eafc7b54370eba99d0c1 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Mon, 27 Nov 2023 14:53:16 +0800 Subject: [PATCH 66/76] Update comment-to-trigger-teamcity.yml --- .../workflows/comment-to-trigger-teamcity.yml | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 26e54fdbfcc7e4..92f0d6b962d8d3 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -35,7 +35,7 @@ jobs: steps: - name: "Parse PR comment" id: parse - run: |- + run: | COMMENT_BODY=$(echo "${COMMENT_BODY}" | xargs) if [[ "${COMMENT_BODY}" == *'run buildall'* || "${COMMENT_BODY}" == *'run compile'* || @@ -50,6 +50,7 @@ jobs: "${COMMENT_BODY}" == *'run tpch'* ]]; then echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT" else + echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT" echo "find no keyword in comment body, skip this action." exit fi @@ -65,15 +66,15 @@ jobs: reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" - echo "COMMENT_TRIGGER_TYPE='${COMMENT_TRIGGER_TYPE}'" | tee -a "$GITHUB_OUTPUT" - echo "COMMENT_REPEAT_TIMES='${COMMENT_REPEAT_TIMES}'" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" + echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" - name: "Checkout master" - if: ${{ steps.parse.outputs.comment_trigger }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) }} uses: actions/checkout@v4 - name: "Check if pr need run build" - if: ${{ steps.parse.outputs.comment_trigger }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) }} id: changes run: | source regression-test/pipeline/common/github-utils.sh @@ -81,22 +82,36 @@ jobs: if _get_pr_changed_files "${{ steps.parse.outputs.PULL_REQUEST_NUM }}"; then if file_changed_fe_ut; then echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_fe_ut=false" | tee -a "$GITHUB_OUTPUT" fi if file_changed_be_ut; then echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT" fi if file_changed_ckb; then echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT" echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_ckb=false" | tee -a "$GITHUB_OUTPUT" + echo "changed_tpch=false" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p0; then echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_external=true" | tee -a "$GITHUB_OUTPUT" echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_p0=false" | tee -a "$GITHUB_OUTPUT" + echo "changed_external=false" | tee -a "$GITHUB_OUTPUT" + echo "changed_pipelinex_p0=false" | tee -a "$GITHUB_OUTPUT" + echo "changed_arm=false" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p1; then echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_p1=false" | tee -a "$GITHUB_OUTPUT" fi else echo "INFO: failed to _get_pr_changed_files, default trigger all" @@ -112,10 +127,11 @@ jobs: fi # - name: "Setup tmate session" + # if: ${{ failure() }} # uses: mxschmitt/action-tmate@v3 - name: "Trigger or Skip feut" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["feut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x @@ -128,7 +144,7 @@ jobs: - name: "Trigger or Skip beut" - if: ${{ steps.parse.outputs.comment_trigger == true && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["beut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x @@ -140,7 +156,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip compile" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x @@ -152,7 +168,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip p0" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then @@ -167,7 +183,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip p1" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh COMMIT_ID_FROM_TRIGGER=${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }} @@ -183,7 +199,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip external" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["external", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["external", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then @@ -198,7 +214,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip pipelinex_p0" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then @@ -213,7 +229,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip arm" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x @@ -225,7 +241,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip clickbench" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x @@ -237,7 +253,7 @@ jobs: "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip tpch" - if: ${{ steps.parse.outputs.comment_trigger && contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then From 0e252fb8f731a3050ec9938138bcb61174f5b92f Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Wed, 29 Nov 2023 11:53:06 +0800 Subject: [PATCH 67/76] Update comment-to-trigger-teamcity.yml From d9d482d1fe26e9610c786afac9fe23c50cf98c40 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Wed, 29 Nov 2023 12:12:52 +0800 Subject: [PATCH 68/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 92f0d6b962d8d3..eb81eff09e5fc9 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -64,8 +64,8 @@ jobs: echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT" reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" - COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p)" - COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p)" + COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')" + COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')" echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_REPEAT_TIMES=${COMMENT_REPEAT_TIMES}" | tee -a "$GITHUB_OUTPUT" From 654113530665655675a84988bde91a18d33be750 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 28 Dec 2023 21:07:44 +0800 Subject: [PATCH 69/76] Update comment-to-trigger-teamcity.yml --- .../workflows/comment-to-trigger-teamcity.yml | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index eb81eff09e5fc9..7634a8ead5b0c6 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -47,7 +47,8 @@ jobs: "${COMMENT_BODY}" == *'run pipelinex_p0'* || "${COMMENT_BODY}" == *'run clickbench'* || "${COMMENT_BODY}" == *'run arm'* || - "${COMMENT_BODY}" == *'run tpch'* ]]; then + "${COMMENT_BODY}" == *'run tpch'* || + "${COMMENT_BODY}" == *'run performance'* ]]; then echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT" else echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT" @@ -63,7 +64,7 @@ jobs: echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT" - reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*" + reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch|performance)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')" COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')" echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" @@ -113,6 +114,11 @@ jobs: else echo "changed_p1=false" | tee -a "$GITHUB_OUTPUT" fi + if file_changed_performance; then + echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT" + fi else echo "INFO: failed to _get_pr_changed_files, default trigger all" echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT" @@ -124,6 +130,7 @@ jobs: echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT" fi # - name: "Setup tmate session" @@ -266,3 +273,15 @@ jobs: "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ "tpch" \ "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + + - name: "Trigger or Skip performance" + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_performance }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "performance" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" From 738149dc98a4b3ace44de344536a7bc458f55a50 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Fri, 29 Dec 2023 15:14:26 +0800 Subject: [PATCH 70/76] Update comment-to-trigger-teamcity.yml --- .../workflows/comment-to-trigger-teamcity.yml | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 7634a8ead5b0c6..7a2f1076604398 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -279,9 +279,20 @@ jobs: run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x - trigger_or_skip_build \ - "${{ steps.changes.outputs.changed_performance }}" \ - "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ - "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ - "performance" \ - "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" || "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then + echo "PR target branch is master or branch-2.0, need run performance" + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_performance }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "performance" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + echo "PR target branch is not master or branch-2.0, skip run performance" + trigger_or_skip_build \ + "false" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "performance" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + fi From c91d0d22da8a73e0da95a3dab611e97ae6f5ca5e Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 28 Mar 2024 22:09:34 +0800 Subject: [PATCH 71/76] Update comment-to-trigger-teamcity.yml --- .../workflows/comment-to-trigger-teamcity.yml | 142 +++++++++++------- 1 file changed, 88 insertions(+), 54 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 7a2f1076604398..7b73fef9f2dbbe 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -41,13 +41,13 @@ jobs: "${COMMENT_BODY}" == *'run compile'* || "${COMMENT_BODY}" == *'run beut'* || "${COMMENT_BODY}" == *'run feut'* || + "${COMMENT_BODY}" == *'run cloudut'* || "${COMMENT_BODY}" == *'run p0'* || "${COMMENT_BODY}" == *'run p1'* || "${COMMENT_BODY}" == *'run external'* || - "${COMMENT_BODY}" == *'run pipelinex_p0'* || - "${COMMENT_BODY}" == *'run clickbench'* || + "${COMMENT_BODY}" == *'run cloud_p0'* || + "${COMMENT_BODY}" == *'run cloud_p1'* || "${COMMENT_BODY}" == *'run arm'* || - "${COMMENT_BODY}" == *'run tpch'* || "${COMMENT_BODY}" == *'run performance'* ]]; then echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT" else @@ -64,7 +64,7 @@ jobs: echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT" echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT" - reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch|performance)( [1-9]*[0-9]+)*" + reg="run (buildall|compile|p0|p1|feut|beut|cloudut|external|clickbench|cloud_p0|cloud_p1|arm|performance)( [1-9]*[0-9]+)*" COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')" COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')" echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT" @@ -91,22 +91,18 @@ jobs: else echo "changed_be_ut=false" | tee -a "$GITHUB_OUTPUT" fi - if file_changed_ckb; then - echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT" - echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT" + if file_changed_cloud_ut; then + echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT" else - echo "changed_ckb=false" | tee -a "$GITHUB_OUTPUT" - echo "changed_tpch=false" | tee -a "$GITHUB_OUTPUT" + echo "changed_cloud_ut=false" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p0; then echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_external=true" | tee -a "$GITHUB_OUTPUT" - echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" else echo "changed_p0=false" | tee -a "$GITHUB_OUTPUT" echo "changed_external=false" | tee -a "$GITHUB_OUTPUT" - echo "changed_pipelinex_p0=false" | tee -a "$GITHUB_OUTPUT" echo "changed_arm=false" | tee -a "$GITHUB_OUTPUT" fi if file_changed_regression_p1; then @@ -119,18 +115,28 @@ jobs: else echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT" fi + if file_changed_cloud_p0; then + echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_cloud_p0=false" | tee -a "$GITHUB_OUTPUT" + fi + if file_changed_cloud_p1; then + echo "changed_cloud_p1=true" | tee -a "$GITHUB_OUTPUT" + else + echo "changed_cloud_p1=false" | tee -a "$GITHUB_OUTPUT" + fi else echo "INFO: failed to _get_pr_changed_files, default trigger all" echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT" echo "changed_be_ut=true" | tee -a "$GITHUB_OUTPUT" - echo "changed_ckb=true" | tee -a "$GITHUB_OUTPUT" - echo "changed_tpch=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_cloud_ut=true" | tee -a "$GITHUB_OUTPUT" echo "changed_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_external=true" | tee -a "$GITHUB_OUTPUT" - echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT" echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT" echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT" echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_cloud_p0=true" | tee -a "$GITHUB_OUTPUT" + echo "changed_cloud_p1=true" | tee -a "$GITHUB_OUTPUT" fi # - name: "Setup tmate session" @@ -162,13 +168,30 @@ jobs: "beut" \ "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + - name: "Trigger or Skip cloudut" + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloudut", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + run: | + source ./regression-test/pipeline/common/teamcity-utils.sh + set -x + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_cloud_ut }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "cloudut" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + - name: "Trigger or Skip compile" if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["compile", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x + if [[ "${{ steps.changes.outputs.changed_p0 }}" == "true" ]] || [[ "${{ steps.changes.outputs.changed_cloud_p0 }}" == "true" ]]; then + changed="true" + else + chenged="false" + fi trigger_or_skip_build \ - "${{ steps.changes.outputs.changed_p0 }}" \ + "${chenged}" \ "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ "compile" \ @@ -220,58 +243,68 @@ jobs: "external" \ "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - - name: "Trigger or Skip pipelinex_p0" - if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["pipelinex_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + - name: "Trigger or Skip cloud_p0" + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p0", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger pipelinex_p0" && exit + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p0" && exit fi set -x - trigger_or_skip_build \ - "${{ steps.changes.outputs.changed_pipelinex_p0 }}" \ - "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ - "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ - "pipelinex_p0" \ - "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - - - name: "Trigger or Skip arm" - if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} - run: | - source ./regression-test/pipeline/common/teamcity-utils.sh - set -x - trigger_or_skip_build \ - "${{ steps.changes.outputs.changed_arm }}" \ - "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ - "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ - "arm" \ - "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then + echo "PR target branch in (master), need run cloud_p0" + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_cloud_p0 }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "cloud_p0" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + echo "PR target branch not in (master), skip run cloud_p0" + trigger_or_skip_build \ + "false" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "cloud_p0" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + fi - - name: "Trigger or Skip clickbench" - if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["clickbench", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + - name: "Trigger or Skip cloud_p1" + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["cloud_p1", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh + if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then + echo "COMMENT_TRIGGER_TYPE is buildall, trigger compile is enough, compile will trigger cloud_p1" && exit + fi set -x - trigger_or_skip_build \ - "${{ steps.changes.outputs.changed_ckb }}" \ - "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ - "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ - "clickbench" \ - "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then + echo "PR target branch in (master), need run cloud_p1" + trigger_or_skip_build \ + "${{ steps.changes.outputs.changed_cloud_p1 }}" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "cloud_p1" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + else + echo "PR target branch not in (master), skip run cloud_p1" + trigger_or_skip_build \ + "false" \ + "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ + "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ + "cloud_p1" \ + "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" + fi - - name: "Trigger or Skip tpch" - if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["tpch", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} + - name: "Trigger or Skip arm" + if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["arm", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }} run: | source ./regression-test/pipeline/common/teamcity-utils.sh - if [[ ${{ steps.parse.outputs.COMMENT_TRIGGER_TYPE }} == "buildall" ]]; then - echo "COMMENT_TRIGGER_TYPE is buildall, trigger clickbench is enough, clickbench will trigger tpch" && exit - fi set -x trigger_or_skip_build \ - "${{ steps.changes.outputs.changed_tpch }}" \ + "${{ steps.changes.outputs.changed_arm }}" \ "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ - "tpch" \ + "arm" \ "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" - name: "Trigger or Skip performance" @@ -279,8 +312,9 @@ jobs: run: | source ./regression-test/pipeline/common/teamcity-utils.sh set -x - if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" || "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then - echo "PR target branch is master or branch-2.0, need run performance" + if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" || + "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'branch-2.0'" ]]; then + echo "PR target branch in (master, branch-2.0), need run performance" trigger_or_skip_build \ "${{ steps.changes.outputs.changed_performance }}" \ "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ @@ -288,7 +322,7 @@ jobs: "performance" \ "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}" else - echo "PR target branch is not master or branch-2.0, skip run performance" + echo "PR target branch not in (master, branch-2.0), skip run performance" trigger_or_skip_build \ "false" \ "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ From ff8d26f9a37455d0910699acd37478e8ab3e064b Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Thu, 28 Mar 2024 22:15:20 +0800 Subject: [PATCH 72/76] Update comment-to-trigger-teamcity.yml --- .github/workflows/comment-to-trigger-teamcity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml index 7b73fef9f2dbbe..d1f0eb3ddff2f1 100644 --- a/.github/workflows/comment-to-trigger-teamcity.yml +++ b/.github/workflows/comment-to-trigger-teamcity.yml @@ -188,10 +188,10 @@ jobs: if [[ "${{ steps.changes.outputs.changed_p0 }}" == "true" ]] || [[ "${{ steps.changes.outputs.changed_cloud_p0 }}" == "true" ]]; then changed="true" else - chenged="false" + changed="false" fi trigger_or_skip_build \ - "${chenged}" \ + "${changed}" \ "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \ "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \ "compile" \ From 5f31b280f4436ebb9ce438e8eb4fdce0fc2fb067 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Tue, 9 Apr 2024 21:53:02 +0800 Subject: [PATCH 73/76] Update clang-format.yml --- .github/workflows/clang-format.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index c804b753a3bbc4..ef39fa6c45fe11 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -17,7 +17,7 @@ # --- -name: BE Code Formatter +name: Code Formatter on: [push, pull_request_target] @@ -52,9 +52,11 @@ jobs: id: filter with: filters: | - be_changes: + changes: - 'be/src/**' - 'be/test/**' + - 'cloud/src/**' + - 'cloud/test/**' - name: Checkout clang-format-lint-action run: | @@ -66,14 +68,14 @@ jobs: popd &>/dev/null - name: "Format it!" - if: ${{ steps.filter.outputs.be_changes == 'true' }} + if: ${{ steps.filter.outputs.changes == 'true' }} uses: ./.github/actions/clang-format-lint-action - id: be_clang_format + id: clang_format with: - source: "be/src be/test" + source: "be/src be/test cloud/src cloud/test" clangFormatVersion: 16 inplace: False - name: Ignore it! - if: ${{ steps.filter.outputs.be_changes == 'false' }} + if: ${{ steps.filter.outputs.changes == 'false' }} run: echo 'No need to check.' From 3134a86d44732be759ba0429f1fc3ba0707c3346 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Tue, 9 Apr 2024 21:59:35 +0800 Subject: [PATCH 74/76] Update file_factory.cpp --- be/src/io/file_factory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index 63ddc3d43096be..c84daaccc91150 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -77,7 +77,7 @@ Status FileFactory::create_file_writer(TFileType::type type, ExecEnv* env, std::shared_ptr fs; RETURN_IF_ERROR(io::BrokerFileSystem::create(broker_addresses[0], properties, &fs)); RETURN_IF_ERROR(fs->create_file(path, &file_writer)); - break; + break; } case TFileType::FILE_S3: { S3URI s3_uri(path); From b78cf1a16145275ac62ce89cd33199b12ce83179 Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Tue, 9 Apr 2024 22:08:22 +0800 Subject: [PATCH 75/76] Update file_factory.cpp --- be/src/io/file_factory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp index c84daaccc91150..63ddc3d43096be 100644 --- a/be/src/io/file_factory.cpp +++ b/be/src/io/file_factory.cpp @@ -77,7 +77,7 @@ Status FileFactory::create_file_writer(TFileType::type type, ExecEnv* env, std::shared_ptr fs; RETURN_IF_ERROR(io::BrokerFileSystem::create(broker_addresses[0], properties, &fs)); RETURN_IF_ERROR(fs->create_file(path, &file_writer)); - break; + break; } case TFileType::FILE_S3: { S3URI s3_uri(path); From 20e26a76b99a3a7b3e8109c71dfd3dbd89b841ce Mon Sep 17 00:00:00 2001 From: Dongyang Li Date: Tue, 9 Apr 2024 22:08:57 +0800 Subject: [PATCH 76/76] Update Config.java --- fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index e62b226bf5dc23..fdd7ad09388b7f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -30,7 +30,7 @@ public class Config extends ConfigBase { /** * sys_log_dir: - * This specifies FE log dir. FE will produces 2 log files: + * This specifies FE log dir. FE will produce 2 log files: * fe.log: all logs of FE process. * fe.warn.log all WARN and ERROR log of FE process. *