diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index 8cdb3d7ad..16e6922aa 100755 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -72,3 +72,8 @@ fi # Export S3_BUCKET and S3_BUCKET_PREFIX, to force our CI to upload to a different S3 target than the actual CI pipeline would. export S3_BUCKET="julialang-ephemeral" export S3_BUCKET_PREFIX="julia-buildkite-uploads/bin" + +# for PRs from a julia-buildkite repo we have to set the target branch manually +if [[ -n "$(echo ${BUILDKITE_PULL_REQUEST_REPO} | grep -E "julia-buildkite\.git$")" ]]; then + export DIFF_FILTER_TARGET_BRANCH="master" +fi diff --git a/pipelines/main/launch_unsigned_jobs.yml b/pipelines/main/launch_unsigned_jobs.yml index 20bfe652b..c85ea3e14 100644 --- a/pipelines/main/launch_unsigned_jobs.yml +++ b/pipelines/main/launch_unsigned_jobs.yml @@ -12,6 +12,24 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +env: + DIFF_FILTER_TARGET_BRANCH: "" + +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "TRIGGER_BUILD" + ignore: + - "*.md" + - ".*" + - "julia.spdx.json" + - "CITATION.*" + - "typos.toml" + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS + steps: - group: "Build" notify: @@ -33,35 +51,45 @@ steps: # that this script doesn't suddenly break when a new patch release of Julia # is released. version: '1.10.6' + - *diff-filter-build + - *pr-labels commands: | - ### Launch Linux build jobs. + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + ### Launch Linux build jobs # Regular: GROUP="Build" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_linux.arches \ .buildkite/pipelines/main/platforms/build_linux.yml # PowerPC (only for Julia prior to 1.12): GROUP="Build" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ .buildkite/pipelines/main/platforms/build_linux.powerpc.arches \ .buildkite/pipelines/main/platforms/build_linux.yml - ### Launch macOS build jobs: + ### Launch macOS packaging jobs GROUP="Build" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_macos.arches \ .buildkite/pipelines/main/platforms/build_macos.yml ### Launch FreeBSD build jobs: GROUP="Build" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_freebsd.arches \ .buildkite/pipelines/main/platforms/build_freebsd.yml ### Launch Windows build jobs: GROUP="Build" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_windows.arches \ .buildkite/pipelines/main/platforms/build_windows.yml @@ -79,17 +107,24 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml - # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + if [[ $${BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/trimming.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml + # buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml + fi agents: queue: "julia" os: "linux" @@ -103,37 +138,43 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | export ALLOW_FAIL="false" - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml - buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml - - ### Launch Linux test jobs. - # Regular: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # i686-linux-gnu: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.i686.arches \ - .buildkite/pipelines/main/platforms/test_linux.i686.yml - ### Launch macOS test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows test jobs: - GROUP="Test" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + if [[ $${BUILD} == 1 ]]; then + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml + buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml + ### Launch Linux test jobs. + # Regular: + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + # i686-linux-gnu: + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.i686.arches \ + .buildkite/pipelines/main/platforms/test_linux.i686.yml + ### Launch macOS test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_macos.arches \ + .buildkite/pipelines/main/platforms/test_macos.yml + ### Launch FreeBSD test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_freebsd.arches \ + .buildkite/pipelines/main/platforms/test_freebsd.yml + ### Launch windows test jobs + GROUP="Test" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_windows.arches \ + .buildkite/pipelines/main/platforms/test_windows.yml + fi agents: queue: "julia" os: "linux" @@ -144,15 +185,22 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail build jobs: + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + # Launch Linux allowed-to-fail build jobs GROUP="Allow Fail" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \ .buildkite/pipelines/main/platforms/build_linux.yml - # Launch macOS allowed-to-fail build jobs: + # Launch macOS allowed-to-fail build jobs GROUP="Allow Fail" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \ .buildkite/pipelines/main/platforms/build_macos.yml @@ -174,35 +222,41 @@ steps: # that this script doesn't suddenly break when a new patch release of Julia # is released. version: '1.10.6' + - *diff-filter-build + - *pr-labels commands: | export ALLOW_FAIL="true" - - ### Launch Linux allowed-to-fail test jobs. - # Regular: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - # PowerPC (only for Julia prior to 1.12): - GROUP="Test" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/test_linux.powerpc.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml - ### Launch macOS allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_macos.yml - ### Launch FreeBSD allowed-to-fail jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_freebsd.yml - ### Launch Windows allowed-to-fail test jobs: - GROUP="Allow Fail" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ - .buildkite/pipelines/main/platforms/test_windows.yml + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + if [[ $${BUILD} == 1 ]]; then + ### Launch Linux allowed-to-fail test jobs. + # Regular: + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + # PowerPC (only for Julia prior to 1.12): + GROUP="Test" \ + julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ + .buildkite/pipelines/main/platforms/test_linux.powerpc.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + ### Launch macOS allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_macos.yml + ### Launch FreeBSD allowed-to-fail jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_freebsd.yml + ### Launch windows allowed-to-fail test jobs + GROUP="Allow Fail" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \ + .buildkite/pipelines/main/platforms/test_windows.yml + fi agents: queue: "julia" os: "linux" diff --git a/pipelines/main/launch_upload_jobs.yml b/pipelines/main/launch_upload_jobs.yml index 93fa9cae7..1c955480f 100644 --- a/pipelines/main/launch_upload_jobs.yml +++ b/pipelines/main/launch_upload_jobs.yml @@ -1,5 +1,23 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +env: + DIFF_FILTER_TARGET_BRANCH: "" + +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "TRIGGER_BUILD" + ignore: + - "*.md" + - ".*" + - "julia.spdx.json" + - "CITATION.*" + - "typos.toml" + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS + steps: - group: "Upload" steps: @@ -18,45 +36,51 @@ steps: # that this script doesn't suddenly break when a new patch release of Julia # is released. version: '1.10.6' + - *diff-filter-build + - *pr-labels commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET export ALLOW_FAIL="false" - - ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done - ### Linux: - # Regular Linux upload jobs: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - # PowerPC Linux upload jobs, which we only run for Julia prior to 1.12: - GROUP="Upload" \ - julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ - .buildkite/pipelines/main/platforms/upload_linux.powerpc.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml - ### macOS: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_macos.arches \ - .buildkite/pipelines/main/platforms/upload_macos.yml - ### FreeBSD: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_freebsd.arches \ - .buildkite/pipelines/main/platforms/upload_freebsd.yml - ### Windows: - GROUP="Upload" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_windows.arches \ - .buildkite/pipelines/main/platforms/upload_windows.yml - - for OS in linux macos freebsd windows; do \ - GROUP="Upload" \ + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + if [[ $${BUILD} == 1 ]]; then + ##### Launch `upload_*` jobs to store tarballs into S3 once tests are done + ### Linux: + # Regular Linux upload jobs: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_linux.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + # PowerPC Linux upload jobs, which we only run for Julia prior to 1.12: + GROUP="Upload" \ + julia .buildkite/pipelines/main/platforms/launch_powerpc.jl \ + .buildkite/pipelines/main/platforms/upload_linux.powerpc.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + ### macOS: + GROUP="Upload" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_$${OS}.arches \ - .buildkite/pipelines/main/misc/upload_buildkite_results.yml - done + .buildkite/pipelines/main/platforms/upload_macos.arches \ + .buildkite/pipelines/main/platforms/upload_macos.yml + ### FreeBSD: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_freebsd.arches \ + .buildkite/pipelines/main/platforms/upload_freebsd.yml + ### Windows: + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_windows.arches \ + .buildkite/pipelines/main/platforms/upload_windows.yml + + for OS in linux macos freebsd windows; do \ + GROUP="Upload" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_$${OS}.arches \ + .buildkite/pipelines/main/misc/upload_buildkite_results.yml + done + fi # Launch doctest deploy job buildkite-agent pipeline upload .buildkite/pipelines/main/misc/deploy_docs.yml @@ -77,17 +101,24 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) export ALLOW_FAIL="true" - # Launch Linux allowed-to-fail upload jobs - GROUP="Allow Fail (Upload)" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \ - .buildkite/pipelines/main/platforms/upload_linux.yml + if [[ $${BUILD} == 1 ]]; then + # Launch Linux allowed-to-fail upload jobs + GROUP="Allow Fail (Upload)" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/main/platforms/upload_linux.soft_fail.arches \ + .buildkite/pipelines/main/platforms/upload_linux.yml + fi # Don't share this with buildkite's env display unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET diff --git a/pipelines/main/platforms/build_freebsd.yml b/pipelines/main/platforms/build_freebsd.yml index df1c7a62a..849404a91 100644 --- a/pipelines/main/platforms/build_freebsd.yml +++ b/pipelines/main/platforms/build_freebsd.yml @@ -9,7 +9,10 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/build_julia.sh" + commands: | + if [[ ${BUILD?} == 1 ]]; then + bash .buildkite/utilities/build_julia.sh + fi agents: queue: "julia" os: "freebsd" diff --git a/pipelines/main/platforms/build_linux.yml b/pipelines/main/platforms/build_linux.yml index f869da0cf..5c10c7955 100644 --- a/pipelines/main/platforms/build_linux.yml +++ b/pipelines/main/platforms/build_linux.yml @@ -21,7 +21,11 @@ steps: - "/cache/repos:/cache/repos" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/build_julia.sh" + commands: | + # always build on x86_64-linux-gnu so that we can at least build docs + if [[ ${BUILD?} == 1 || "${TRIPLET?}" == "x86_64-linux-gnu" ]]; then + bash .buildkite/utilities/build_julia.sh + fi agents: queue: "julia" # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing diff --git a/pipelines/main/platforms/build_macos.yml b/pipelines/main/platforms/build_macos.yml index 4c4037153..eaa3cd056 100644 --- a/pipelines/main/platforms/build_macos.yml +++ b/pipelines/main/platforms/build_macos.yml @@ -9,7 +9,10 @@ steps: repo_url: "https://github.com/JuliaCI/julia-buildkite" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} - commands: "bash .buildkite/utilities/build_julia.sh" + commands: | + if [[ ${BUILD?} == 1 ]]; then + bash .buildkite/utilities/build_julia.sh + fi agents: queue: "julia" os: "macos" diff --git a/pipelines/main/platforms/build_windows.yml b/pipelines/main/platforms/build_windows.yml index 63839b23c..f66d2099a 100644 --- a/pipelines/main/platforms/build_windows.yml +++ b/pipelines/main/platforms/build_windows.yml @@ -10,7 +10,6 @@ steps: - docker#v3.13.0: image: "juliapackaging/package-windows-${DOCKER_ARCH?}:${DOCKER_TAG?}" always-pull: true - command: ["bash", ".buildkite/utilities/build_julia.sh"] propagate-environment: true volumes: # Mount buildkite-agent as well @@ -22,6 +21,7 @@ steps: - "JULIA_CPU_THREADS" # Have to include this for `buildkite-agent` to work: - "BUILDKITE_AGENT_ACCESS_TOKEN" + command: "IF ${BUILD?} == 1 bash .buildkite/utilities/build_julia.sh" timeout_in_minutes: ${TIMEOUT?} soft_fail: ${ALLOW_FAIL?} agents: diff --git a/pipelines/scheduled/launch_unsigned_jobs.yml b/pipelines/scheduled/launch_unsigned_jobs.yml index e4de50f4f..d6e9d1e62 100644 --- a/pipelines/scheduled/launch_unsigned_jobs.yml +++ b/pipelines/scheduled/launch_unsigned_jobs.yml @@ -12,6 +12,24 @@ # and only need to touch the webui configuration when we need to alter # something about the privileged steps. +env: + DIFF_FILTER_TARGET_BRANCH: "" + +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "TRIGGER_BUILD" + ignore: + - "*.md" + - ".*" + - "julia.spdx.json" + - "CITATION.*" + - "typos.toml" + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS + steps: - group: "Source Build" steps: @@ -20,9 +38,15 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) GROUP="Source Build" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/scheduled/platforms/build_linux.schedule.arches \ .buildkite/pipelines/main/platforms/build_linux.yml @@ -37,11 +61,16 @@ steps: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" commands: | - GROUP="Source Tests (Allow Fail)" \ - ALLOW_FAIL="true" \ - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \ - .buildkite/pipelines/main/platforms/test_linux.yml + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + if [[ $${BUILD} == 1 ]]; then + GROUP="Source Tests (Allow Fail)" \ + ALLOW_FAIL="true" \ + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/test_linux.schedule.arches \ + .buildkite/pipelines/main/platforms/test_linux.yml + fi agents: queue: "julia" os: "linux" @@ -52,19 +81,27 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) GROUP="no_GPL" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/scheduled/platforms/build_linux.no_gpl.arches \ .buildkite/pipelines/main/platforms/build_linux.yml GROUP="no_GPL" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/scheduled/platforms/build_macos.no_gpl.arches \ .buildkite/pipelines/main/platforms/build_macos.yml GROUP="no_GPL" \ ALLOW_FAIL="false" \ + BUILD="$${BUILD}" \ bash .buildkite/utilities/arches_pipeline_upload.sh \ .buildkite/pipelines/scheduled/platforms/build_windows.no_gpl.arches \ .buildkite/pipelines/main/platforms/build_windows.yml diff --git a/pipelines/scheduled/launch_upload_jobs.yml b/pipelines/scheduled/launch_upload_jobs.yml index fc30d2344..9c405c3e3 100644 --- a/pipelines/scheduled/launch_upload_jobs.yml +++ b/pipelines/scheduled/launch_upload_jobs.yml @@ -1,5 +1,23 @@ # This file launches upload jobs that wait upon previous jobs, then upload their artifacts to S3 +env: + DIFF_FILTER_TARGET_BRANCH: "" + +common: + - diff-filter-build_plugin: &diff-filter-build + https://github.com/fatteneder/diff-filter-buildkite-plugin#main: + name: "TRIGGER_BUILD" + ignore: + - "*.md" + - ".*" + - "julia.spdx.json" + - "CITATION.*" + - "typos.toml" + target_branch: "${DIFF_FILTER_TARGET_BRANCH?}" + - pr-labels_plugin: &pr-labels + sv-oss/github-pr-labels#v0.0.2: + publish-env-var: PULL_REQUEST_LABELS + steps: - group: "Upload (no GPL)" steps: @@ -8,20 +26,26 @@ steps: - JuliaCI/external-buildkite#v1: version: "./.buildkite-external-version" repo_url: "https://github.com/JuliaCI/julia-buildkite" + - *diff-filter-build + - *pr-labels commands: | # Explicitly pass along the cryptic token to child pipelines export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET - - # Launch `upload_*` jobs to store tarballs into S3 once tests are done - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml - bash .buildkite/utilities/arches_pipeline_upload.sh \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches \ - .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml + FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep -E "\bci-force-build\b")" && echo 1 || echo 0) + MERGED=$(test "$${BUILDKITE_PULL_REQUEST}" = "false" && echo 1 || echo 0) + BUILD=$(($${TRIGGER_BUILD} || $${FORCE} || $${MERGED})) + if [[ $${BUILD} == 1 ]]; then + # Launch `upload_*` jobs to store tarballs into S3 once tests are done + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.arches \ + .buildkite/pipelines/scheduled/platforms/upload_linux.no_gpl.yml + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.arches \ + .buildkite/pipelines/scheduled/platforms/upload_macos.no_gpl.yml + bash .buildkite/utilities/arches_pipeline_upload.sh \ + .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.arches \ + .buildkite/pipelines/scheduled/platforms/upload_windows.no_gpl.yml + fi # Don't share this with buildkite's env display unset BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET