From f6161ea7ad456ba2b89d98739f866f04a93992cc Mon Sep 17 00:00:00 2001
From: Ray Douglass <ray@raydouglass.com>
Date: Tue, 14 Jan 2025 15:35:21 -0500
Subject: [PATCH 1/5] Separate building RAPIDS & cuVS images

---
 .../{build-image.yml => build-cuvs-image.yml} |  38 ------
 .github/workflows/build-rapids-image.yml      | 110 ++++++++++++++++++
 .../workflows/build-test-publish-images.yml   |  62 ++++++++--
 ci/create-multiarch-manifest.sh               |  59 ++++++----
 4 files changed, 200 insertions(+), 69 deletions(-)
 rename .github/workflows/{build-image.yml => build-cuvs-image.yml} (76%)
 create mode 100644 .github/workflows/build-rapids-image.yml

diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-cuvs-image.yml
similarity index 76%
rename from .github/workflows/build-image.yml
rename to .github/workflows/build-cuvs-image.yml
index 7b7e62b1..7dd4ac6b 100644
--- a/.github/workflows/build-image.yml
+++ b/.github/workflows/build-cuvs-image.yml
@@ -24,12 +24,6 @@ on:
       RAPIDS_VER:
         required: true
         type: string
-      BASE_TAG:
-        required: true
-        type: string
-      NOTEBOOKS_TAG:
-        required: true
-        type: string
       CUVS_BENCH_TAG:
         required: true
         type: string
@@ -88,38 +82,6 @@ jobs:
         with:
           driver: docker
           endpoint: builders
-      - name: Build base image
-        uses: docker/build-push-action@v6
-        with:
-          context: context
-          file: Dockerfile
-          target: base
-          push: true
-          pull: true
-          build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_DISTRO=${{ inputs.LINUX_DISTRO }}
-            LINUX_DISTRO_VER=${{ inputs.LINUX_DISTRO_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
-            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-          tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
-      - name: Build notebooks image
-        uses: docker/build-push-action@v6
-        with:
-          context: context
-          file: Dockerfile
-          target: notebooks
-          push: true
-          pull: true
-          build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_DISTRO=${{ inputs.LINUX_DISTRO }}
-            LINUX_DISTRO_VER=${{ inputs.LINUX_DISTRO_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
-            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-          tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
       - name: Build cuVS Benchmarks GPU image
         uses: docker/build-push-action@v6
         with:
diff --git a/.github/workflows/build-rapids-image.yml b/.github/workflows/build-rapids-image.yml
new file mode 100644
index 00000000..9b5fb2f9
--- /dev/null
+++ b/.github/workflows/build-rapids-image.yml
@@ -0,0 +1,110 @@
+name: Build and push image variant
+
+on:
+  workflow_call:
+    inputs:
+      ARCHES:
+        required: true
+        type: string
+      CUDA_VER:
+        required: true
+        type: string
+      LINUX_DISTRO:
+        required: true
+        type: string
+      LINUX_DISTRO_VER:
+        required: true
+        type: string
+      LINUX_VER:
+        required: true
+        type: string
+      PYTHON_VER:
+        required: true
+        type: string
+      RAPIDS_VER:
+        required: true
+        type: string
+      BASE_TAG:
+        required: true
+        type: string
+      NOTEBOOKS_TAG:
+        required: true
+        type: string
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        ARCH: ${{ fromJSON(inputs.ARCHES) }}
+        CUDA_VER: ["${{ inputs.CUDA_VER }}"]
+        LINUX_VER: ["${{ inputs.LINUX_VER }}"]
+        PYTHON_VER: ["${{ inputs.PYTHON_VER }}"]
+        RAPIDS_VER: ["${{ inputs.RAPIDS_VER }}"]
+      fail-fast: false
+    runs-on: "linux-${{ matrix.ARCH }}-cpu4"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install gha-tools
+        run: |
+          mkdir -p /tmp/gha-tools
+          curl -s -L 'https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz' | tar -xz -C /tmp/gha-tools
+          echo "/tmp/gha-tools" >> "${GITHUB_PATH}"
+      - name: Clean up condarc for release builds
+        run: |
+          GIT_DESCRIBE_TAG="$(git describe --tags --abbrev=0)"
+          GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v'
+          if [[ ! $GIT_DESCRIBE_TAG =~ [a-z] ]]; then
+            rapids-logger "Most recent tag is for release, adding `rapidsai` channel and removing `rapidsai-nightly` and `dask/label/dev` channels."
+            sed -i 's|rapidsai-nightly|rapidsai|;\|dask/label/dev|d' context/condarc
+          else
+            rapids-logger "Most recent tag is an alpha. Build will use nightly channels."
+          fi
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Set up Docker Context for Buildx
+        id: buildx-context
+        run: |
+          docker context create builders
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+          endpoint: builders
+      - name: Build base image
+        uses: docker/build-push-action@v6
+        with:
+          context: context
+          file: Dockerfile
+          target: base
+          push: true
+          pull: true
+          build-args: |
+            CUDA_VER=${{ inputs.CUDA_VER }}
+            LINUX_DISTRO=${{ inputs.LINUX_DISTRO }}
+            LINUX_DISTRO_VER=${{ inputs.LINUX_DISTRO_VER }}
+            LINUX_VER=${{ inputs.LINUX_VER }}
+            PYTHON_VER=${{ inputs.PYTHON_VER }}
+            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+          tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }}
+      - name: Build notebooks image
+        uses: docker/build-push-action@v6
+        with:
+          context: context
+          file: Dockerfile
+          target: notebooks
+          push: true
+          pull: true
+          build-args: |
+            CUDA_VER=${{ inputs.CUDA_VER }}
+            LINUX_DISTRO=${{ inputs.LINUX_DISTRO }}
+            LINUX_DISTRO_VER=${{ inputs.LINUX_DISTRO_VER }}
+            LINUX_VER=${{ inputs.LINUX_VER }}
+            PYTHON_VER=${{ inputs.PYTHON_VER }}
+            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+          tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }}
diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml
index 428bdbf0..872b320c 100644
--- a/.github/workflows/build-test-publish-images.yml
+++ b/.github/workflows/build-test-publish-images.yml
@@ -36,8 +36,10 @@ jobs:
     needs:
       - checks
       - compute-matrix
-      - build
-      - build-multiarch-manifest
+      - build-rapids
+      - build-rapids-multiarch-manifest
+      - build-cuvs
+      - build-cuvs-multiarch-manifest
       - test
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02
@@ -137,13 +139,13 @@ jobs:
           export TEST_MATRIX
 
           echo "TEST_MATRIX=$(yq -n -o json 'env(TEST_MATRIX)' | jq -c '{include: .}')" | tee --append "${GITHUB_OUTPUT}"
-  build:
+  build-rapids:
     needs: [checks, compute-matrix]
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
       fail-fast: false
     secrets: inherit
-    uses: ./.github/workflows/build-image.yml
+    uses: ./.github/workflows/build-rapids-image.yml
     with:
       ARCHES: ${{ toJSON(matrix.ARCHES) }}
       CUDA_VER: ${{ matrix.CUDA_VER }}
@@ -152,7 +154,6 @@ jobs:
       LINUX_VER: ${{ matrix.LINUX_VER }}
       PYTHON_VER: ${{ matrix.PYTHON_VER }}
       RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
-      BUILD_CUVS_BENCH_CPU_IMAGE: ${{ matrix.BUILD_CUVS_BENCH_CPU_IMAGE }}
       BASE_TAG:
         "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\
         ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}\
@@ -167,6 +168,22 @@ jobs:
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
         cuda${{ matrix.CUDA_TAG }}-\
         py${{ matrix.PYTHON_VER }}"
+  build-cuvs:
+    needs: [checks, compute-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      fail-fast: false
+    secrets: inherit
+    uses: ./.github/workflows/build-cuvs-image.yml
+    with:
+      ARCHES: ${{ toJSON(matrix.ARCHES) }}
+      CUDA_VER: ${{ matrix.CUDA_VER }}
+      LINUX_DISTRO: ${{ matrix.LINUX_DISTRO }}
+      LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }}
+      LINUX_VER: ${{ matrix.LINUX_VER }}
+      PYTHON_VER: ${{ matrix.PYTHON_VER }}
+      RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+      BUILD_CUVS_BENCH_CPU_IMAGE: ${{ matrix.BUILD_CUVS_BENCH_CPU_IMAGE }}
       CUVS_BENCH_TAG:
         "rapidsai/${{ needs.compute-matrix.outputs.CUVS_BENCH_IMAGE_REPO }}:\
         ${{ needs.compute-matrix.outputs.CUVS_BENCH_TAG_PREFIX }}\
@@ -187,8 +204,8 @@ jobs:
         ${{ needs.compute-matrix.outputs.RAPIDS_VER }}\
         ${{ needs.compute-matrix.outputs.ALPHA_TAG }}-\
         py${{ matrix.PYTHON_VER }}"
-  build-multiarch-manifest:
-    needs: [build, compute-matrix]
+  build-rapids-multiarch-manifest:
+    needs: [build-rapids, compute-matrix]
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
       fail-fast: false
@@ -206,7 +223,6 @@ jobs:
       - name: Create multiarch manifest
         shell: bash
         env:
-          CUVS_BENCH_CPU_IMAGE_BUILT: ${{ matrix.BUILD_CUVS_BENCH_CPU_IMAGE }}
           BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}
           BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }}
           RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
@@ -215,6 +231,34 @@ jobs:
           PYTHON_VER: ${{ matrix.PYTHON_VER }}
           NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }}
           NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }}
+          GPUCIBOT_DOCKERHUB_USER: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+          ARCHES: ${{ toJSON(matrix.ARCHES) }}
+        run: ci/create-multiarch-manifest.sh
+  build-cuvs-multiarch-manifest:
+    needs: [build-cuvs, compute-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      fail-fast: false
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Create multiarch manifest
+        shell: bash
+        env:
+          RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }}
+          ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }}
+          CUDA_TAG: ${{ matrix.CUDA_TAG }}
+          PYTHON_VER: ${{ matrix.PYTHON_VER }}
+          CUVS_BENCH_CPU_IMAGE_BUILT: ${{ matrix.BUILD_CUVS_BENCH_CPU_IMAGE }}
           CUVS_BENCH_IMAGE_REPO: ${{ needs.compute-matrix.outputs.CUVS_BENCH_IMAGE_REPO }}
           CUVS_BENCH_TAG_PREFIX: ${{ needs.compute-matrix.outputs.CUVS_BENCH_TAG_PREFIX }}
           CUVS_BENCH_DATASETS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.CUVS_BENCH_DATASETS_IMAGE_REPO }}
@@ -226,7 +270,7 @@ jobs:
           ARCHES: ${{ toJSON(matrix.ARCHES) }}
         run: ci/create-multiarch-manifest.sh
   test:
-    needs: [compute-matrix, build]
+    needs: [compute-matrix, build-rapids]
     if: inputs.run_tests
     strategy:
       matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
diff --git a/ci/create-multiarch-manifest.sh b/ci/create-multiarch-manifest.sh
index 2ccde452..1656828b 100755
--- a/ci/create-multiarch-manifest.sh
+++ b/ci/create-multiarch-manifest.sh
@@ -2,6 +2,13 @@
 
 set -eEuo pipefail
 
+if [ -n "$BASE_IMAGE_REPO" ] || [ -n "$NOTEBOOKS_IMAGE_REPO" ]; then
+    RAPIDS_MANIFEST=true
+fi
+if [ -n "$CUVS_BENCH_IMAGE_REPO" ] || [ -n "$CUVS_BENCH_DATASETS_IMAGE_REPO" ] || [ -n "$CUVS_BENCH_CPU_IMAGE_REPO" ]; then
+    CUVS_MANIFEST=true
+fi
+
 # Authenticate and retrieve DockerHub token
 HUB_TOKEN=$(
 curl -s -H "Content-Type: application/json" \
@@ -49,38 +56,46 @@ for arch in $(echo "${ARCHES}" | jq .[] -r); do
     full_cuvs_bench_datasets_tag="${cuvs_bench_datasets_tag}-${arch}"
     full_cuvs_bench_cpu_tag="${cuvs_bench_cpu_tag}-${arch}"
 
-    check_tag_exists "$BASE_IMAGE_REPO" "$full_base_tag"
-    base_source_tags+=("${org}/${BASE_IMAGE_REPO}:$full_base_tag")
+    if [ "$RAPIDS_MANIFEST" == "true" ]; then
+        check_tag_exists "$BASE_IMAGE_REPO" "$full_base_tag"
+        base_source_tags+=("${org}/${BASE_IMAGE_REPO}:$full_base_tag")
 
-    check_tag_exists "$NOTEBOOKS_IMAGE_REPO" "$full_notebooks_tag"
-    notebooks_source_tags+=("${org}/${NOTEBOOKS_IMAGE_REPO}:$full_notebooks_tag")
+        check_tag_exists "$NOTEBOOKS_IMAGE_REPO" "$full_notebooks_tag"
+        notebooks_source_tags+=("${org}/${NOTEBOOKS_IMAGE_REPO}:$full_notebooks_tag")
+    fi
 
-    check_tag_exists "$CUVS_BENCH_IMAGE_REPO" "$full_cuvs_bench_tag"
-    cuvs_bench_source_tags+=("${org}/${CUVS_BENCH_IMAGE_REPO}:$full_cuvs_bench_tag")
+    if [ "$CUVS_MANIFEST" == "true" ]; then
+        check_tag_exists "$CUVS_BENCH_IMAGE_REPO" "$full_cuvs_bench_tag"
+        cuvs_bench_source_tags+=("${org}/${CUVS_BENCH_IMAGE_REPO}:$full_cuvs_bench_tag")
 
-    check_tag_exists "$CUVS_BENCH_DATASETS_IMAGE_REPO" "$full_cuvs_bench_datasets_tag"
-    cuvs_bench_datasets_source_tags+=("${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:$full_cuvs_bench_datasets_tag")
+        check_tag_exists "$CUVS_BENCH_DATASETS_IMAGE_REPO" "$full_cuvs_bench_datasets_tag"
+        cuvs_bench_datasets_source_tags+=("${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:$full_cuvs_bench_datasets_tag")
 
-    if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
-        check_tag_exists "$CUVS_BENCH_CPU_IMAGE_REPO" "$full_cuvs_bench_cpu_tag"
-        cuvs_bench_cpu_source_tags+=("${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:$full_cuvs_bench_cpu_tag")
+        if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
+            check_tag_exists "$CUVS_BENCH_CPU_IMAGE_REPO" "$full_cuvs_bench_cpu_tag"
+            cuvs_bench_cpu_source_tags+=("${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:$full_cuvs_bench_cpu_tag")
+        fi
     fi
 done
 
 # Create and push Docker multi-arch manifests
-docker manifest create "${org}/${BASE_IMAGE_REPO}:${base_tag}" "${base_source_tags[@]}"
-docker manifest push "${org}/${BASE_IMAGE_REPO}:${base_tag}"
+if [ "$RAPIDS_MANIFEST" == "true" ]; then
+    docker manifest create "${org}/${BASE_IMAGE_REPO}:${base_tag}" "${base_source_tags[@]}"
+    docker manifest push "${org}/${BASE_IMAGE_REPO}:${base_tag}"
 
-docker manifest create "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}" "${notebooks_source_tags[@]}"
-docker manifest push "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}"
+    docker manifest create "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}" "${notebooks_source_tags[@]}"
+    docker manifest push "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}"
+fi
 
-docker manifest create "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}" "${cuvs_bench_source_tags[@]}"
-docker manifest push "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}"
+if [ "$CUVS_MANIFEST" == "true" ]; then
+    docker manifest create "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}" "${cuvs_bench_source_tags[@]}"
+    docker manifest push "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}"
 
-docker manifest create "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}" "${cuvs_bench_datasets_source_tags[@]}"
-docker manifest push "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}"
+    docker manifest create "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}" "${cuvs_bench_datasets_source_tags[@]}"
+    docker manifest push "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}"
 
-if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
-    docker manifest create "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}" "${cuvs_bench_cpu_source_tags[@]}"
-    docker manifest push "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}"
+    if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
+        docker manifest create "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}" "${cuvs_bench_cpu_source_tags[@]}"
+        docker manifest push "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}"
+    fi
 fi

From c8a4e49964d19e5563231dd5fcbe5933ba100ece Mon Sep 17 00:00:00 2001
From: Ray Douglass <ray@raydouglass.com>
Date: Tue, 14 Jan 2025 15:56:28 -0500
Subject: [PATCH 2/5] Use rapids-mamba-retry

---
 Dockerfile                | 13 ++++++++++---
 cuvs-bench/cpu/Dockerfile | 16 +++++++++++-----
 cuvs-bench/gpu/Dockerfile | 12 +++++++++---
 3 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 8e82f784..290e8a11 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -42,6 +42,13 @@ ARG RAPIDS_VER
 
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
+RUN <<EOF
+apt-get update
+apt-get install -y wget
+wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
+apt-get purge -y --auto-remove wget
+rm -rf /var/lib/apt/lists/*
+EOF
 RUN useradd -rm -d /home/rapids -s /bin/bash -g conda -u 1001 rapids
 
 USER rapids
@@ -57,7 +64,7 @@ conda config --show-sources
 conda list --show-channel-urls
 
 # Install RAPIDS
-mamba install -y -n base \
+rapids-mamba-retry install -y -n base \
     "rapids=${RAPIDS_VER}.*" \
     "python=${PYTHON_VER}.*" \
     "cuda-version=${CUDA_VER%.*}.*" \
@@ -90,12 +97,12 @@ COPY --from=dependencies --chown=rapids /test_notebooks_dependencies.yaml test_n
 COPY --from=dependencies --chown=rapids /notebooks /home/rapids/notebooks
 
 RUN <<EOF
-mamba env update -n base -f test_notebooks_dependencies.yaml
+rapids-mamba-retry env update -n base -f test_notebooks_dependencies.yaml
 conda clean -afy
 EOF
 
 RUN <<EOF
-mamba install -y -n base \
+rapids-mamba-retry install -y -n base \
     "jupyterlab=4" \
     dask-labextension \
     jupyterlab-nvdashboard
diff --git a/cuvs-bench/cpu/Dockerfile b/cuvs-bench/cpu/Dockerfile
index c3365793..75661039 100644
--- a/cuvs-bench/cpu/Dockerfile
+++ b/cuvs-bench/cpu/Dockerfile
@@ -21,16 +21,22 @@ echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/bash.bas
 EOF
 
 # we need perl temporarily for the remaining benchmark perl scripts
-RUN apt-get install perl -y
+RUN <<EOF
+apt-get install -y \
+  perl
+  wget
+wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
+rm -rf /var/lib/apt/lists/*
+EOF
 
 # update everything before other environment changes, to ensure mixing
 # an older conda with newer packages still works well
 # ref: https://github.com/rapidsai/ci-imgs/issues/185
 RUN <<EOF
-mamba update --all -y -n base
-mamba install -y -n base "python=${PYTHON_VER}"
-mamba update --all -y -n base
-mamba install -y -n base \
+rapids-mamba-retry update --all -y -n base
+rapids-mamba-retry install -y -n base "python=${PYTHON_VER}"
+rapids-mamba-retry update --all -y -n base
+rapids-mamba-retry install -y -n base \
   "cuvs-bench-cpu=${RAPIDS_VER}.*" \
   "python=${PYTHON_VER}"
 conda clean -afy
diff --git a/cuvs-bench/gpu/Dockerfile b/cuvs-bench/gpu/Dockerfile
index 552518a9..26e34a14 100644
--- a/cuvs-bench/gpu/Dockerfile
+++ b/cuvs-bench/gpu/Dockerfile
@@ -24,11 +24,17 @@ echo ". /opt/conda/etc/profile.d/conda.sh; conda activate base" >> /etc/bash.bas
 EOF
 
 # we need perl temporarily for the remaining benchmark perl scripts
-RUN apt-get install perl -y
+RUN <<EOF
+apt-get install -y \
+  perl
+  wget
+wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
+rm -rf /var/lib/apt/lists/*
+EOF
 
 RUN <<EOF
-mamba update --all -y -n base
-mamba install -y -n base \
+rapids-mamba-retry update --all -y -n base
+rapids-mamba-retry install -y -n base \
     "cuvs-bench=${RAPIDS_VER}.*" \
     "cuda-version=${CUDA_VER%.*}.*"
 conda clean -afy

From 5adab549b1b115f60237831aa25acd21fccfd6e5 Mon Sep 17 00:00:00 2001
From: Ray Douglass <ray@raydouglass.com>
Date: Tue, 14 Jan 2025 15:57:21 -0500
Subject: [PATCH 3/5] Temporarily disable building cuvs-bench-datasets

---
 .github/workflows/build-cuvs-image.yml | 28 +++++++++++++-------------
 ci/create-multiarch-manifest.sh        |  4 ++--
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/build-cuvs-image.yml b/.github/workflows/build-cuvs-image.yml
index 7dd4ac6b..71f977fb 100644
--- a/.github/workflows/build-cuvs-image.yml
+++ b/.github/workflows/build-cuvs-image.yml
@@ -96,20 +96,20 @@ jobs:
             PYTHON_VER=${{ inputs.PYTHON_VER }}
             RAPIDS_VER=${{ inputs.RAPIDS_VER }}
           tags: ${{ inputs.CUVS_BENCH_TAG }}-${{ matrix.ARCH }}
-      - name: Build cuVS Benchmarks GPU with datasets image
-        uses: docker/build-push-action@v6
-        with:
-          context: context
-          file: cuvs-bench/gpu/Dockerfile
-          target: cuvs-bench-datasets
-          push: true
-          pull: true
-          build-args: |
-            CUDA_VER=${{ inputs.CUDA_VER }}
-            LINUX_VER=${{ inputs.LINUX_VER }}
-            PYTHON_VER=${{ inputs.PYTHON_VER }}
-            RAPIDS_VER=${{ inputs.RAPIDS_VER }}
-          tags: ${{ inputs.CUVS_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }}
+      # - name: Build cuVS Benchmarks GPU with datasets image
+      #   uses: docker/build-push-action@v6
+      #   with:
+      #     context: context
+      #     file: cuvs-bench/gpu/Dockerfile
+      #     target: cuvs-bench-datasets
+      #     push: true
+      #     pull: true
+      #     build-args: |
+      #       CUDA_VER=${{ inputs.CUDA_VER }}
+      #       LINUX_VER=${{ inputs.LINUX_VER }}
+      #       PYTHON_VER=${{ inputs.PYTHON_VER }}
+      #       RAPIDS_VER=${{ inputs.RAPIDS_VER }}
+      #     tags: ${{ inputs.CUVS_BENCH_DATASETS_TAG }}-${{ matrix.ARCH }}
       - name: Build cuVS Benchmarks CPU image
         if: inputs.BUILD_CUVS_BENCH_CPU_IMAGE
         uses: docker/build-push-action@v6
diff --git a/ci/create-multiarch-manifest.sh b/ci/create-multiarch-manifest.sh
index 1656828b..47779c1b 100755
--- a/ci/create-multiarch-manifest.sh
+++ b/ci/create-multiarch-manifest.sh
@@ -68,8 +68,8 @@ for arch in $(echo "${ARCHES}" | jq .[] -r); do
         check_tag_exists "$CUVS_BENCH_IMAGE_REPO" "$full_cuvs_bench_tag"
         cuvs_bench_source_tags+=("${org}/${CUVS_BENCH_IMAGE_REPO}:$full_cuvs_bench_tag")
 
-        check_tag_exists "$CUVS_BENCH_DATASETS_IMAGE_REPO" "$full_cuvs_bench_datasets_tag"
-        cuvs_bench_datasets_source_tags+=("${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:$full_cuvs_bench_datasets_tag")
+        # check_tag_exists "$CUVS_BENCH_DATASETS_IMAGE_REPO" "$full_cuvs_bench_datasets_tag"
+        # cuvs_bench_datasets_source_tags+=("${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:$full_cuvs_bench_datasets_tag")
 
         if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
             check_tag_exists "$CUVS_BENCH_CPU_IMAGE_REPO" "$full_cuvs_bench_cpu_tag"

From d89d9252dfc3a91286cabc0ebadcb8550acf85c2 Mon Sep 17 00:00:00 2001
From: Ray Douglass <ray@raydouglass.com>
Date: Tue, 14 Jan 2025 16:01:02 -0500
Subject: [PATCH 4/5] Fix installing wget

---
 cuvs-bench/cpu/Dockerfile | 3 ++-
 cuvs-bench/gpu/Dockerfile | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/cuvs-bench/cpu/Dockerfile b/cuvs-bench/cpu/Dockerfile
index 75661039..6afa557f 100644
--- a/cuvs-bench/cpu/Dockerfile
+++ b/cuvs-bench/cpu/Dockerfile
@@ -22,8 +22,9 @@ EOF
 
 # we need perl temporarily for the remaining benchmark perl scripts
 RUN <<EOF
+apt-get update
 apt-get install -y \
-  perl
+  perl \
   wget
 wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
 rm -rf /var/lib/apt/lists/*
diff --git a/cuvs-bench/gpu/Dockerfile b/cuvs-bench/gpu/Dockerfile
index 26e34a14..8e7ea1cf 100644
--- a/cuvs-bench/gpu/Dockerfile
+++ b/cuvs-bench/gpu/Dockerfile
@@ -25,8 +25,9 @@ EOF
 
 # we need perl temporarily for the remaining benchmark perl scripts
 RUN <<EOF
+apt-get update
 apt-get install -y \
-  perl
+  perl \
   wget
 wget https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz -O - | tar -xz -C /usr/local/bin
 rm -rf /var/lib/apt/lists/*

From 1290d8776a80ab302ecbbad9f136beee5af82382 Mon Sep 17 00:00:00 2001
From: Ray Douglass <ray@raydouglass.com>
Date: Tue, 14 Jan 2025 16:21:55 -0500
Subject: [PATCH 5/5] Split manifest scripts

---
 .../workflows/build-test-publish-images.yml   |   4 +-
 ci/common.sh                                  |  29 +++++
 ci/create-cuvs-multiarch-manifest.sh          |  45 ++++++++
 ci/create-multiarch-manifest.sh               | 101 ------------------
 ci/create-rapids-multiarch-manifest.sh        |  34 ++++++
 5 files changed, 110 insertions(+), 103 deletions(-)
 create mode 100644 ci/common.sh
 create mode 100755 ci/create-cuvs-multiarch-manifest.sh
 delete mode 100755 ci/create-multiarch-manifest.sh
 create mode 100755 ci/create-rapids-multiarch-manifest.sh

diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml
index 872b320c..2cba0705 100644
--- a/.github/workflows/build-test-publish-images.yml
+++ b/.github/workflows/build-test-publish-images.yml
@@ -234,7 +234,7 @@ jobs:
           GPUCIBOT_DOCKERHUB_USER: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
           GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
           ARCHES: ${{ toJSON(matrix.ARCHES) }}
-        run: ci/create-multiarch-manifest.sh
+        run: ci/create-rapids-multiarch-manifest.sh
   build-cuvs-multiarch-manifest:
     needs: [build-cuvs, compute-matrix]
     strategy:
@@ -268,7 +268,7 @@ jobs:
           GPUCIBOT_DOCKERHUB_USER: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
           GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
           ARCHES: ${{ toJSON(matrix.ARCHES) }}
-        run: ci/create-multiarch-manifest.sh
+        run: ci/create-cuvs-multiarch-manifest.sh
   test:
     needs: [compute-matrix, build-rapids]
     if: inputs.run_tests
diff --git a/ci/common.sh b/ci/common.sh
new file mode 100644
index 00000000..7143f066
--- /dev/null
+++ b/ci/common.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -eEuo pipefail
+
+# Authenticate and retrieve DockerHub token
+HUB_TOKEN=$(
+curl -s -H "Content-Type: application/json" \
+    -X POST \
+    -d "{\"username\": \"$GPUCIBOT_DOCKERHUB_USER\", \"password\": \"$GPUCIBOT_DOCKERHUB_TOKEN\"}" \
+    https://hub.docker.com/v2/users/login/ | jq -r .token \
+)
+echo "::add-mask::${HUB_TOKEN}"
+export HUB_TOKEN
+
+# Function to check if a Docker tag exists
+check_tag_exists() {
+    local repo="$1"
+    local tag="$2"
+    local exists
+    exists=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: JWT $HUB_TOKEN" \
+        "https://hub.docker.com/v2/repositories/${org}/${repo}/tags/${tag}/")
+
+    if [ "$exists" -ne 200 ]; then
+        echo "Error: Required image tag ${repo}:${tag} does not exist. This implies that the image was not built successfully in the build job."
+        exit 1
+    fi
+}
+
+export org="rapidsai"
diff --git a/ci/create-cuvs-multiarch-manifest.sh b/ci/create-cuvs-multiarch-manifest.sh
new file mode 100755
index 00000000..41dec178
--- /dev/null
+++ b/ci/create-cuvs-multiarch-manifest.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -eEuo pipefail
+
+common_path="$(dirname "$(realpath "$0")")/common.sh"
+# shellcheck source=common.sh
+source "$common_path"
+
+cuvs_bench_source_tags=()
+cuvs_bench_datasets_source_tags=()
+cuvs_bench_cpu_source_tags=()
+
+# Define tag arrays for different images
+cuvs_bench_tag="${CUVS_BENCH_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
+cuvs_bench_datasets_tag="${CUVS_BENCH_DATASETS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
+cuvs_bench_cpu_tag="${CUVS_BENCH_CPU_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-py${PYTHON_VER}"
+
+# Check if all source tags exist and add to source tags array
+for arch in $(echo "${ARCHES}" | jq .[] -r); do
+    full_cuvs_bench_tag="${cuvs_bench_tag}-${arch}"
+    full_cuvs_bench_datasets_tag="${cuvs_bench_datasets_tag}-${arch}"
+    full_cuvs_bench_cpu_tag="${cuvs_bench_cpu_tag}-${arch}"
+
+    check_tag_exists "$CUVS_BENCH_IMAGE_REPO" "$full_cuvs_bench_tag"
+    cuvs_bench_source_tags+=("${org}/${CUVS_BENCH_IMAGE_REPO}:$full_cuvs_bench_tag")
+
+    # check_tag_exists "$CUVS_BENCH_DATASETS_IMAGE_REPO" "$full_cuvs_bench_datasets_tag"
+    # cuvs_bench_datasets_source_tags+=("${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:$full_cuvs_bench_datasets_tag")
+
+    if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
+        check_tag_exists "$CUVS_BENCH_CPU_IMAGE_REPO" "$full_cuvs_bench_cpu_tag"
+        cuvs_bench_cpu_source_tags+=("${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:$full_cuvs_bench_cpu_tag")
+    fi
+done
+
+docker manifest create "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}" "${cuvs_bench_source_tags[@]}"
+docker manifest push "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}"
+
+# docker manifest create "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}" "${cuvs_bench_datasets_source_tags[@]}"
+# docker manifest push "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}"
+
+if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
+    docker manifest create "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}" "${cuvs_bench_cpu_source_tags[@]}"
+    docker manifest push "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}"
+fi
diff --git a/ci/create-multiarch-manifest.sh b/ci/create-multiarch-manifest.sh
deleted file mode 100755
index 47779c1b..00000000
--- a/ci/create-multiarch-manifest.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-
-set -eEuo pipefail
-
-if [ -n "$BASE_IMAGE_REPO" ] || [ -n "$NOTEBOOKS_IMAGE_REPO" ]; then
-    RAPIDS_MANIFEST=true
-fi
-if [ -n "$CUVS_BENCH_IMAGE_REPO" ] || [ -n "$CUVS_BENCH_DATASETS_IMAGE_REPO" ] || [ -n "$CUVS_BENCH_CPU_IMAGE_REPO" ]; then
-    CUVS_MANIFEST=true
-fi
-
-# Authenticate and retrieve DockerHub token
-HUB_TOKEN=$(
-curl -s -H "Content-Type: application/json" \
-    -X POST \
-    -d "{\"username\": \"$GPUCIBOT_DOCKERHUB_USER\", \"password\": \"$GPUCIBOT_DOCKERHUB_TOKEN\"}" \
-    https://hub.docker.com/v2/users/login/ | jq -r .token \
-)
-echo "::add-mask::${HUB_TOKEN}"
-
-org="rapidsai"
-
-# Initialize arrays to store source tags for each image
-base_source_tags=()
-notebooks_source_tags=()
-cuvs_bench_source_tags=()
-cuvs_bench_datasets_source_tags=()
-cuvs_bench_cpu_source_tags=()
-
-# Define tag arrays for different images
-base_tag="${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
-notebooks_tag="${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
-cuvs_bench_tag="${CUVS_BENCH_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
-cuvs_bench_datasets_tag="${CUVS_BENCH_DATASETS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
-cuvs_bench_cpu_tag="${CUVS_BENCH_CPU_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-py${PYTHON_VER}"
-
-# Function to check if a Docker tag exists
-check_tag_exists() {
-    local repo="$1"
-    local tag="$2"
-    local exists
-    exists=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: JWT $HUB_TOKEN" \
-        "https://hub.docker.com/v2/repositories/${org}/${repo}/tags/${tag}/")
-
-    if [ "$exists" -ne 200 ]; then
-        echo "Error: Required image tag ${repo}:${tag} does not exist. This implies that the image was not built successfully in the build job."
-        exit 1
-    fi
-}
-
-# Check if all source tags exist and add to source tags array
-for arch in $(echo "${ARCHES}" | jq .[] -r); do
-    full_base_tag="${base_tag}-${arch}"
-    full_notebooks_tag="${notebooks_tag}-${arch}"
-    full_cuvs_bench_tag="${cuvs_bench_tag}-${arch}"
-    full_cuvs_bench_datasets_tag="${cuvs_bench_datasets_tag}-${arch}"
-    full_cuvs_bench_cpu_tag="${cuvs_bench_cpu_tag}-${arch}"
-
-    if [ "$RAPIDS_MANIFEST" == "true" ]; then
-        check_tag_exists "$BASE_IMAGE_REPO" "$full_base_tag"
-        base_source_tags+=("${org}/${BASE_IMAGE_REPO}:$full_base_tag")
-
-        check_tag_exists "$NOTEBOOKS_IMAGE_REPO" "$full_notebooks_tag"
-        notebooks_source_tags+=("${org}/${NOTEBOOKS_IMAGE_REPO}:$full_notebooks_tag")
-    fi
-
-    if [ "$CUVS_MANIFEST" == "true" ]; then
-        check_tag_exists "$CUVS_BENCH_IMAGE_REPO" "$full_cuvs_bench_tag"
-        cuvs_bench_source_tags+=("${org}/${CUVS_BENCH_IMAGE_REPO}:$full_cuvs_bench_tag")
-
-        # check_tag_exists "$CUVS_BENCH_DATASETS_IMAGE_REPO" "$full_cuvs_bench_datasets_tag"
-        # cuvs_bench_datasets_source_tags+=("${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:$full_cuvs_bench_datasets_tag")
-
-        if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
-            check_tag_exists "$CUVS_BENCH_CPU_IMAGE_REPO" "$full_cuvs_bench_cpu_tag"
-            cuvs_bench_cpu_source_tags+=("${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:$full_cuvs_bench_cpu_tag")
-        fi
-    fi
-done
-
-# Create and push Docker multi-arch manifests
-if [ "$RAPIDS_MANIFEST" == "true" ]; then
-    docker manifest create "${org}/${BASE_IMAGE_REPO}:${base_tag}" "${base_source_tags[@]}"
-    docker manifest push "${org}/${BASE_IMAGE_REPO}:${base_tag}"
-
-    docker manifest create "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}" "${notebooks_source_tags[@]}"
-    docker manifest push "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}"
-fi
-
-if [ "$CUVS_MANIFEST" == "true" ]; then
-    docker manifest create "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}" "${cuvs_bench_source_tags[@]}"
-    docker manifest push "${org}/${CUVS_BENCH_IMAGE_REPO}:${cuvs_bench_tag}"
-
-    docker manifest create "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}" "${cuvs_bench_datasets_source_tags[@]}"
-    docker manifest push "${org}/${CUVS_BENCH_DATASETS_IMAGE_REPO}:${cuvs_bench_datasets_tag}"
-
-    if [ "$CUVS_BENCH_CPU_IMAGE_BUILT" = "true" ]; then
-        docker manifest create "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}" "${cuvs_bench_cpu_source_tags[@]}"
-        docker manifest push "${org}/${CUVS_BENCH_CPU_IMAGE_REPO}:${cuvs_bench_cpu_tag}"
-    fi
-fi
diff --git a/ci/create-rapids-multiarch-manifest.sh b/ci/create-rapids-multiarch-manifest.sh
new file mode 100755
index 00000000..75bab316
--- /dev/null
+++ b/ci/create-rapids-multiarch-manifest.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -eEuo pipefail
+
+common_path="$(dirname "$(realpath "$0")")/common.sh"
+# shellcheck source=common.sh
+source "$common_path"
+
+# Initialize arrays to store source tags for each image
+base_source_tags=()
+notebooks_source_tags=()
+
+# Define tag arrays for different images
+base_tag="${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
+notebooks_tag="${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}"
+
+# Check if all source tags exist and add to source tags array
+for arch in $(echo "${ARCHES}" | jq .[] -r); do
+    full_base_tag="${base_tag}-${arch}"
+    full_notebooks_tag="${notebooks_tag}-${arch}"
+
+    check_tag_exists "$BASE_IMAGE_REPO" "$full_base_tag"
+    base_source_tags+=("${org}/${BASE_IMAGE_REPO}:$full_base_tag")
+
+    check_tag_exists "$NOTEBOOKS_IMAGE_REPO" "$full_notebooks_tag"
+    notebooks_source_tags+=("${org}/${NOTEBOOKS_IMAGE_REPO}:$full_notebooks_tag")
+done
+
+# Create and push Docker multi-arch manifests
+docker manifest create "${org}/${BASE_IMAGE_REPO}:${base_tag}" "${base_source_tags[@]}"
+docker manifest push "${org}/${BASE_IMAGE_REPO}:${base_tag}"
+
+docker manifest create "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}" "${notebooks_source_tags[@]}"
+docker manifest push "${org}/${NOTEBOOKS_IMAGE_REPO}:${notebooks_tag}"