ci: Add workflows for pyspark images (#108)

stackabletech · Oct 16, 2024 · dc7549d · dc7549d
1 parent 7dd08ba
commit dc7549d
Show file tree

Hide file tree

Showing 4 changed files with 178 additions and 4 deletions.
diff --git a/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml b/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml
@@ -0,0 +1,89 @@
+---
+name: Build and publish jupyter-pyspark-with-alibi-detect
+
+env:
+  IMAGE_NAME: jupyter-pyspark-with-alibi-detect
+  IMAGE_VERSION: python-3.9
+  REGISTRY_PATH: stackable
+  DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-jupyter"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - demos/signal-processing/Dockerfile-jupyter
+      - demos/signal-processing/requirements.txt
+      - .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml
+
+jobs:
+  build:
+    name: Build/Publish ${{ matrix.runner.arch }} Image
+    permissions:
+      id-token: write
+    runs-on: ${{ matrix.runner.name }}
+    strategy:
+      matrix:
+        runner:
+          - {name: "ubuntu-latest", arch: "amd64"}
+          - {name: "ubicloud-standard-8-arm", arch: "arm64"}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Build image
+        id: build
+        uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-name: ${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          container-file: ${{ env.DOCKERFILE_PATH }}
+
+      - name: Publish Container Image on docker.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+      - name: Publish Container Image on oci.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+  publish_manifests:
+    name: Build/Publish Manifest
+    needs: [build]
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Publish and Sign Image Index Manifest to docker.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+
+      - name: Publish and Sign Image Index Manifest to oci.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
diff --git a/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml b/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml
@@ -0,0 +1,89 @@
+---
+name: Build and publish pyspark-k8s-with-scikit-learn
+
+env:
+  IMAGE_NAME: pyspark-k8s-with-scikit-learn
+  IMAGE_VERSION: 3.4.0-stackable0.0.0-dev
+  REGISTRY_PATH: stackable
+  DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile
+      - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt
+      - .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml
+
+jobs:
+  build:
+    name: Build/Publish ${{ matrix.runner.arch }} Image
+    permissions:
+      id-token: write
+    runs-on: ${{ matrix.runner.name }}
+    strategy:
+      matrix:
+        runner:
+          - {name: "ubuntu-latest", arch: "amd64"}
+          - {name: "ubicloud-standard-8-arm", arch: "arm64"}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Build image
+        id: build
+        uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-name: ${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          container-file: ${{ env.DOCKERFILE_PATH }}
+
+      - name: Publish Container Image on docker.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+      - name: Publish Container Image on oci.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+  publish_manifests:
+    name: Build/Publish Manifest
+    needs: [build]
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Publish and Sign Image Index Manifest to docker.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+
+      - name: Publish and Sign Image Index Manifest to oci.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile
@@ -1,5 +1,3 @@
-# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.4.0-stackable0.0.0-dev .
-
 FROM docker.stackable.tech/stackable/pyspark-k8s:3.4.0-stackable23.7.0
 
 COPY requirements.txt .

diff --git a/demos/signal-processing/Dockerfile-jupyter b/demos/signal-processing/Dockerfile-jupyter
@@ -1,5 +1,3 @@
-# docker build -f ./Dockerfile-jupyter -t docker.stackable.tech/demos/jupyter-pyspark-with-alibi-detect:python-3.9 .
-
 FROM jupyter/pyspark-notebook:python-3.9
 
 COPY requirements.txt .