feat: add s3 publisher (#52)

* refactor(frontend): update svelte-kit * feat(publisher): add blueprints * chore(backend): increase memory limits * feat(frontend): togglable fetch data * style(frontend): npm run format * docs: update * fix(frontend): add proto if missing * chore(housekeeper): delete old jobs * refactor(frontend): rename snapshot variable * chore(frontend): add builder image build * chore(skaffold): add dev s3 secret to deploy flow * chore(frontend): add builder image * feat(publisher): implement job * chore(publisher): bump cide image with aws-cli * chore(frontend): disable when disabled * chore(housekeeper): forbid concurrent jobs * chore(chart): bump version
agrrh · Sep 14, 2024 · 4ffddfd · 4ffddfd
1 parent 49eae37
commit 4ffddfd
Show file tree

Hide file tree

Showing 25 changed files with 1,210 additions and 498 deletions.
diff --git a/.dev/.gitstore b/.dev/.gitstore
diff --git a/.github/workflows/build-frontend.yaml b/.github/workflows/build-frontend.yaml
@@ -22,8 +22,23 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Docker meta
-        id: meta
+      - name: Docker meta (builder)
+        id: meta_builder
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            agrrh/pagetron-frontend-builder
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+
+      - name: Docker meta (static)
+        id: meta_static
         uses: docker/metadata-action@v5
         with:
           images: |
@@ -49,9 +64,17 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Build and push
+      - name: Build and push (builder)
+        uses: docker/build-push-action@v5
+        with:
+          context: ./frontend
+          target: builder
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta_builder.outputs.tags }}
+
+      - name: Build and push (static)
         uses: docker/build-push-action@v5
         with:
           context: ./frontend
           push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
+          tags: ${{ steps.meta_static.outputs.tags }}
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+*.secret.yaml
diff --git a/charts/pagetron/Chart.yaml b/charts/pagetron/Chart.yaml
@@ -8,4 +8,4 @@ type: application
 
 version: 0.1.0
 
-appVersion: "0.1.9"
+appVersion: "0.2.0"
diff --git a/charts/pagetron/templates/common.ingress.yaml b/charts/pagetron/templates/common.ingress.yaml
@@ -1,3 +1,4 @@
+{{- if $.Values.publicAccess.ingress.enabled }}
 {{- with $.Values.frontend }}
 
 ---
@@ -72,3 +73,4 @@ spec:
 {{- end }}
 
 {{- end }}
+{{- end }}
diff --git a/charts/pagetron/templates/frontend.deployment.yaml b/charts/pagetron/templates/frontend.deployment.yaml
@@ -1,3 +1,4 @@
+{{- if $.Values.publicAccess.ingress.enabled }}
 {{- with $.Values.frontend }}
 
 ---
@@ -59,3 +60,4 @@ spec:
       {{- end }}
 
 {{- end }}
+{{- end }}
diff --git a/charts/pagetron/templates/frontend.service.yaml b/charts/pagetron/templates/frontend.service.yaml
@@ -1,3 +1,4 @@
+{{- if $.Values.publicAccess.ingress.enabled }}
 {{- with $.Values.frontend }}
 
 ---
@@ -24,3 +25,4 @@ spec:
     component: frontend
 
 {{- end }}
+{{- end }}
diff --git a/charts/pagetron/templates/housekeeper.cronjob.yaml b/charts/pagetron/templates/housekeeper.cronjob.yaml
@@ -13,8 +13,11 @@ metadata:
 
 spec:
   schedule: {{ .schedule | quote }}
+  concurrencyPolicy: Forbid
+
   jobTemplate:
     spec:
+      ttlSecondsAfterFinished: 3600
       template:
         spec:
           restartPolicy: OnFailure

diff --git a/charts/pagetron/templates/publisher.configmap.yaml b/charts/pagetron/templates/publisher.configmap.yaml
@@ -0,0 +1,49 @@
+{{- with $.Values.publisher }}
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+
+metadata:
+  name: pagetron-publisher-job
+
+data:
+  build.sh: |
+    #!/bin/bash
+
+    set -e
+    set -x
+
+    echo "# Starting"
+
+    echo "# Building website"
+
+    export PATH="/app/node_modules/.bin:${PATH}"
+    npm run build
+
+    mv build /data/build
+
+    echo "# Done"
+
+  publish.sh: |
+    #!/bin/ash
+
+    set -e
+    set -x
+
+    echo "# Starting"
+
+    echo "# Publishing website"
+
+    cd /data/build
+
+    aws s3 \
+      --endpoint-url=${AWS_ENDPOINT_URL_S3} \
+      sync \
+        ./ \
+        s3://${S3_BUCKET_NAME}/
+
+    echo "# Done"
+
+{{- end }}
diff --git a/charts/pagetron/templates/publisher.cronjob.yaml b/charts/pagetron/templates/publisher.cronjob.yaml
@@ -0,0 +1,100 @@
+{{- if $.Values.publicAccess.s3.enabled }}
+{{- with $.Values.publisher }}
+
+---
+
+apiVersion: batch/v1
+kind: CronJob
+
+metadata:
+  name: pagetron-publisher
+  labels:
+    app: pagetron
+    component: publisher
+
+spec:
+  schedule: {{ .schedule | quote }}
+  concurrencyPolicy: Replace
+
+  jobTemplate:
+    spec:
+      ttlSecondsAfterFinished: 3600
+      template:
+        spec:
+          restartPolicy: OnFailure
+          initContainers:
+            - name: builder
+              image: "{{ .buildImage.repository }}:{{ .buildImage.tag | default $.Chart.AppVersion }}"
+              imagePullPolicy: IfNotPresent
+
+              env:
+                - name: TZ
+                  value: {{ $.Values.timezone | quote }}
+
+                - name: VITE_API_URL
+                  value: http://pagetron-backend:80
+                - name: VITE_BUILD_SNAPSHOT
+                  value: "1"
+
+              command: ["/bin/bash"]
+              args:
+                - /job/build.sh
+
+              volumeMounts:
+                - name: job
+                  mountPath: /job
+                - name: data
+                  mountPath: /data
+
+          containers:
+            - name: publisher
+              image: "{{ .image.repository }}:{{ .image.tag }}"
+              imagePullPolicy: IfNotPresent
+
+              env:
+                - name: TZ
+                  value: {{ $.Values.timezone | quote }}
+
+                - name: AWS_ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .s3.secretName | quote }}
+                      key: AWS_ACCESS_KEY_ID
+                - name: AWS_SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .s3.secretName | quote }}
+                      key: AWS_SECRET_ACCESS_KEY
+                - name: AWS_DEFAULT_REGION
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .s3.secretName | quote }}
+                      key: AWS_DEFAULT_REGION
+                - name: AWS_ENDPOINT_URL_S3
+                  valueFrom:
+                    secretKeyRef:
+                      name: {{ .s3.secretName | quote }}
+                      key: AWS_ENDPOINT_URL_S3
+
+                - name: S3_BUCKET_NAME
+                  value: {{ .s3.bucketName | quote }}
+
+              command: ["/bin/ash"]
+              args:
+                - /job/publish.sh
+
+              volumeMounts:
+                - name: job
+                  mountPath: /job
+                - name: data
+                  mountPath: /data
+
+          volumes:
+            - name: job
+              configMap:
+                name: pagetron-publisher-job
+            - name: data
+              emptyDir: {}
+
+{{- end }}
+{{- end }}
diff --git a/charts/pagetron/values.dev.yaml b/charts/pagetron/values.dev.yaml
@@ -7,6 +7,12 @@ targets:
   - https://fastapi.tiangolo.com
   - https://prometheus.io
 
+publicAccess:
+  ingress:
+    enabled: true
+  s3:
+    enabled: true
+
 # May be a good idea to use nip.io and minikube, e.g.:
 #   publicUrl: http://pagetron.192-168-42-1.nip.io
 # (know your minikube address with `minikube ip` command)
@@ -21,3 +27,9 @@ backend:
 
 frontend:
   replicas: 1
+
+publisher:
+  schedule: "*/1 * * * *"
+
+  buildImage:
+    tag: 0.2.0-dev-1726335786
diff --git a/charts/pagetron/values.yaml b/charts/pagetron/values.yaml
@@ -10,6 +10,12 @@ targets:
   - https://fastapi.tiangolo.com
   - https://prometheus.io
 
+publicAccess:
+  ingress:
+    enabled: true
+  s3:
+    enabled: true
+
 publicUrl: "https://pagetron.example.org"
 
 # Supported classes:
@@ -104,7 +110,49 @@ backend:
   resources:
     limits:
       cpu: 500m
-      memory: 128Mi
+      memory: 256Mi
+    requests:
+      cpu: 50m
+      memory: 64Mi
+
+  nodeSelector: {}
+  tolerations: []
+
+publisher:
+  schedule: "*/5 * * * *"
+
+  s3:
+    secretName: s3-publish-credentials
+    bucketName: pagetron-status-page
+
+    # Secret example for AWS
+    # data:
+    #   AWS_ACCESS_KEY_ID: ""
+    #   AWS_SECRET_ACCESS_KEY: ""
+    #   AWS_DEFAULT_REGION: ""
+    #   AWS_ENDPOINT_URL_S3: ""
+
+    # Secret example for Yandex Cloud
+    # data:
+    #   AWS_ACCESS_KEY_ID: ""
+    #   AWS_SECRET_ACCESS_KEY: ""
+    #   AWS_DEFAULT_REGION: "ru-central1"
+    #   AWS_ENDPOINT_URL_S3: "https://storage.yandexcloud.net/"
+
+  buildImage:
+    repository: agrrh/pagetron-frontend-builder
+    pullPolicy: IfNotPresent
+    tag: ""
+
+  image:
+    repository: agrrh/cider
+    pullPolicy: IfNotPresent
+    tag: 0.7.0
+
+  resources:
+    limits:
+      cpu: 100m
+      memory: 64Mi
     requests:
       cpu: 50m
       memory: 64Mi
@@ -178,7 +226,7 @@ housekeeper:
   image:
     repository: agrrh/cider
     pullPolicy: IfNotPresent
-    tag: 0.5.0
+    tag: 0.7.0
 
   resources:
     limits:
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,4 +8,4 @@ type: application

		version: 0.1.0

		appVersion: "0.1.9"
		appVersion: "0.2.0"