Get pipeline secrets from Vault rather than Drone (#3542)

* Get pipeline secrets from Vault rather than Drone * Move secrets to util
grafana · Apr 13, 2023 · 8054546 · 8054546
1 parent 54970a2
commit 8054546
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 60 deletions.
diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet
@@ -6,4 +6,4 @@ local pipelines = import './pipelines.jsonnet';
 (import 'pipelines/crosscompile.jsonnet') +
 (import 'pipelines/publish.jsonnet') +
 (import 'pipelines/test_packages.jsonnet') +
-(import 'secrets.jsonnet')
+(import 'util/secrets.jsonnet').asList
diff --git a/.drone/drone.yml b/.drone/drone.yml
@@ -39,9 +39,9 @@ steps:
     ./build-image
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
   image: docker
   name: Build
   volumes:
@@ -98,9 +98,9 @@ steps:
   - docker push grafana/agent-build-image:$IMAGE_TAG
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
   image: docker:windowsservercore-1809
   name: Build
   volumes:
@@ -901,9 +901,9 @@ steps:
   - docker buildx rm multiarch-agent-agent-${DRONE_COMMIT_SHA}
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
     GCR_CREDS:
       from_secret: gcr_admin
   image: grafana/agent-build-image:0.24.0
@@ -945,9 +945,9 @@ steps:
   - docker buildx rm multiarch-agent-agentctl-${DRONE_COMMIT_SHA}
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
     GCR_CREDS:
       from_secret: gcr_admin
   image: grafana/agent-build-image:0.24.0
@@ -989,9 +989,9 @@ steps:
   - docker buildx rm multiarch-agent-agent-operator-${DRONE_COMMIT_SHA}
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
     GCR_CREDS:
       from_secret: gcr_admin
   image: grafana/agent-build-image:0.24.0
@@ -1033,9 +1033,9 @@ steps:
   - docker buildx rm multiarch-agent-smoke-${DRONE_COMMIT_SHA}
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
     GCR_CREDS:
       from_secret: gcr_admin
   image: grafana/agent-build-image:0.24.0
@@ -1077,9 +1077,9 @@ steps:
   - docker buildx rm multiarch-agent-crow-${DRONE_COMMIT_SHA}
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
     GCR_CREDS:
       from_secret: gcr_admin
   image: grafana/agent-build-image:0.24.0
@@ -1108,9 +1108,9 @@ steps:
   - '& "C:/Program Files/git/bin/bash.exe" ./tools/ci/docker-containers-windows agent'
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
   image: grafana/agent-build-image:0.24.0-windows
   name: Build containers
   volumes:
@@ -1137,9 +1137,9 @@ steps:
   - '& "C:/Program Files/git/bin/bash.exe" ./tools/ci/docker-containers-windows agentctl'
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
   image: grafana/agent-build-image:0.24.0-windows
   name: Build containers
   volumes:
@@ -1228,11 +1228,11 @@ steps:
     VERSION=${DRONE_TAG} RELEASE_DOC_TAG=$(echo ${DRONE_TAG} | awk -F '.' '{print $1"."$2}') ./tools/release
   environment:
     DOCKER_LOGIN:
-      from_secret: DOCKER_LOGIN
+      from_secret: docker_login
     DOCKER_PASSWORD:
-      from_secret: DOCKER_PASSWORD
+      from_secret: docker_password
     GITHUB_TOKEN:
-      from_secret: GITHUB_KEY
+      from_secret: gh_token
     GPG_PASSPHRASE:
       from_secret: gpg_passphrase
     GPG_PRIVATE_KEY:
@@ -1281,6 +1281,18 @@ volumes:
     path: /var/run/docker.sock
   name: docker
 ---
+get:
+  name: username
+  path: infra/data/ci/docker_hub
+kind: secret
+name: docker_login
+---
+get:
+  name: password
+  path: infra/data/ci/docker_hub
+kind: secret
+name: docker_password
+---
 get:
   name: .dockerconfigjson
   path: secret/data/common/gcr
@@ -1300,10 +1312,10 @@ kind: secret
 name: gh_token
 ---
 get:
-  name: public-key
+  name: passphrase
   path: infra/data/ci/packages-publish/gpg
 kind: secret
-name: gpg_public_key
+name: gpg_passphrase
 ---
 get:
   name: private-key
@@ -1312,12 +1324,12 @@ kind: secret
 name: gpg_private_key
 ---
 get:
-  name: passphrase
+  name: public-key
   path: infra/data/ci/packages-publish/gpg
 kind: secret
-name: gpg_passphrase
+name: gpg_public_key
 ---
 kind: signature
-hmac: fcb2db578d46920304124037a3ffb63024492007b274d134e3ab9810e5bb9499
+hmac: 4012653de455ff05a36c98cf059b21ca823e38cb5742a0175c04088df9210a58
 
 ...
diff --git a/.drone/pipelines/build_images.jsonnet b/.drone/pipelines/build_images.jsonnet
@@ -1,4 +1,5 @@
 local pipelines = import '../util/pipelines.jsonnet';
+local secrets = import '../util/secrets.jsonnet';
 
 local locals = {
   on_merge: {
@@ -10,8 +11,8 @@ local locals = {
     ref: ['refs/tags/build-image/v*'],
   },
   docker_environment: {
-    DOCKER_LOGIN: { from_secret: 'DOCKER_LOGIN' },
-    DOCKER_PASSWORD: { from_secret: 'DOCKER_PASSWORD' },
+    DOCKER_LOGIN: secrets.docker_login.fromSecret,
+    DOCKER_PASSWORD: secrets.docker_password.fromSecret,
   },
 };
 

diff --git a/.drone/pipelines/publish.jsonnet b/.drone/pipelines/publish.jsonnet
@@ -1,5 +1,6 @@
 local build_image = import '../util/build_image.jsonnet';
 local pipelines = import '../util/pipelines.jsonnet';
+local secrets = import '../util/secrets.jsonnet';
 
 // job_names gets the list of job names for use in depends_on.
 local job_names = function(jobs) std.map(function(job) job.name, jobs);
@@ -34,9 +35,9 @@ local linux_containers_jobs = std.map(function(container) (
         path: '/var/run/docker.sock',
       }],
       environment: {
-        DOCKER_LOGIN: { from_secret: 'DOCKER_LOGIN' },
-        DOCKER_PASSWORD: { from_secret: 'DOCKER_PASSWORD' },
-        GCR_CREDS: { from_secret: 'gcr_admin' },
+        DOCKER_LOGIN: secrets.docker_login.fromSecret,
+        DOCKER_PASSWORD: secrets.docker_password.fromSecret,
+        GCR_CREDS: secrets.gcr_admin.fromSecret,
       },
       commands: [
         'mkdir -p $HOME/.docker',
@@ -75,8 +76,8 @@ local windows_containers_jobs = std.map(function(container) (
         path: '//./pipe/docker_engine/',
       }],
       environment: {
-        DOCKER_LOGIN: { from_secret: 'DOCKER_LOGIN' },
-        DOCKER_PASSWORD: { from_secret: 'DOCKER_PASSWORD' },
+        DOCKER_LOGIN: secrets.docker_login.fromSecret,
+        DOCKER_PASSWORD: secrets.docker_password.fromSecret,
       },
       commands: [
         '& "C:/Program Files/git/bin/bash.exe" ./tools/ci/docker-containers-windows %s' % container,
@@ -132,9 +133,7 @@ linux_containers_jobs + windows_containers_jobs + [
               ]
             }
           |||,
-          github_token: {
-            from_secret: 'gh_token',
-          },
+          github_token: secrets.gh_token.fromSecret,
         },
       },
     ],
@@ -154,12 +153,12 @@ linux_containers_jobs + windows_containers_jobs + [
         path: '/var/run/docker.sock',
       }],
       environment: {
-        DOCKER_LOGIN: { from_secret: 'DOCKER_LOGIN' },
-        DOCKER_PASSWORD: { from_secret: 'DOCKER_PASSWORD' },
-        GITHUB_TOKEN: { from_secret: 'GITHUB_KEY' },
-        GPG_PRIVATE_KEY: { from_secret: 'gpg_private_key' },
-        GPG_PUBLIC_KEY: { from_secret: 'gpg_public_key' },
-        GPG_PASSPHRASE: { from_secret: 'gpg_passphrase' },
+        DOCKER_LOGIN: secrets.docker_login.fromSecret,
+        DOCKER_PASSWORD: secrets.docker_password.fromSecret,
+        GITHUB_TOKEN: secrets.gh_token.fromSecret,
+        GPG_PRIVATE_KEY: secrets.gpg_private_key.fromSecret,
+        GPG_PUBLIC_KEY: secrets.gpg_public_key.fromSecret,
+        GPG_PASSPHRASE: secrets.gpg_passphrase.fromSecret,
       },
       commands: [
         'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD',

diff --git a/.drone/secrets.jsonnet b/.drone/secrets.jsonnet
diff --git a/.drone/util/secrets.jsonnet b/.drone/util/secrets.jsonnet
@@ -0,0 +1,23 @@
+local newSecret(name) = {
+  kind: 'secret',
+  name: name,
+
+  getFrom(path, name):: self {
+    get: { path: path, name: name },
+  },
+
+  fromSecret:: local secret = self; { from_secret: secret.name },
+};
+
+{
+  dockerconfigjson: newSecret('dockerconfigjson').getFrom(path='secret/data/common/gcr', name='.dockerconfigjson'),
+  gcr_admin: newSecret('gcr_admin').getFrom(path='infra/data/ci/gcr-admin', name='.dockerconfigjson'),
+  gh_token: newSecret('gh_token').getFrom(path='infra/data/ci/github/grafanabot', name='pat'),
+  gpg_public_key: newSecret('gpg_public_key').getFrom(path='infra/data/ci/packages-publish/gpg', name='public-key'),
+  gpg_private_key: newSecret('gpg_private_key').getFrom(path='infra/data/ci/packages-publish/gpg', name='private-key'),
+  gpg_passphrase: newSecret('gpg_passphrase').getFrom(path='infra/data/ci/packages-publish/gpg', name='passphrase'),
+  docker_login: newSecret('docker_login').getFrom(path='infra/data/ci/docker_hub', name='username'),
+  docker_password: newSecret('docker_password').getFrom(path='infra/data/ci/docker_hub', name='password'),
+
+  asList:: [self[k] for k in std.sort(std.objectFields(self))],
+}