Merge pull request #35940 from github/repo-sync

Repo sync

.github/workflows/sync-staging-repo-files.yml

@@ -13,7 +13,8 @@ on:
       - 'src/deployments/staging/.github/**'
       - 'src/deployments/staging/Dockerfile'
       - 'src/deployments/staging/.env.example'
-      - 'src/deployments/staging/README.example.md'
+      - 'src/deployments/staging/README.staging.md'
+      - 'src/deployments/staging/ownership.yaml'
       - 'src/deployments/staging/config/**'
 
 permissions:
@@ -47,18 +48,21 @@ jobs:
         id: generate-repos
         run: |
           NUMBER_OF_REPOS=${{ steps.read-config.outputs.number_of_repos }}
+          # Since we use 0-based index e.g. docs-staging-0, we need to subtract 1
+          END=$((NUMBER_OF_REPOS - 1))
           repos=()
-          for i in $(seq 0 $NUMBER_OF_REPOS); do
+          for i in $(seq 0 $END); do
             repos+=("{\"repo\": \"github/docs-staging-$i\", \"index\": $i}")
           done
           json_repos=$(printf '%s\n' "${repos[@]}" | jq -s -c .)
           echo "repos=$json_repos" >> $GITHUB_OUTPUT
 
-      - name: Set matrix output with repo and index
+      - name: Set matrix
         id: set-matrix
         run: |
-          repos=${{ steps.generate-repos.outputs.repos }}
-          echo "matrix={\"include\": $repos}" >> $GITHUB_OUTPUT
+          echo "matrix={\"include\": $REPOS}" >> $GITHUB_OUTPUT
+        env:
+          REPOS: ${{ steps.generate-repos.outputs.repos }}
 
       - uses: ./.github/actions/slack-alert
         if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
@@ -77,17 +81,20 @@ jobs:
       - name: Checkout source repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          fetch-depth: 0
+          # Only need latest commits to sync with
+          fetch-depth: 2
 
       - name: Checkout target repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           repository: ${{ matrix.repo }}
-          token: ${{ secrets.DOCS_BOT_PAT_READPUBLICKEY }}
+          token: ${{ secrets.DOCS_BOT_PAT_WORKFLOW }}
           path: target_repo
-          fetch-depth: 0
+          fetch-depth: 2
 
       - name: Synchronize files to target repo
+        env:
+          INDEX: ${{ matrix.index }}
         run: |
           # Create necessary directories if they DNE
           mkdir -p target_repo/build-scripts
@@ -111,11 +118,14 @@ jobs:
             cp src/deployments/staging/.env.example target_repo/.env
           fi
 
-          # Conditional copy for README.md if not present
-          if [ ! -f target_repo/README.md ]; then
-            cp src/deployments/staging/README.example.md target_repo/README.md
+          # Only copy README.md for non-review servers e.g. index >= 2 
+          if [ "$INDEX" -ge 2 ]; then
+            cp src/deployments/staging/README.staging.md target_repo/README.md
           fi
 
+          # Copy ownership.yaml
+          cp src/deployments/staging/ownership.yaml target_repo/ownership.yaml
+
       - name: Install jq
         run: sudo apt-get update && sudo apt-get install -y jq
 
@@ -146,17 +156,36 @@ jobs:
           # If any files still contain {{x}}, replace them with the current index
           find target_repo -type f -exec sed -i "s/{{x}}/$INDEX/g" {} +
 
-      - name: Commit and push changes
+      - name: Commit Changes to new branch
+        id: commit_changes
         run: |
+          BRANCH_NAME=sync-staging-files-${{ github.run_id }}
           cd target_repo
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
+          git checkout -b $BRANCH_NAME
           git add .
-          # If there are changes, commit and push
           if ! git diff --cached --quiet; then
-            git commit -m "Synchronize files from source repository with index ${{ matrix.index }}"
-            git push
+            git commit -m "Synchronize files from source repository with docs-staging-${{ matrix.index }}"
+            git push origin $BRANCH_NAME
           fi
+          echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_OUTPUT
+
+      # We want to create a PR instead of committing directly in order to trigger a deployment
+      - name: Create Pull Request
+        id: create_pr
+        env:
+          GH_TOKEN: ${{ secrets.DOCS_BOT_PAT_READPUBLICKEY }}
+        run: |
+          cd target_repo
+          PR_URL=$(gh pr create \
+            --title "Sync files from docs-internal" \
+            --body "This PR synchronized the files of this repo with the source of truth files in doc-internal. The PR should automatically merge, if it doesn't please fix files in docs-internal so that the fix is applied to every docs-staging-x repo." \
+            --base main \
+            --head ${{ steps.commit_changes.outputs.BRANCH_NAME }} \
+          )
+          # Enable auto-merge on PR
+          gh pr merge $PR_URL --auto --squash
 
       - uses: ./.github/actions/slack-alert
         if: ${{ failure() && github.event_name != 'workflow_dispatch' }}

.github/workflows/update-docs-staging-x-repo.yml

@@ -18,7 +18,7 @@ on:
     types: [labeled]
 
 permissions:
-  contents: read
+  contents: write
 
 jobs:
   dispatch-sha:
@@ -72,6 +72,13 @@ jobs:
             # Get the commit SHA from the pull request head
             COMMIT_SHA="${{ github.event.pull_request.head.sha }}"
 
+            # Update the docs-staging-x branch to the latest SHA from the PR branch
+            git config user.name "github-actions[bot]"
+            git config user.email "github-actions[bot]@users.noreply.github.com"
+            git checkout docs-staging-$STAGING_NUMBER || { echo "Failed to checkout docs-staging-$STAGING_NUMBER"; exit 1; }
+            git reset --hard $COMMIT_SHA || { echo "Git reset failed"; exit 1; }
+            git push origin docs-staging-$STAGING_NUMBER --force || { echo "Git push failed"; exit 1; }
+
           else
             echo "Event type $EVENT_TYPE not supported."
             echo "should_dispatch=false" >> $GITHUB_OUTPUT

src/deployments/staging/.github/workflows/deploy-on-repo-dispatch.yml

@@ -22,51 +22,55 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
-          fetch-depth: 0 # Ensure full history for PR creation
+          fetch-depth: 2
 
-      # Extract SHA from the dispatch payload
+      # Extract SHA from the dispatch payload and set it as an output
       - name: Set SHA from Payload
         id: set_sha
-        run: echo "SHA=${{ github.event.client_payload.SHA }}" >> $GITHUB_ENV
+        run: echo "SHA=${{ github.event.client_payload.SHA }}" >> $GITHUB_OUTPUT
 
       # Update the .env file with the new SHA
       - name: Update .env File
         run: |
+          SHA=${{ steps.set_sha.outputs.SHA }}
           if grep -q "^SHA=" .env; then
             sed -i "s/^SHA=.*/SHA=${SHA}/" .env
           else
             echo "SHA=${SHA}" >> .env
+          fi
 
       - name: Commit Changes to new branch
+        id: commit_changes
         run: |
           BRANCH_NAME=update-sha-${{ github.run_id }}
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           git checkout -b $BRANCH_NAME
           git add .env
-          git commit -m "Update SHA to ${{ env.SHA }}"
-          echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV
+          git commit -m "Update SHA to ${{ steps.set_sha.outputs.SHA }}"
+          echo "BRANCH_NAME=${BRANCH_NAME}" >> $GITHUB_OUTPUT
 
       - name: Push Branch
-        run: git push origin ${{ env.BRANCH_NAME }}
+        run: git push origin ${{ steps.commit_changes.outputs.BRANCH_NAME }}
 
+      # Create a Pull Request and set the PR URL as an output
       - name: Create Pull Request
         id: create_pr
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           PR_URL=$(gh pr create \
-            --title "Update SHA to ${{ env.SHA }}" \
-            --body "This PR updates the SHA in the \`.env\` file to \`${{ env.SHA }}\`." \
+            --title "Update SHA to ${{ steps.set_sha.outputs.SHA }}" \
+            --body "This PR updates the SHA in the \`.env\` file to \`${{ steps.set_sha.outputs.SHA }}\`." \
             --base main \
-            --head ${{ env.BRANCH_NAME }} \
-            --json url \
-            --jq .url)
-          echo "PR_URL=$PR_URL" >> $GITHUB_ENV
+            --head ${{ steps.commit_changes.outputs.BRANCH_NAME }} \
+          )
+          echo "PR_URL=${PR_URL}" >> $GITHUB_OUTPUT
 
+      # Merge the Pull Request
       - name: Merge Pull Request
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          PR_NUMBER=$(gh pr view $PR_URL --json number --jq .number)
-          gh pr merge $PR_NUMBER --merge --delete-branch --auto --squash --yes
+          PR_NUMBER=$(gh pr view ${{ steps.create_pr.outputs.PR_URL }} --json number --jq .number)
+          gh pr merge $PR_NUMBER --auto --squash

src/deployments/staging/.github/workflows/moda-ci.yaml

@@ -0,0 +1,95 @@
+name: docs-staging-{{x}} Moda CI
+
+# More info on CI actions setup can be found here:
+# https://github.com/github/ops/blob/master/docs/playbooks/build-systems/moving-moda-apps-from-bp-to-actions.md
+
+on:
+  workflow_dispatch:
+  push:
+    branches-ignore:
+      - 'gh-readonly-queue/**'
+  merge_group:
+    types: [checks_requested]
+
+jobs:
+  ##########################
+  # Add DOCS_BOT_PAT_READPUBLICKEY to vault-keys
+  ##########################
+  set-vault-keys:
+    runs-on: ubuntu-latest
+    outputs:
+      modified_vault_keys: ${{ steps.modify_vault_keys.outputs.modified }}
+    steps:
+      - name: Set vault-keys output
+        id: modify_vault_keys
+        run: |
+          if [ -z "${{ vars.VAULT_KEYS }}" ]; then
+            echo "modified=DOCS_BOT_PAT_READPUBLICKEY" >> $GITHUB_OUTPUT
+          else
+            echo "modified=${{ vars.VAULT_KEYS }},DOCS_BOT_PAT_READPUBLICKEY" >> $GITHUB_OUTPUT
+          fi
+
+  #############
+  # Moda jobs
+  #############
+  moda-config-bundle:
+    name: ${{ matrix.ci_job.job }}
+    needs: set-vault-keys
+    strategy:
+      fail-fast: false
+      matrix:
+        ci_job: [{ 'job': 'docs-staging-{{x}}-moda-config-bundle' }]
+    uses: github/internal-actions/.github/workflows/moda.yml@main
+    with:
+      ci-formatted-job-name: ${{ matrix.ci_job.job }}
+      vault-keys: ${{ needs.set-vault-keys.outputs.modified_vault_keys }}
+    secrets:
+      dx-bot-token: ${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}
+      datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  #############
+  # Docker Image jobs
+  #############
+  docker-image:
+    name: ${{ matrix.ci_job.job }}
+    needs: set-vault-keys
+    strategy:
+      fail-fast: false
+      matrix:
+        ci_job: [{ 'job': 'docs-staging-{{x}}-docker-image' }]
+    uses: github/internal-actions/.github/workflows/kube.yml@main
+    with:
+      ci-formatted-job-name: ${{ matrix.ci_job.job }}
+      # Fetches the 'DOCS_BOT_PAT_READPUBLICKEY' secret from Vault to pass to the docker build environment as --secret id=DOCS_BOT_PAT_READPUBLICKEY,src=$(cat DOCS_BOT_PAT_READPUBLICKEY)
+      vault-keys: ${{ needs.set-vault-keys.outputs.modified_vault_keys }}
+      docker-build-env-secrets: 'DOCS_BOT_PAT_READPUBLICKEY'
+    secrets:
+      dx-bot-token: ${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}
+      datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
+
+  #############
+  # Docker Security jobs
+  #############
+  docker-security:
+    name: ${{ matrix.ci_job.job }}
+    needs: set-vault-keys
+    strategy:
+      fail-fast: false
+      matrix:
+        ci_job: [{ 'job': 'docs-staging-{{x}}-docker-security' }]
+    uses: github/internal-actions/.github/workflows/docker_security.yml@main
+    with:
+      ci-formatted-job-name: ${{ matrix.ci_job.job }}
+      # Fetches the 'DOCS_BOT_PAT_READPUBLICKEY' secret from Vault to pass to the docker build environment as --secret id=DOCS_BOT_PAT_READPUBLICKEY,src=$(cat DOCS_BOT_PAT_READPUBLICKEY)
+      vault-keys: ${{ needs.set-vault-keys.outputs.modified_vault_keys }}
+      docker-build-env-secrets: 'DOCS_BOT_PAT_READPUBLICKEY'
+    secrets:
+      dx-bot-token: ${{ secrets.INTERNAL_ACTIONS_DX_BOT_ACCOUNT_TOKEN }}
+      datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
+
+permissions:
+  actions: read
+  checks: read
+  contents: read
+  statuses: read
+  id-token: write

src/deployments/staging/README.md

@@ -26,7 +26,7 @@ The dedicated review servers are deployed in a similar fashion that the staging
 
 Each staging server requires its own `github/` repo in order to deploy to Moda in the form of `github/docs-staging-X` where X is the number of that staging server e.g. `github/docs-staging-0` or `github/docs-staging-1`.
 
-The URLs of the staging servers also follow this pattern, `docs-staging-x.github.net`, e.g. `docs-staging-2.github.net`
+The URLs of the staging servers also follow this pattern, `https://docs-staging-{{x}}.service.iad.github.net`, e.g. `https://docs-staging-2.service.iad.github.net`
 
 With the exception of the first 2 which are our review servers:
 
@@ -37,10 +37,13 @@ Ideally there should always be enough staging servers for each developer on the
 
 So we have 8 dedicated staging servers, `docs-staging-{2-9}`:
 
-- `docs-staging-2` -> https://docs-staging-2.github.net
-- `docs-staging-3` -> https://docs-staging-3.github.net
+- `docs-staging-2` -> https://docs-staging-2.service.iad.github.net
+- `docs-staging-3` -> https://docs-staging-3.service.iad.github.net
 - etc
-- `docs-staging-9` -> https://docs-staging-9.github.net
+- `docs-staging-9` -> https://docs-staging-9.service.iad.github.net
+
+> [!NOTE]
+> [Developer VPN](https://thehub.github.com/security/security-operations/developer-vpn-access/) access is required to view a staging server. Initial set up takes some work, but connecting to it after it's configured is rather simple.
 
 ## How do staging deploys work from docs-internal?
 
@@ -60,7 +63,7 @@ sequenceDiagram
     WF2->WF2: 1. Extracts SHA from `repository_dispatch` event <br />2. Updates `.env` in docs-staging-x with SHA value<br />3. Auto-merges the PR into docs-staging-x
     WF2->MD: Auto-merge kicks off Moda deploy
     MD->MD: Dockerfile build clones docs-internal code from SHA target set in `.env`
-    note over MD: Deployed to <br/> `docs-staging-X.github.net`
+    note over MD: Deployed to <br/> `https://docs-staging-{{x}}.service.iad.github.net`
 ```
 
 Whenever a developer pushes code to a staging branch in `docs-internal`, e.g. `docs-staging-2`, a pipeline begins with the final result being a staging server running with the latest changes from that branch. See the above diagram, or read below for a textual explanation.
@@ -81,13 +84,14 @@ The pipeline is as follows:
 
 1. The PR merge kicks off an automatic Moda deploy for the `docs-staging-X` server.
 
-1. At build time, the [Dockerfile](./Dockerfile) clones the `SHA` from `docs-internal` and builds, runs, and deploys it to https://docs-staging-X.github.net which is only accessible behind the devvpn.
+1. At build time, the [Dockerfile](./Dockerfile) clones the `SHA` from `docs-internal` and builds, runs, and deploys it to `https://docs-staging-{{x}}.service.iad.github.net` which is only accessible behind the [Developer VPN](https://thehub.github.com/security/security-operations/developer-vpn-access/).
 
 ## How do review server deploys work from docs-internal?
 
 The process is very similar to the process in the previous section for staging servers. The differences are as follows:
 
 1. Review servers live in:
+
    1. Repo [docs-staging-0](https://github.com/github/doc-staging-0) (internal) @ https://docs-review.github.com
    1. Repo [docs-staging-1](https://gthub.com/github/doc-staging-1) (external) @ https://os-docs-review.github.com