feat: add new implementation versions (go, js, rust) (#349)

* feat: add new implementation versions (go) * ci: generalise adding new implementation versions * ci: automate new version addition for rust-libp2p and js-libp2p
libp2p · Mar 26, 2024 · ed111cd · ed111cd
1 parent 9fd3f85
commit ed111cd
Show file tree

Hide file tree

Showing 5 changed files with 484 additions and 186 deletions.
diff --git a/.github/workflows/add-new-impl-versions.yml b/.github/workflows/add-new-impl-versions.yml
@@ -16,56 +16,139 @@ permissions:
   actions: write
 
 jobs:
-  go:
+  add:
+    name: Add new ${{ matrix.repository_name }} version to ${{ matrix.test_name }}
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - repository_name: go-libp2p
+            implementation_name: go-libp2p
+            test_name: perf
+            implementation_update_script: |
+              sed -i "1s/$LOCAL_MAJOR_MINOR_VERSION/$REMOTE_MAJOR_MINOR_VERSION/g" go.mod
+              go mod tidy -go=$GO_MAJOR_MINOR_VERSION
+              go mod tidy
+              go get github.com/libp2p/go-libp2p@$REMOTE_VERSION
+          - repository_name: go-libp2p
+            implementation_name: go
+            test_name: transport-interop
+            implementation_update_script: |
+              sed -i "s/image_name := go-${LOCAL_MAJOR_MINOR_VERSION}/image_name := go-${REMOTE_MAJOR_MINOR_VERSION}/g" Makefile
+              sed -i "s/version := ${LOCAL_MAJOR_MINOR_VERSION#v}\..*/version := ${REMOTE_VERSION#v}/g" Makefile
+              make version.lock
+          - repository_name: rust-libp2p
+            implementation_name: rust-libp2p
+            test_name: perf
+            implementation_update_script: |
+              sed -i "s/commitSha := .*/commitSha := ${REMOTE_SHA}/g" Makefile
+          - repository_name: rust-libp2p
+            implementation_name: rust
+            test_name: transport-interop
+            implementation_update_script: |
+              sed -i "s/image_name := rust-${LOCAL_MAJOR_MINOR_VERSION}/image_name := rust-${REMOTE_MAJOR_MINOR_VERSION}/g" Makefile
+              sed -i "s/commitSha := .*/commitSha := ${REMOTE_SHA}/g" Makefile
+          - repository_name: js-libp2p
+            implementation_name: js-libp2p
+            test_name: perf
+            implementation_update_script: |
+              suffix="$(echo "${REMOTE_MAJOR_MINOR_VERSION#v}" | sed 's/\./-/g')"
+              sed -i "s/perf-js-libp2p-[^\"]*/perf-js-libp2p-$suffix/g" package.json
+              npm install libp2p@$REMOTE_VERSION
+          - repository_name: js-libp2p
+            implementation_name: js
+            test_name: transport-interop
+            implementation_update_script: |
+              sed -i "s/image_name := js-${LOCAL_MAJOR_MINOR_VERSION}\..*/image_name := js-${REMOTE_VERSION}/g" Makefile
+              npm install libp2p@$REMOTE_VERSION
     env:
-      DIR: perf/impl/go-libp2p
-      REPO: libp2p/go-libp2p
-      BRANCH: perf/go-libp2p
+      BRANCH: add/${{ matrix.test_name }}/${{ matrix.repository_name }}
     steps:
       - name: Checkout test-plans
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Configure git
         run: |
           git fetch origin $BRANCH && git checkout $BRANCH || git checkout -b $BRANCH
           git rebase $GITHUB_REF -X theirs || git rebase --abort
           git config --global user.email [email protected]
           git config --global user.name $GITHUB_ACTOR
       - id: go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: stable
       - name: Get the latest version (local)
         id: local
-        run: ls -d v* | sort -V | tail -n-1 | xargs -I{} echo "version={}" | tee -a $GITHUB_OUTPUT
-        working-directory: ${{ env.DIR }}
+        env:
+          IMPLEMENTATION_DIRECTORY: ${{ matrix.test_name }}/impl/${{ matrix.implementation_name }}
+        run: |
+          pushd $IMPLEMENTATION_DIRECTORY
+          ls -d v* | sort -V | tail -n-1 | xargs -I{} echo "version={}" | tee -a $GITHUB_OUTPUT
+          popd
       - name: Get the latest version (remote)
         id: remote
         env:
           GITHUB_TOKEN: ${{ github.token }}
-        run: gh api repos/$REPO/releases/latest --jq '.tag_name' | xargs -I{} echo "version={}" | tee -a $GITHUB_OUTPUT
+          REPOSITORY_NAME: ${{ matrix.repository_name }}
+        run: |
+          releases="$(gh api --paginate repos/libp2p/$REPOSITORY_NAME/releases)"
+          latest="$(jq 'map(select(.tag_name | startswith("libp2p-"))) | first' <<< "$releases")"
+          if [[ "$latest" == "null" ]]; then
+            latest="$(jq 'first' <<< "$releases")"
+          fi
+          tag_name=$(jq -r '.tag_name' <<< "$latest")
+          target_commitish=$(jq -r '.target_commitish' <<< "$latest")
+          echo "version=${tag_name#libp2p-}" | tee -a $GITHUB_OUTPUT
+          echo "target_commitish=${target_commitish}" | tee -a $GITHUB_OUTPUT
+      - id: mm
+        env:
+          REMOTE_VERSION: ${{ steps.remote.outputs.version }}
+          GO_VERSION: ${{ steps.go.outputs.go-version }}
+        run: |
+          echo $REMOTE_VERSION | sed 's/\.[0-9]*$//' | xargs -I{} echo "remote={}" | tee -a $GITHUB_OUTPUT
+          echo $GO_VERSION | sed 's/\.[0-9]*$//' | xargs -I{} echo "go={}" | tee -a $GITHUB_OUTPUT
       - name: Add the latest version
         if: ${{ !startsWith(steps.remote.outputs.version, steps.local.outputs.version) }}
         env:
-          LOCAL_VERSION: ${{ steps.local.outputs.version }}
+          LOCAL_MAJOR_MINOR_VERSION: ${{ steps.local.outputs.version }}
           REMOTE_VERSION: ${{ steps.remote.outputs.version }}
+          REMOTE_MAJOR_MINOR_VERSION: ${{ steps.mm.outputs.remote }}
+          REMOTE_SHA: ${{ steps.remote.outputs.sha }}
           GO_VERSION: ${{ steps.go.outputs.go-version }}
+          GO_MAJOR_MINOR_VERSION: ${{ steps.mm.outputs.go }}
           GITHUB_TOKEN: ${{ github.token }}
+          IMPLEMENTATION_DIRECTORY: ${{ matrix.test_name }}/impl/${{ matrix.implementation_name }}
+          VERSIONS_DIRECTORY: ${{ matrix.test_name == 'perf' && format('{0}/runner', matrix.test_name) || matrix.test_name }}
+          IMPLEMENTATION_NAME: ${{ matrix.implementation_name }}
+          REPOSITORY_NAME: ${{ matrix.repository_name }}
+          TEST_NAME: ${{ matrix.test_name }}
         run: |
-          majorMinorRemoteVersion=$(echo $REMOTE_VERSION | sed 's/\.[0-9]*$//')
-          majorMinorGoVersion=$(echo $GO_VERSION | sed 's/\.[0-9]*$//')
-          cp -r $LOCAL_VERSION $majorMinorRemoteVersion
-          cd $majorMinorRemoteVersion
-          sed -i "1s/$LOCAL_VERSION/$majorMinorRemoteVersion/g" go.mod
-          go mod tidy -go=$majorMinorGoVersion
-          go mod tidy
-          go get github.com/libp2p/go-libp2p@$REMOTE_VERSION
+          pushd $IMPLEMENTATION_DIRECTORY
+            cp -r $LOCAL_MAJOR_MINOR_VERSION $REMOTE_MAJOR_MINOR_VERSION
+            pushd $REMOTE_MAJOR_MINOR_VERSION
+              ${{ matrix.implementation_update_script }}
+            popd
+          popd
+
+          pushd $VERSIONS_DIRECTORY
+            versions=''
+            if [[ "$TEST_NAME" == "perf" ]]; then
+              versions="$(jq '. + map(select(.implementation == env.IMPLEMENTATION_NAME and .id == env.LOCAL_MAJOR_MINOR_VERSION) | .id = env.REMOTE_MAJOR_MINOR_VERSION)' versionsInput.json)"
+            else
+              versions="$(jq '. + map(select(.id == "\(env.IMPLEMENTATION_NAME)-\(env.LOCAL_MAJOR_MINOR_VERSION)") | .id = "\(env.IMPLEMENTATION_NAME)-\(env.REMOTE_MAJOR_MINOR_VERSION)")' versionsInput.json)"
+            fi
+            echo "$versions" > versionsInput.json
+          popd
+
           git add .
-          git commit -m "chore: add go-libp2p@$REMOTE_VERSION to $DIR"
+          git commit -m "chore: add $REPOSITORY_NAME@$REMOTE_VERSION to $TEST_NAME"
+
           git push origin $BRANCH --force
-          # create a PR if an open one doesn't exist yet
+
           if [[ $(gh pr list --state open --base $GITHUB_REF --head $BRANCH | wc -l) -eq 0 ]]; then
-            gh pr create --title "chore: add go-libp2p@$REMOTE_VERSION to $DIR" --body "This PR adds go-libp2p@$REMOTE_VERSION to $DIR" --head $BRANCH --base $GITHUB_REF
+            gh pr create --title "chore: add $REPOSITORY_NAME@$REMOTE_VERSION to $TEST_NAME" --body "This PR adds $REPOSITORY_NAME@$REMOTE_VERSION to $TEST_NAME" --head $BRANCH --base $GITHUB_REF
+          fi
+
+          if [[ "$TEST_NAME" == "perf" ]]; then
+            gh workflow run $TEST_NAME.yml --ref $BRANCH
           fi
-          gh workflow run perf.yml --ref $BRANCH
-        working-directory: ${{ env.DIR }}
diff --git a/perf/runner/src/versions.ts b/perf/runner/src/versions.ts
@@ -1,53 +1,10 @@
+import fs from 'fs';
+import path from 'path';
+
 export type Version = {
     id: string,
     implementation: "go-libp2p" | "js-libp2p" | "nim-libp2p" | "rust-libp2p" | "zig-libp2p" | "https" | "quic-go",
     transportStacks: string[],
 }
 
-export const versions: Array<Version> = [
-    {
-        id: "v0.34",
-        implementation: "quic-go",
-        transportStacks: ["quic-v1"]
-    },
-    {
-        id: "v0.53",
-        implementation: "rust-libp2p",
-        transportStacks: ["tcp", "quic-v1"]
-    },
-    {
-        id: "v0.1",
-        implementation: "https",
-        transportStacks: ["tcp"]
-    },
-    {
-        id: "v0.27",
-        implementation: "go-libp2p",
-        transportStacks: ["tcp", "quic-v1"]
-    },
-    {
-        id: "v0.28",
-        implementation: "go-libp2p",
-        transportStacks: ["tcp", "quic-v1"]
-    },
-    {
-        id: "v0.29",
-        implementation: "go-libp2p",
-        transportStacks: ["tcp", "quic-v1"]
-    },
-    {
-        id: "v0.31",
-        implementation: "go-libp2p",
-        transportStacks: ["tcp", "quic-v1"]
-    },
-    {
-        id: "v0.46",
-        implementation: "js-libp2p",
-        transportStacks: ["tcp"]
-    },
-    {
-        id: "v1.0",
-        implementation: "js-libp2p",
-        transportStacks: ["tcp"]
-    },
-]
+export const versions: Array<Version> = JSON.parse(fs.readFileSync(path.join(__dirname, '../versionsInput.json'), 'utf8'));
diff --git a/perf/runner/versionsInput.json b/perf/runner/versionsInput.json
@@ -0,0 +1,70 @@
+[
+  {
+    "id": "v0.34",
+    "implementation": "quic-go",
+    "transportStacks": [
+      "quic-v1"
+    ]
+  },
+  {
+    "id": "v0.53",
+    "implementation": "rust-libp2p",
+    "transportStacks": [
+      "tcp",
+      "quic-v1"
+    ]
+  },
+  {
+    "id": "v0.1",
+    "implementation": "https",
+    "transportStacks": [
+      "tcp"
+    ]
+  },
+  {
+    "id": "v0.27",
+    "implementation": "go-libp2p",
+    "transportStacks": [
+      "tcp",
+      "quic-v1"
+    ]
+  },
+  {
+    "id": "v0.28",
+    "implementation": "go-libp2p",
+    "transportStacks": [
+      "tcp",
+      "quic-v1"
+    ]
+  },
+  {
+    "id": "v0.29",
+    "implementation": "go-libp2p",
+    "transportStacks": [
+      "tcp",
+      "quic-v1"
+    ]
+  },
+  {
+    "id": "v0.31",
+    "implementation": "go-libp2p",
+    "transportStacks": [
+      "tcp",
+      "quic-v1"
+    ]
+  },
+  {
+    "id": "v0.46",
+    "implementation": "js-libp2p",
+    "transportStacks": [
+      "tcp"
+    ]
+  },
+  {
+    "id": "v1.0",
+    "implementation": "js-libp2p",
+    "transportStacks": [
+      "tcp"
+    ]
+  }
+]