ci: refactor ci workflow

1. add ci scripts 2. add options to convertor for ci 3. fix ctr record-trace Signed-off-by: yuchen.cc <[email protected]>
containerd · Jan 2, 2024 · c5ae0b1 · c5ae0b1
1 parent 526b680
commit c5ae0b1
Show file tree

Hide file tree

Showing 26 changed files with 590 additions and 201 deletions.
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -0,0 +1,93 @@
+name: Check
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  #
+  # Linter checker
+  #
+  linters:
+    name: Linters
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        go-version: [1.19]
+
+    steps:
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - uses: actions/checkout@v3
+      - uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.50.1
+          skip-cache: true
+          args: --timeout=5m
+
+  #
+  # Project checker
+  #
+  # based on https://github.com/containerd/project-checks/blob/main/action.yml
+  project:
+    name: Project Checks
+    runs-on: ubuntu-22.04
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: src/github.com/containerd/accelerated-container-image
+          fetch-depth: 100
+
+      - name: set env
+        shell: bash
+        run: |
+          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+
+      # TODO(fuweid): remove the env GO111MODULE=off in new version of go
+      - name: install dependencies
+        shell: bash
+        env:
+          GO111MODULE: off
+        run: |
+          echo "::group:: install dependencies"
+          go get -u -v github.com/vbatts/git-validation
+          go get -u -v github.com/kunalkushwaha/ltag
+          echo "::endgroup::"
+
+      - name: DCO checker
+        shell: bash
+        working-directory: src/github.com/containerd/accelerated-container-image
+        env:
+          GITHUB_COMMIT_URL: ${{ github.event.pull_request.commits_url }}
+          DCO_VERBOSITY: "-v"
+          DCO_RANGE: ""
+        run: |
+          echo "::group:: DCO checks"
+          set -eu -o pipefail
+          if [ -z "${GITHUB_COMMIT_URL}" ]; then
+          DCO_RANGE=$(jq -r '.after + "..HEAD"' ${GITHUB_EVENT_PATH})
+          else
+          DCO_RANGE=$(curl ${GITHUB_COMMIT_URL} | jq -r '.[0].parents[0].sha + "..HEAD"')
+          fi
+
+          range=
+          [ ! -z "${DCO_RANGE}" ] && range="-range ${DCO_RANGE}"
+          git-validation ${DCO_VERBOSITY} ${range} -run DCO,short-subject,dangling-whitespace
+          echo "::endgroup::"
+
+      - name: validate file headers
+        shell: bash
+        working-directory: src/github.com/containerd/accelerated-container-image
+        run: |
+          set -eu -o pipefail
+          echo "::group:: file headers"
+          ltag -t "script/validate/template" --excludes "vendor contrib" --check -v
+          echo "::endgroup::"
diff --git a/.github/workflows/ci-basic.yml b/.github/workflows/ci-basic.yml
@@ -0,0 +1,79 @@
+name: CI | Basic
+on:
+  workflow_call:
+    inputs:
+      image-tag:
+        required: true
+        type: string
+
+jobs:
+  run-ci-basic:
+    name: Run CI | Basic
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    container:
+      image: ghcr.io/${{ github.actor }}/ci-images:${{ inputs.image-tag }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      volumes:
+        - /dev:/dev
+        - /lib/modules:/lib/modules
+        - /sys/kernel/config:/sys/kernel/config
+        - /mnt:/var/lib/containerd
+      options: --privileged
+
+    steps:
+      - name: Start OverlayBD
+        working-directory: /app
+        shell: bash
+        run: |
+          set -x
+          bash start_services.sh
+          sleep 5s
+          ps axf | grep overlaybd
+          ps axf | grep containerd
+          lsmod | grep target
+          lsmod | grep overlay
+          mount | grep cgroup
+          cat /sys/fs/cgroup/cgroup.subtree_control
+          uname -a
+          containerd -v
+          runc -v
+
+      - name: CI - run overlaybd container
+        shell: bash
+        run: |
+          /opt/overlaybd/snapshotter/ctr rpull docker.io/overlaybd/redis:6.2.1_obd
+          ctr run -d --net-host --snapshotter=overlaybd docker.io/overlaybd/redis:6.2.1_obd test
+          ctr t ls | grep test
+          ctr t kill -s 9 test && sleep 5s && ctr t ls
+          ctr c rm test && ctr c ls
+          ctr i rm docker.io/overlaybd/redis:6.2.1_obd && ctr i ls
+
+      - name: CI - obdconv
+        shell: bash
+        run: |
+          ctr i pull registry.hub.docker.com/library/redis:6.2.1
+          /opt/overlaybd/snapshotter/ctr obdconv registry.hub.docker.com/library/redis:6.2.1 registry.hub.docker.com/overlaybd/redis:6.2.1_obd_new
+          ctr i ls | grep 6.2.1_obd_new
+
+      - name: CI - record trace
+        shell: bash
+        run: |
+          /opt/overlaybd/snapshotter/ctr rpull registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv
+          echo "[ by record-trace ]"
+          /opt/overlaybd/snapshotter/ctr record-trace --runtime "io.containerd.runc.v2" --disable-network-isolation --time 15 registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv_trace
+          ctr i ls | grep 6.2.1_obdconv_trace
+          echo "[ by label ]"
+          touch /tmp/trace_file
+          ctr run -d --snapshotter=overlaybd --snapshotter-label containerd.io/snapshot/overlaybd/record-trace=yes --snapshotter-label containerd.io/snapshot/overlaybd/record-trace-path=/tmp/trace_file registry.hub.docker.com/overlaybd/redis:6.2.1_obdconv demo
+          sleep 1s
+          ls -l /tmp/ | grep trace_file.lock
+          sleep 15s
+          ctr t ls | grep demo
+          ctr t kill -s 9 demo && sleep 5s && ctr t ls
+          ctr c rm demo && ctr c ls
+          sleep 1s
+          ls -l /tmp/ | grep trace_file.ok
+          ctr i ls
diff --git a/.github/workflows/ci-build-image.yml b/.github/workflows/ci-build-image.yml
@@ -0,0 +1,58 @@
+name: CI | Build Image
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      image-tag:
+        required: true
+        type: string
+
+env:
+  GO_VERSION: "1.19"
+  OBD_VERSION: "1.0.8"
+
+jobs:
+  build-image:
+    name: Build Image
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Setup Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login ghcr.io
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Env
+        shell: bash
+        run: |
+          git fetch --tags
+          releasever=$(git tag --sort=v:refname -l v* | tail -1)
+          releasever=${releasever#v}
+          echo "RELEASE_VERSION=${releasever}" >> $GITHUB_ENV
+
+      - name: Build and Push
+        uses: docker/build-push-action@v4
+        with:
+          tags: ghcr.io/${{ github.actor }}/ci-images:${{ inputs.image-tag }}
+          push: true
+          context: .
+          platforms: linux/amd64
+          file: ci/build_image/Dockerfile
+          build-args: |
+            RELEASE_VERSION=${{ env.RELEASE_VERSION }}
+            GO_VERSION=${{ env.GO_VERSION }}
+            OBD_VERSION=${{ env.OBD_VERSION }}
diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml
@@ -0,0 +1,37 @@
+name: CI | E2E
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      image-tag:
+        required: true
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-test-image-amd64:
+    uses: ./.github/workflows/ci-build-image.yml
+    with:
+      commit-hash: ${{ inputs.commit-hash }}
+      image-tag: ${{ inputs.image-tag }}
+    secrets: inherit
+
+  run-ci-basic:
+    needs: build-test-image-amd64
+    uses: ./.github/workflows/ci-basic.yml
+    with:
+      image-tag: ${{ inputs.image-tag }}
+    secrets: inherit
+
+  run-ci-userspace-convertor:
+    needs: build-test-image-amd64
+    uses: ./.github/workflows/ci-userspace-convertor.yml
+    with:
+      commit-hash: ${{ inputs.commit-hash }}
+      image-tag: ${{ inputs.image-tag }}
+    secrets: inherit
diff --git a/.github/workflows/ci-userspace-convertor.yml b/.github/workflows/ci-userspace-convertor.yml
@@ -0,0 +1,47 @@
+name: CI | Userspace Convertor
+on:
+  workflow_call:
+    inputs:
+      commit-hash:
+        required: true
+        type: string
+      image-tag:
+        required: true
+        type: string
+
+jobs:
+  run-ci-userspace-convertor:
+    name: Run CI | Userspace Convertor
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    container:
+      image: ghcr.io/${{ github.actor }}/ci-images:${{ inputs.image-tag }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+      volumes:
+        - /dev:/dev
+        - /lib/modules:/lib/modules
+        - /sys/kernel/config:/sys/kernel/config
+        - /mnt:/var/lib/containerd
+      options: --privileged
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Start OverlayBD
+        working-directory: /app
+        shell: bash
+        run: |
+          bash start_services.sh
+          sleep 5s
+
+      - name: CI - uconv reproduce
+        working-directory: ci/uconv_reproduce
+        shell: bash
+        run: |
+          bash ci-uconv-reproduce.sh