build_packages.yaml

name: Build packages

concurrency:
  group: build-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_dispatch:
  pull_request:
  release:
    types:
      - published

jobs:
  prepare:
    runs-on: ubuntu-latest
    outputs:
      arch: ${{ steps.arch.outputs.arch }}

    steps:
      - id: arch
        run: |
          if ${{ contains(fromJSON('["release", "workflow_dispatch"]'), github.event_name) }}; then
            echo "arch=[\"linux/amd64\", \"linux/arm64\", \"linux/arm/v7\"]" >> $GITHUB_OUTPUT
          else
            echo "arch=[\"linux/amd64\"]" >> $GITHUB_OUTPUT
          fi

  build:
    runs-on: ubuntu-latest
    needs: prepare

    strategy:
      matrix:
        golang:
          - 1.23.1
        arch: ${{fromJson(needs.prepare.outputs.arch)}}
        os:
          - debian

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - uses: docker/setup-qemu-action@v3
    - uses: docker/setup-buildx-action@v3
    - name: build
      if: matrix.os == 'debian'
      run: |
        docker run -i --rm \
        -v $(pwd):/ekuiper \
        --workdir /ekuiper \
        --env KUIPER_SOURCE='/ekuiper' \
        --platform ${{ matrix.arch }} \
        ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os }} \
        bash -euc "git config --global --add safe.directory /ekuiper && make pkg && make pkg_core && make pkg_full"
    - name: create sha file
      run: |
        cd _packages && for var in $(ls); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
    - name: Set artifact name
      id: artifact_name
      env:
        ARCH: ${{ matrix.arch }}
        REF: ${{ github.ref_name }}
      run: |
        arch=$(echo $ARCH | sed -r 's|linux/([a-z]*)(/.*)*|\1|g')
        ref=$(echo ${REF/\//-})
        artifact_name="kuiper-${ref}-${{ matrix.os }}-${arch}"
        echo "artifact_name=$artifact_name" >> $GITHUB_OUTPUT
    - uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.artifact_name.outputs.artifact_name }}
        path: _packages/

  build-on-mac:
    runs-on: macos-latest

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - uses: actions/setup-go@v5
      with:
        go-version: '1.23.1'
    - name: prepare
      run: |
        brew install zip unzip gnu-sed zmq
          echo "/usr/local/bin:$PATH" >> ~/.bashrc
    - name: build
      run: |
        make pkg
        cd _packages && for var in $(ls); do openssl dgst -sha256 $var | awk '{print $2}' > $var.sha256; done && cd -
    - name: Set artifact name
      id: artifact_name
      env:
        REF: ${{ github.ref_name }}
      run: |
        arch=$(echo $ARCH | sed -r 's|linux/([a-z]*)(/.*)*|\1|g')
        ref=$(echo ${REF/\//-})
        artifact_name="kuiper-${ref}-darwin-${arch}"
        echo "artifact_name=$artifact_name" >> $GITHUB_OUTPUT
    - uses: actions/upload-artifact@v4
      with:
        name: ${{ steps.artifact_name.outputs.artifact_name }}
        path: _packages/

  build-docker-images:
    runs-on: ubuntu-latest

    strategy:
        fail-fast: false
        matrix:
            suffix:
            - ""
            - "-alpine"
            - "-dev"
            - "-slim"
            - "-slim-python"
            - "-full"
            golang:
              - 1.23.1

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - uses: docker/setup-buildx-action@v3
    - uses: docker/setup-qemu-action@v3
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - name: Build single platform image
      if: endsWith( matrix.suffix, 'python') == false
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64
        push: false
        load: true
        tags: docker.io/lfedge/ekuiper
        build-args: GO_VERSION=${{ matrix.golang }}
        file: deploy/docker/Dockerfile${{ matrix.suffix }}
    - name: Test docker image
      run: |
        docker run -d --name ekuiper docker.io/lfedge/ekuiper
        ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ekuiper)
        sleep 5
        if ! curl ${ip_address}:9081  >/dev/null 2>&1; then
            echo "docker image failed"
            docker logs ekuiper
            exit 1
        fi
    - uses: docker/metadata-action@v5
      id: meta
      with:
        images: docker.io/lfedge/ekuiper
        flavor: |
          latest=${{ github.event_name == 'release' && matrix.suffix == '-alpine'  && !github.event.release.prerelease}}
          suffix=${{ matrix.suffix }}
        tags: |
          type=ref,event=branch
          type=ref,event=pr
          type=ref,event=tag
          type=semver,pattern={{version}}
          type=semver,pattern={{major}}.{{minor}}
    - uses: docker/login-action@v2
      if: github.event_name == 'release'
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}
    - name: Build multi platform image
      if: contains(fromJSON('["release", "workflow_dispatch"]'), github.event_name)
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64,linux/arm64,linux/arm/v7
        push: ${{ github.event_name == 'release' }}
        tags: ${{ steps.meta.outputs.tags }}
        build-args: GO_VERSION=${{ matrix.golang }}
        labels: ${{ steps.meta.outputs.labels }}
        file: deploy/docker/Dockerfile${{ matrix.suffix }}

  build-plugins:
    runs-on: ubuntu-latest
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        plugin:
          - sinks/image
          - sinks/influx
          - sinks/influx2
          - sinks/zmq
          - sinks/kafka
          - sinks/sql
          - sources/random
          - sources/zmq
          - sources/sql
          - sources/video
          - sources/kafka
          - functions/accumulateWordCount
          - functions/countPlusOne
          - functions/echo
          - functions/image
          - functions/geohash
          - functions/labelImage
          - functions/tfLite
        arch: ${{fromJson(needs.prepare.outputs.arch)}}
        os:
          - [ debian, "slim" ]
          - [ alpine, "alpine" ]
        golang:
          - 1.23.1
        exclude:
          - arch: linux/arm/v7
          - os: [ alpine,"alpine" ]
            plugin: functions/labelImage
          - os: [ alpine,"alpine" ]
            plugin: functions/tfLite

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: docker/setup-buildx-action@v3
      - uses: docker/setup-qemu-action@v3
        with:
          image: tonistiigi/binfmt:latest
          platforms: all
      - name: build plugins
        run: |
          docker run -i --rm \
              -v $(pwd):/ekuiper \
              --workdir /ekuiper \
              --platform ${{ matrix.arch }} \
              ghcr.io/lf-edge/ekuiper/base:${{ matrix.golang }}-${{ matrix.os[0] }} \
              bash -euc "git config --global --add safe.directory /ekuiper && make ${{ matrix.plugin }}"
      - name: Build ekuiper image
        uses: docker/build-push-action@v6
        if: matrix.arch == 'linux/amd64'
        with:
          context: .
          platforms: linux/amd64
          push: false
          load: true
          tags: docker.io/lfedge/ekuiper-${{ matrix.os[0] }}
          file: deploy/docker/Dockerfile-${{ matrix.os[1] }}
      - name: test docker and plugins
        if: matrix.arch == 'linux/amd64'
        env:
          PLUGIN: ${{ matrix.plugin }}
        run: |
          set -e -x -u
          plugin_type=$(echo ${PLUGIN%%/*})
          plugin_name=$(echo ${PLUGIN##*/})
          
          container_id=$(docker run -u 0 -d -v $(pwd)/_plugins:/var/plugins docker.io/lfedge/ekuiper-${{ matrix.os[0] }})
          ip_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $container_id)
          os=$(docker exec -i ${container_id} sh -c "sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g'" )
          sleep 5
          if ! curl ${ip_address}:9081  >/dev/null 2>&1; then echo "docker image failed"; exit 1; fi
          if [ "${plugin_name}" = "image" ]; then
              curl \
              ${ip_address}:9081/plugins/${plugin_type} \
              -X POST \
              -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"resize\",\"thumbnail\"]}"
          elif [ "${plugin_name}" = "geohash" ]; then
              curl \
              ${ip_address}:9081/plugins/${plugin_type} \
              -X POST \
              -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\", \"functions\": [\"geohashEncode\", \"geohashEncodeInt\", \"geohashDecode\", \"geohashDecodeInt\", \"geohashBoundingBox\", \"geohashBoundingBoxInt\", \"geohashNeighbor\", \"geohashNeighborInt\", \"geohashNeighbors\", \"geohashNeighborsInt\"]}"
          else
              curl \
              ${ip_address}:9081/plugins/${plugin_type} \
              -X POST \
              -d "{\"name\":\"${plugin_name}\", \"file\":\"file:///var/plugins/${os}/${plugin_type}/${plugin_name}_amd64.zip\"}"
          fi
          docker logs ${container_id}
          [ $plugin_name = $(curl ${ip_address}:9081/plugins/${plugin_type}/${plugin_name} | jq '.name'| sed 's/\"//g' ) ] || exit 1
      - name: Set artifact name
        id: artifact_name
        env:
          PLUGIN: ${{ matrix.plugin }}
          ARCH: ${{ matrix.arch }}
          REF: ${{ github.ref_name }}
        run: |
          plugin_type=$(echo ${PLUGIN%%/*})
          plugin_name=$(echo ${PLUGIN##*/})
          arch=$(echo $ARCH | sed -r 's|linux/([a-z]*)(/.*)*|\1|g')
          ref=$(echo ${REF/\//-})

          artifact_name="plugin-${ref}-${{ matrix.os[0] }}-${plugin_type}-${plugin_name}-${arch}"
          echo "artifact_name=$artifact_name" >> $GITHUB_OUTPUT
      - uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.artifact_name.outputs.artifact_name }}
          path: _plugins/

  build-kubernetes-tool:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - uses: docker/setup-buildx-action@v3
    - uses: docker/setup-qemu-action@v3
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - name: Build single platform image
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64
        push: false
        load: true
        tags: docker.io/lfedge/ekuiper-kubernetes-tool
        file: deploy/docker/Dockerfile-kubernetes-tool
    - name: Test docker image
      run: |
        docker run -d --name kuiper-kubernetes-tool docker.io/lfedge/ekuiper-kubernetes-tool
        sleep 5
        if [[ "$(docker logs kuiper-kubernetes-tool)" != *"Kuiper kubernetes tool is started successfully!"*  ]]; then exit 1; fi
    - uses: docker/metadata-action@v5
      id: meta
      with:
        images: docker.io/lfedge/ekuiper-kubernetes-tool
        flavor: |
          latest=${{ github.event_name == 'release' && !github.event.release.prerelease}}
        tags: |
          type=ref,event=branch
          type=ref,event=pr
          type=ref,event=tag
          type=semver,pattern={{version}}
          type=semver,pattern={{major}}.{{minor}}
    - uses: docker/login-action@v2
      if: github.event_name == 'release'
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}
    - name: Update version for setup.py
      if: github.event_name == 'release'
      run: sed -i -r "s|([ \t]*version=).*|\1'${GITHUB_REF#refs/tags/}',|1" sdk/python/setup.py
    - name: Build multi platform image
      uses: docker/build-push-action@v6
      with:
        context: .
        platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386
        push: ${{ github.event_name == 'release' }}
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        file: deploy/docker/Dockerfile-kubernetes-tool

  release:
    runs-on: ubuntu-latest

    needs:
    - build
    - build-on-mac
    - build-plugins

    steps:
    - uses: actions/checkout@v4
    - uses: actions/download-artifact@v4
      with:
        pattern: kuiper-*
        path: _packages
        merge-multiple: true
    - uses: actions/download-artifact@v4
      with:
        pattern: plugin-*
        path: _plugins
        merge-multiple: true
    - name: check packages
      run: |
        cd _packages && for var in $( ls |grep -v sha256); do
          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
        done
    - uses: Rory-Z/upload-release-asset@v1
      if: github.event_name == 'release'
      with:
        repo: ekuiper
        path: "_packages/kuiper*"
        token: ${{ secrets.GITHUB_TOKEN }}
    - name: upload packages to s3
      if: github.event_name == 'release'
      run: |
        version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws configure set default.region us-west-2
        aws s3 rm --quiet --recursive s3://packages.emqx/kuiper/$version
        aws s3 cp --quiet --recursive ./_packages s3://packages.emqx/kuiper/$version
        aws s3 cp --quiet --recursive ./_plugins s3://packages.emqx/kuiper-plugins/$version
        aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/kuiper/$version/*,/kuiper-plugins/$version/*"
    - name: update ekuiper.org
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        curl -fvs -w %{http_code} \
          --insecure \
          -H "Content-Type: application/json" \
          -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
          -X POST \
          -d "{\"repo\":\"lf-edge/ekuiper\", \"tag\": \"${{ github.ref_name }}\" }" \
          ${{ secrets.EMQX_IO_RELEASE_API }}
    - uses: geekyeggo/delete-artifact@v5
      with:
        name: "*"