[experimental] Testing development workflows

.github/workflows/dev-workflow-test.yaml

@@ -0,0 +1,200 @@
+name: dev-test
+on:
+  push:
+    branches:
+      - "!release"
+      - "!alpha"
+      - "!beta"
+      - "!rc"
+  workflow_call:
+  pull_request:
+    paths:
+      - scripts/install_*.sh
+      - Makefile
+      - docker/Dockerfile
+      - docker/Dockerfile.*
+      - test/docker/docker-compose.yml
+      - test/docker/Dockerfile
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Prerequisites (make, go, docker)
+        run: |
+          sudo sh scripts/install_go.sh
+          echo "/usr/local/go/bin" >> $GITHUB_PATH # Go installation
+          echo "/home/runner/go/bin" >> $GITHUB_PATH # Default GOPATH
+
+      - name: Host only workflows
+        run: |
+          make clean
+          make deps
+          make clean
+          make generate
+          make clean
+          make lint
+
+      - name: Host only (requires FoundationDB clients)
+        run: |
+          make clean
+          make bins
+
+      - name: Containerized with test on the host, or IDE (requries FoundationDB clients)
+        run: |
+          make clean
+          make run
+          make local_test
+          make local_test #containers and test results should be cached
+          make clean
+          make run_full
+
+      - name: Fully containerized workflows
+        run: |
+          make clean
+          make test
+          make test #containers and test results should be cached
+          make clean
+          make logs >/dev/null 2>&1
+
+  macos:
+    runs-on: macos-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      # This brings working docker, but everything is veeeeeery slow
+      #- uses: docker-practice/actions-setup-docker@master
+
+      # Requires to enable Oracle VM in UI
+      #- run: |
+      #    brew install docker docker-machine
+      #    brew install --cask virtualbox
+      #    docker-machine create --driver virtualbox default
+      #    docker-machine env default
+      #    eval "$(docker-machine env default)"
+      #    docker run hello-world
+
+      - name: Prerequisites (make, go, docker)
+        run: |
+          brew uninstall [email protected]
+          brew install colima coreutils docker docker-compose [email protected]
+          colima start
+          echo "/Users/runner/go/bin" >> $GITHUB_PATH # Default GOPATH
+          mkdir -p ~/.docker/cli-plugins
+          ln -sfn $(brew --prefix)/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
+
+      - name: Host only workflows
+        run: |
+          make clean
+          make deps
+          make clean
+          make generate
+          make clean
+          make lint
+
+      - name: Host only (requires FoundationDB clients)
+        run: |
+          make clean
+          make bins
+
+      - name: Containerized with test on the host, or IDE (requires FoundationDB clients)
+        run: |
+          make clean
+          make run
+          make local_test
+          make local_test #containers and test results should be cached
+          make clean
+          make run_full
+
+      - name: Fully containerized workflows
+        run: |
+          make clean
+          make test
+          make test #containers and test results should be cached
+          make clean
+          make logs >/dev/nul 2>&1
+
+  windows:
+    runs-on: windows-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      # Looks there is no way to enable linux containers in Docker on Github
+      #- run: |
+      #    DockerCli.exe -SwitchLinuxEngine
+
+      #Add-Content $env:GITHUB_PATH "/usr/local/go/bin"
+      #Add-Content $env:GITHUB_PATH "/home/runner/go/bin"
+      #Add-Content $env:GITHUB_PATH "/usr/local/bin"
+      #Add-Content $env:GITHUB_PATH "C:\Users\runneradmin\go/bin"
+
+      - name: Prerequisites (make, go, docker)
+        run: |
+          sh scripts/install_go.sh
+          echo "/usr/local/go/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "/home/runner/go/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "/usr/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          echo "C:\Users\runneradmin\go/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+          choco install protoc
+
+      - name: Host only workflows
+        run: |
+          make clean
+          make deps
+          echo "aaaa $PATH aaaa"
+          echo "bbbb $GOPATH bbbb"
+          which protoc
+          which go
+          which protoc-gen-go
+          /usr/local/bin/protoc.exe --help || true
+          /usr/local/bin/protoc --help || true
+          pwd
+          tree
+          make clean
+          make generate
+          make clean
+          make lint
+
+      - name: Host only workflows
+        run: |
+          make clean
+          make deps
+          make clean
+          make generate
+          make clean
+          make lint
+
+      - name: Host only (requires FoundationDB clients)
+        if: false
+        run: |
+          make clean
+          make bins
+
+      - name: Containerized with test on the host, or IDE (requries FoundationDB clients)
+        if: false
+        run: |
+          make clean
+          make run
+          make local_test
+          make local_test #containers and test results should be cached
+          make clean
+          make run_full
+
+      - name: Fully containerized workflows
+        if: false
+        run: |
+          make clean
+          make test
+          make test #containers and test results should be cached
+          make clean
+          make logs >/dev/nul 2>&1

.github/workflows/push-docker-image.yaml

@@ -17,6 +17,7 @@ env:
 
 jobs:
   build-and-push-image:
+    if: false
     runs-on: ubuntu-latest
     permissions:
       contents: read

.github/workflows/push-docker-local-image.yaml

@@ -17,6 +17,7 @@ env:
 
 jobs:
   build-and-push-local-image:
+    if: false
     runs-on: ubuntu-latest
     permissions:
       contents: read

Makefile

@@ -7,7 +7,7 @@ V=v1
 GEN_DIR=${API_DIR}/server/${V}
 PROTO_DIR=${API_DIR}/proto/server/${V}
 DATA_PROTO_DIR=internal
-LINT_TIMEOUT=5m
+LINT_TIMEOUT=15m
 
 # Needed to be able to build amd64 binaries on MacOS M1
 DOCKER_DIR=test/docker
@@ -97,6 +97,7 @@ ${DATA_PROTO_DIR}/%.pb.go: ${DATA_PROTO_DIR}/%.proto
 
 coverdir:
 	mkdir -p /tmp/tigris_coverdata && chmod a+w /tmp/tigris_coverdata; rm -f /tmp/tigris_coverdata/*
+	ls -la /tmp/tigris_coverdata
 
 generate: ${GEN_DIR}/api.pb.go ${GEN_DIR}/api.pb.gw.go ${GEN_DIR}/health.pb.go ${GEN_DIR}/health.pb.gw.go ${GEN_DIR}/admin.pb.go ${GEN_DIR}/admin.pb.gw.go ${DATA_PROTO_DIR}/data.pb.go
 
@@ -120,3 +121,4 @@ dump_integration_coverage:
 	pkill -SIGTERM -f "/server/service" --exact
 	sleep 5
 	/usr/local/go/bin/go tool covdata textfmt -i=/tmp/tigris_coverdata/ -o coverage1.out
+

scripts/install_build_deps.sh

@@ -46,6 +46,8 @@ case "${OS}-${ARCH}" in
   BINARIES="apt-get curl go"
   FDB_SHA=471f6bf4a7af40abc69027aa0d4f452ee83715a43a555008303ca255f6bd6db1
   ;;
+"MINGW"*)
+  ;;
 *)
   echo "Unsupported architecture ${ARCH} or operating system ${OS}."
   exit 1
@@ -80,6 +82,9 @@ case "${OS}" in
     ;;
   esac
   ;;
+"MINGW"*)
+  PROTO_PKG=protoc-$PROTO_VERSION-win64.zip
+  ;;
 *)
   echo "No supported proto compiler for ${ARCH} or operating system ${OS}."
   exit 1
@@ -90,9 +95,15 @@ if [ -n "$PROTO_PKG" ]; then
   DOWNLOAD_URL="$PROTO_RELEASES/download/v$PROTO_VERSION/$PROTO_PKG"
   echo "Fetching protobuf release ${DOWNLOAD_URL}"
   curl -LO "$DOWNLOAD_URL"
-  sudo unzip "$PROTO_PKG" -d "/usr/local/"
-  sudo chmod +x "/usr/local/bin/protoc"
-  sudo chmod -R 755 "/usr/local/include/"
+  if [[ "$OS" == "MINGW"* ]]; then
+    unzip "$PROTO_PKG" -d "/usr/local/"
+    chmod +x "/usr/local/bin/protoc.exe"
+    chmod -R 755 "/usr/local/include/"
+  else
+    sudo unzip "$PROTO_PKG" -d "/usr/local/"
+    sudo chmod +x "/usr/local/bin/protoc"
+    sudo chmod -R 755 "/usr/local/include/"
+  fi
   rm -f "$PROTO_PKG"
 fi
 
@@ -123,4 +134,7 @@ case "${OS}" in
   sudo dpkg -i "$FDB_PACKAGE_PATH" # provides /lib/libfdb_c.so shared library in the docker for CGO
   rm -f "$FDB_PACKAGE_PATH"
   ;;
+"MINGW"*)
+  echo "TODO: Install FoundationDB for Windows packages"
+  ;;
 esac

scripts/install_go.sh

@@ -13,12 +13,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+set -ex
 
 # This intended to be run duing Docker build only
 
 set -e
 
 VERSION=1.20
+#ARCH=$(uname -m)
+OS=$(uname -s)
 ARCH=$(dpkg --print-architecture)
 FN="go${VERSION}.linux-${ARCH}.tar.gz"
 
@@ -42,5 +45,45 @@ mkdir -p /usr/local
 tar -C /usr/local -xzf "$FN"
 rm "$FN"
 
+
+if false; then
+case "${OS}-${ARCH}" in
+"Darwin-arm64")
+  V="darwin-arm64.tar.gz"
+  SHA="49e394ab92bc6fa3df3d27298ddf3e4491f99477bee9dd4934525a526f3a391c"
+  ;;
+"Darwin-x86_64")
+  V="darwin-amd64.tar.gz"
+  SHA="7fa09a9a34cb6f794e61e9ada1d6d18796f936a2b35f22724906cad71396e590"
+  ;;
+"Linux-arm64")
+  V="linux-arm64.tar.gz"
+  SHA="99de2fe112a52ab748fb175edea64b313a0c8d51d6157dba683a6be163fd5eab"
+  ;;
+"Linux-x86_64")
+  V="linux-amd64.tar.gz"
+  SHA="74b9640724fd4e6bb0ed2a1bc44ae813a03f1e72a4c76253e2d5c015494430ba"
+  ;;
+"MINGW"*)
+  V="windows-amd64.zip"
+  SHA="b51549a9f21ee053f8a3d8e38e45b1b8b282d976f3b60f1f89b37ac54e272d31"
+  ;;
+*)
+  echo "Unsupported architecture ${ARCH} or operating system ${OS}."
+  exit 1
+  ;;
+esac
+
+echo "$SHA  $FN" | shasum -a 256 -c
+
+curl -LO "https://go.dev/dl/go${VERSION}.${V}"
+
+if [[ "$OS" == "MINGW"* ]]; then
+    unzip "go${VERSION}.${V}" -d "/usr/local/"
+else
+	tar -C /usr/local -xzf "go${VERSION}.${V}"
+fi
+fi
+
 export PATH=$PATH:/usr/local/go/bin
 

scripts/install_test_deps.sh

@@ -24,6 +24,6 @@ if [ "$(uname -s)" = "Darwin" ]; then
   if command -v brew > /dev/null 2>&1; then
     brew install shellcheck
   fi
-else
+elif [ "$(uname -s)" = "Linux" ]; then
   sudo apt-get install -y shellcheck
 fi

test/docker/docker-compose.yml

@@ -72,6 +72,8 @@ services:
       - type: bind
         source: /tmp/tigris_coverdata
         target: /tmp/tigris_coverdata
+        bind:
+          create_host_path: true
     ports:
       - "8081:8081"
     command: >