feat(runtime): the build & test runtime (#2)

Kong · Dec 5, 2022 · f8faae2 · f8faae2
1 parent af0f357
commit f8faae2
Show file tree

Hide file tree

Showing 21 changed files with 4,466 additions and 13 deletions.
diff --git a/.env b/.env
@@ -0,0 +1,7 @@
+OPENRESTY_VERSION=1.21.4.1
+LUAROCKS_VERSION=3.9.1
+PCRE_VERSION=8.45
+KONG_NGINX_MODULE_VERSION=0.4.0
+LUA_RESTY_LMDB_VERSION=1.0.0
+LUA_RESTY_EVENTS_VERSION=0.1.3
+ATC_ROUTER_VERSION=1.0.35
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -21,8 +21,6 @@ repos:
   hooks:
   - id: forbid-crlf
   - id: remove-crlf
-  - id: forbid-tabs
-  - id: remove-tabs
 - repo: https://github.com/Yelp/detect-secrets
   rev: v1.1.0
   hooks:

diff --git a/Dockerfile b/Dockerfile
@@ -4,18 +4,19 @@ ARG DOCKER_REGISTRY=ghcr.io
 ARG DOCKER_IMAGE_NAME
 
 # List out all image permutations to trick dependabot
-FROM --platform=linux/amd64 kong/kong-build-tools:apk-1.8.1 as x86_64-linux-musl
-FROM --platform=linux/amd64 kong/kong-build-tools:rpm-1.8.1 as x86_64-linux-gnu
-FROM --platform=linux/arm64 kong/kong-build-tools:apk-1.8.1 as aarch64-linux-musl
-FROM --platform=linux/arm64 kong/kong-build-tools:rpm-1.8.1 as aarch64-linux-gnu
+FROM --platform=linux/amd64 ghcr.io/kong/kong-openssl:1.1.0-x86_64-linux-musl as x86_64-linux-musl
+FROM --platform=linux/amd64 ghcr.io/kong/kong-openssl:1.1.0-x86_64-linux-gnu as x86_64-linux-gnu
+FROM --platform=linux/arm64 ghcr.io/kong/kong-openssl:1.1.0-aarch64-linux-musl as aarch64-linux-musl
+FROM --platform=linux/arm64 ghcr.io/kong/kong-openssl:1.1.0-aarch64-linux-gnu as aarch64-linux-gnu
 
 
 # Run the build script
 FROM $ARCHITECTURE-$OSTYPE as build
 
-COPY . /src
-RUN /src/build.sh && /src/test.sh
+COPY . /tmp
+WORKDIR /tmp
 
+RUN /tmp/build.sh
 
 # Copy the build result to scratch so we can export the result
 FROM scratch as package

diff --git a/build.sh b/build.sh
@@ -1,14 +1,139 @@
 #!/usr/bin/env bash
 
-set -euo pipefail
-IFS=$'\n\t'
+set -eo pipefail
 
 if [ -n "${DEBUG:-}" ]; then
     set -x
 fi
 
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+export $(grep -v '^#' $SCRIPT_DIR/.env | xargs)
+
 function main() {
-    rm -rf /tmp/build/* && uname -a >> /tmp/build/out
+    with_backoff curl --fail -sSLo pcre.tar.gz "https://downloads.sourceforge.net/project/pcre/pcre/${PCRE_VERSION}/pcre-${PCRE_VERSION}.tar.gz"
+    tar -xzvf pcre.tar.gz
+    ln -s pcre-${PCRE_VERSION} pcre
+
+    with_backoff curl --fail -sSLo openresty.tar.gz "https://openresty.org/download/openresty-${OPENRESTY_VERSION}.tar.gz"
+    tar -xzvf openresty.tar.gz
+    ln -s openresty-${OPENRESTY_VERSION} openresty
+
+    with_backoff curl --fail -sSLo luarocks.tar.gz "https://luarocks.org/releases/luarocks-${LUAROCKS_VERSION}.tar.gz"
+    tar -xzvf luarocks.tar.gz
+    ln -s luarocks-${LUAROCKS_VERSION} luarocks
+
+    with_backoff git clone --single-branch --branch ${KONG_NGINX_MODULE_VERSION} https://github.com/Kong/lua-kong-nginx-module
+
+    with_backoff git clone --single-branch --branch ${LUA_RESTY_LMDB_VERSION} https://github.com/Kong/lua-resty-lmdb --recursive
+
+    with_backoff git clone --single-branch --branch ${LUA_RESTY_EVENTS_VERSION} https://github.com/Kong/lua-resty-events --recursive
+
+    pushd openresty-${OPENRESTY_VERSION}/bundle
+        for patch_file in $(ls /tmp/patches/*.patch); do
+            patch -p1 < $patch_file
+        done
+
+        lj_dir=$(ls -d LuaJIT*)
+        lj_release_date=$(echo ${lj_dir} | sed -e 's/LuaJIT-[[:digit:]]\+.[[:digit:]]\+-\([[:digit:]]\+\)/\1/')
+        lj_version_tag="LuaJIT\ 2.1.0-${lj_release_date}"
+    popd
+
+    pushd openresty-${OPENRESTY_VERSION}
+        OPENRESTY_OPTS=(
+            "--prefix=/usr/local/openresty"
+            "--with-pcre-jit"
+            "--with-http_ssl_module"
+            "--with-http_sub_module"
+            "--with-http_realip_module"
+            "--with-http_stub_status_module"
+            "--with-http_v2_module"
+            "--without-http_encrypted_session_module"
+            "--with-luajit-xcflags='-DLUAJIT_VERSION=\\\"${lj_version_tag}\\\"'"
+            "-j2"
+        )
+
+        OPENRESTY_OPTS+=("--add-module=/tmp/lua-kong-nginx-module")
+        OPENRESTY_OPTS+=("--add-module=/tmp/lua-kong-nginx-module/stream")
+        OPENRESTY_OPTS+=("--add-module=/tmp/lua-resty-lmdb")
+        OPENRESTY_OPTS+=("--add-module=/tmp/lua-resty-events")
+        OPENRESTY_OPTS+=('--with-stream_realip_module')
+        OPENRESTY_OPTS+=('--with-stream_ssl_preread_module')
+        OPENRESTY_OPTS+=('--with-pcre=/tmp/pcre')
+        OPENRESTY_OPTS+=("--with-ld-opt='-L/tmp/build/usr/local/kong/lib -Wl,--disable-new-dtags,-rpath,/usr/local/kong/lib'")
+        OPENRESTY_OPTS+=("--with-cc-opt='-I/tmp/build/usr/local/kong/include'")
+
+        eval ./configure ${OPENRESTY_OPTS[*]} || tee /tmp/00-openresty-configure.log
+
+        make -j2 || tee /tmp/01-openresty-build.log
+        make -j2 install DESTDIR=/tmp/build || tee /tmp/02-openresty-install.log
+    popd
+
+    pushd /tmp/lua-kong-nginx-module
+        make install LUA_LIB_DIR=/tmp/build/usr/local/openresty/lualib
+    popd
+
+    pushd /tmp/lua-resty-lmdb
+        make install LUA_LIB_DIR=/tmp/build/usr/local/openresty/lualib
+    popd
+
+    pushd /tmp/lua-resty-events
+        make install LUA_LIB_DIR=/tmp/build/usr/local/openresty/lualib
+    popd
+
+    pushd /tmp/luarocks-${LUAROCKS_VERSION}
+        ./configure \
+            --prefix=/usr/local \
+            --with-lua=/tmp/build/usr/local/openresty/luajit \
+            --with-lua-include=/tmp/build/usr/local/openresty/luajit/include/luajit-2.1
+
+        make build -j2
+        make install DESTDIR=/tmp/build
+    popd
+
+    arch=$(uname -m)
+
+    package_architecture=x86_64
+    if [ "$(arch)" == "aarch64" ]; then
+    package_architecture=aarch64
+    fi
+
+    curl -fsSLo atc-router.tar.gz https://github.com/hutchic/atc-router/releases/download/$ATC_ROUTER_VERSION/$package_architecture-unknown-$OSTYPE.tar.gz
+    tar -C /tmp/build -xvf atc-router.tar.gz
+}
+
+# Retries a command a configurable number of times with backoff.
+#
+# The retry count is given by ATTEMPTS (default 5), the initial backoff
+# timeout is given by TIMEOUT in seconds (default 1.)
+#
+# Successive backoffs double the timeout.
+function with_backoff {
+    local max_attempts=${ATTEMPTS-5}
+    local timeout=${TIMEOUT-5}
+    local attempt=1
+    local exitCode=0
+
+    while (( $attempt < $max_attempts ))
+    do
+        if "$@"
+        then
+            return 0
+        else
+            exitCode=$?
+        fi
+
+        echo "Failure! Retrying in $timeout.." 1>&2
+        sleep $timeout
+        attempt=$(( attempt + 1 ))
+        timeout=$(( timeout * 2 ))
+    done
+
+    if [[ $exitCode != 0 ]]
+    then
+        echo "You've failed me for the last time! ($@)" 1>&2
+    fi
+
+    return $exitCode
 }
 
 main
diff --git a/patches/LuaJIT-2.1-20220411_01_patch_macro_luajit_version.patch b/patches/LuaJIT-2.1-20220411_01_patch_macro_luajit_version.patch
@@ -0,0 +1,25 @@
+From f53c8fa441f4233b9a3f19fcd870207fe8795456 Mon Sep 17 00:00:00 2001
+From: Qi <[email protected]>
+Date: Wed, 25 May 2022 18:35:08 +0800
+Subject: [PATCH] Patch macro `LUAJIT_VERSION`
+
+---
+ src/luajit.h | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/LuaJIT-2.1-20220411/src/luajit.h b/LuaJIT-2.1-20220411/src/luajit.h
+index a4d33001..e35f4e7e 100644
+--- a/LuaJIT-2.1-20220411/src/luajit.h
++++ b/LuaJIT-2.1-20220411/src/luajit.h
+@@ -32,7 +32,9 @@
+
+ #define OPENRESTY_LUAJIT
+
++#ifndef LUAJIT_VERSION
+ #define LUAJIT_VERSION		"LuaJIT 2.1.0-beta3"
++#endif
+ #define LUAJIT_VERSION_NUM	20100  /* Version 2.1.0 = 02.01.00. */
+ #define LUAJIT_VERSION_SYM	luaJIT_version_2_1_0_beta3
+ #define LUAJIT_COPYRIGHT	"Copyright (C) 2005-2022 Mike Pall"
+--
+2.34.1
diff --git a/patches/LuaJIT-2.1-20220411_02.patch b/patches/LuaJIT-2.1-20220411_02.patch
@@ -0,0 +1,24 @@
+From dad04f1754723e76ba9dcf9f401f3134a0cd3972 Mon Sep 17 00:00:00 2001
+From: Mike Pall <mike>
+Date: Wed, 14 Sep 2022 12:26:53 +0200
+Subject: [PATCH] Fix trace join to BC_JLOOP originating from BC_ITERN.
+
+Reported by OpenResty Inc.
+---
+ src/lj_record.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/LuaJIT-2.1-20220411/src/lj_record.c b/LuaJIT-2.1-20220411/src/lj_record.c
+index 5d02d24a1..bfd412365 100644
+--- a/LuaJIT-2.1-20220411/src/lj_record.c
++++ b/LuaJIT-2.1-20220411/src/lj_record.c
+@@ -2572,7 +2572,8 @@ void lj_record_ins(jit_State *J)
+     break;
+   case BC_JLOOP:
+     rec_loop_jit(J, rc, rec_loop(J, ra,
+-				 !bc_isret(bc_op(traceref(J, rc)->startins))));
++				 !bc_isret(bc_op(traceref(J, rc)->startins)) &&
++				 bc_op(traceref(J, rc)->startins) != BC_ITERN));
+     break;
+
+   case BC_IFORL:
diff --git a/patches/lua-cjson-2.1.0.10_01-empty_array.patch b/patches/lua-cjson-2.1.0.10_01-empty_array.patch
@@ -0,0 +1,12 @@
+diff -ruN a/lua-cjson-2.1.0.8/lua_cjson.c b/lua-cjson-2.1.0.8/lua_cjson.c
+--- a/lua-cjson-2.1.0.10/lua_cjson.c	2022-01-11 15:11:17.495464192 +0800
++++ b/lua-cjson-2.1.0.10/lua_cjson.c	2022-01-11 14:58:55.150669748 +0800
+@@ -800,7 +800,7 @@
+     case LUA_TLIGHTUSERDATA:
+         if (lua_touserdata(l, -1) == NULL) {
+             strbuf_append_mem(json, "null", 4);
+-        } else if (lua_touserdata(l, -1) == &json_array) {
++        } else if (lua_touserdata(l, -1) == json_lightudata_mask(&json_array)) {
+             json_append_array(l, cfg, current_depth, json, 0);
+         }
+         break;