Refactor all make targets into just (#406)

* Start refactoring Justfile into multiple files

Add Justfile for build/test/run in each product
Separate CI specific targets into ci.Justfile

* Implement separate ci.Justfile

* Use Justfile instead of Makefile

* Fix Justfile path and regex

* Add connect-content-init Justfile

* Change content to use just

* Update documentation and add Justfile for float

* Remove Makefile references and replace with Justfile

* Final updates to remove make references with just

* Add docstrings to all Justfiles

* Add `just` documentation reference

* Collapse `sed` commands and fix just doc links

* Fix accidental tag safe version var rename

* Fix tag safe version var name in root Justfile for consistency

.github/workflows/build-latest.yaml

@@ -51,13 +51,13 @@ jobs:
       - name: Get Version
         id: get-version
         run: |
-          VERSION=`just get-version ${{ matrix.config.product }} --type=release --local`
+          VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=release --local`
           echo "::set-output name=VERSION::$VERSION"
 
       - name: Build Image
         id: build-image
         run: |
-          TAGS=`just BUILDX_PATH=${{ steps.buildx.outputs.name }} build-release ${{ matrix.config.product }} ${{ matrix.config.os }} ${{ steps.get-version.outputs.VERSION }}`
+          TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} build-release ${{ matrix.config.product }} ${{ matrix.config.os }} ${{ steps.get-version.outputs.VERSION }}`
           echo "::set-output name=TAGS::$TAGS"
 
       - name: Show image size
@@ -112,5 +112,5 @@ jobs:
       - name: Push image(s) to Docker Hub
         if: ${{ github.ref == 'refs/heads/main' }}
         run: |
-          just push-images ${{ steps.build-image.outputs.TAGS }}
+          just -f ci.Justfile push-images ${{ steps.build-image.outputs.TAGS }}
 

.github/workflows/build-preview.yaml

@@ -61,13 +61,13 @@ jobs:
       - name: Get Version
         id: get-version
         run: |
-          VERSION=`just get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local`
+          VERSION=`just -f ci.Justfile get-version ${{ matrix.config.product }} --type=${{ matrix.config.type }} --local`
           echo "::set-output name=VERSION::$VERSION"
 
       - name: Build Image
         id: build-image
         run: |
-          TAGS=`just BUILDX_PATH=${{ steps.buildx.outputs.name }} build-preview ${{ matrix.config.type }} ${{ matrix.config.product }} ${{ matrix.config.os }} ${{ steps.get-version.outputs.VERSION }}`
+          TAGS=`just -f ci.Justfile BUILDX_PATH=${{ steps.buildx.outputs.name }} build-preview ${{ matrix.config.type }} ${{ matrix.config.product }} ${{ matrix.config.os }} ${{ steps.get-version.outputs.VERSION }}`
           echo "::set-output name=TAGS::$TAGS"
 
       - name: Show image size
@@ -100,4 +100,4 @@ jobs:
       - name: Push image(s) to Docker Hub
         if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/dev-rspm' }}
         run: |
-          just push-images ${{ steps.build-image.outputs.TAGS }}
+          just -f ci.Justfile push-images ${{ steps.build-image.outputs.TAGS }}

Justfile

@@ -6,206 +6,159 @@ sed_vars := if os() == "macos" { "-i ''" } else { "-i" }
 
 BUILDX_PATH := ""
 
-RSW_VERSION := "2022.07.2+576.pro12"
-RSW_TAG_VERSION := replace(RSW_VERSION, "+", "-")
 RSC_VERSION := "2022.09.0"
 RSPM_VERSION := "2022.07.2-11"
+RSW_VERSION := "2022.07.2+576.pro12"
+RSC_TAG_SAFE_VERSION := replace(RSW_VERSION, "+", "-")
+
+DRIVERS_VERSION := "2021.10.0"
+
 R_VERSION := "3.6.2"
 R_VERSION_ALT := "4.1.0"
 
 PYTHON_VERSION := "3.9.5"
 PYTHON_VERSION_ALT := "3.8.10"
 
-# just RSW_VERSION=1.2.3 update-versions
+# just RSW_VERSION=1.2.3 R_VERSION=4.1.0 update-versions
 update-versions:
   just \
-    RSW_VERSION={{RSW_VERSION}} RSC_VERSION={{RSC_VERSION}} RSPM_VERSION={{RSPM_VERSION}} \
-    R_VERSION={{R_VERSION}} R_VERSION_ALT={{R_VERSION_ALT}} \
-    update-rsw-versions update-rspm-versions update-rsc-versions update-r-versions
+    RSW_VERSION={{RSW_VERSION}} \
+    RSC_VERSION={{RSC_VERSION}} \
+    RSPM_VERSION={{RSPM_VERSION}} \
+    R_VERSION={{R_VERSION}} \
+    R_VERSION_ALT={{R_VERSION_ALT}} \
+    PYTHON_VERSION={{PYTHON_VERSION}} \
+    PYTHON_VERSION_ALT={{PYTHON_VERSION_ALT}} \
+    DRIVERS_VERSION={{DRIVERS_VERSION}} \
+    update-rsw-versions update-rspm-versions update-rsc-versions update-r-versions update-py-versions update-drivers-versions
 
 # just RSW_VERSION=1.2.3 update-rsw-versions
 update-rsw-versions:
   #!/usr/bin/env bash
   set -euxo pipefail
-  sed {{ sed_vars }} "s/^RSW_VERSION=.*/RSW_VERSION={{ RSW_VERSION }}/g" workbench/.env
-  sed {{ sed_vars }} "s/^RSW_VERSION=.*/RSW_VERSION={{ RSW_VERSION }}/g" r-session-complete/.env
-  sed {{ sed_vars }} "s/^ARG RSW_VERSION=.*/ARG RSW_VERSION={{ RSW_VERSION }}/g" r-session-complete/Dockerfile.bionic
-  sed {{ sed_vars }} "s/^ARG RSW_VERSION=.*/ARG RSW_VERSION={{ RSW_VERSION }}/g" r-session-complete/Dockerfile.centos7
-  sed {{ sed_vars }} "s/^ARG RSW_VERSION=.*/ARG RSW_VERSION={{ RSW_VERSION }}/g" workbench/Dockerfile.bionic
+  sed {{ sed_vars }} "s/RSW_VERSION=.*/RSW_VERSION={{ RSW_VERSION }}/g" \
+    workbench/.env \
+    r-session-complete/.env \
+    workbench-for-microsoft-azure-ml/.env \
+    r-session-complete/Dockerfile.bionic \
+    r-session-complete/Dockerfile.centos7 \
+    workbench/Dockerfile.bionic \
+    workbench-for-microsoft-azure-ml/Dockerfile.bionic
   sed {{ sed_vars }} "s/RSW_VERSION:.*/RSW_VERSION: {{ RSW_VERSION }}/g" docker-compose.yml
-  sed {{ sed_vars }} "s/rstudio\/rstudio-workbench:.*/rstudio\/rstudio-workbench:{{ RSW_TAG_VERSION }}/g" docker-compose.yml
-  sed {{ sed_vars }} "s/^ARG RSW_VERSION=.*/ARG RSW_VERSION={{ RSW_VERSION }}/g" workbench-for-microsoft-azure-ml/Dockerfile.bionic
+  sed {{ sed_vars }} "s/rstudio\/rstudio-workbench:.*/rstudio\/rstudio-workbench:{{ RSC_TAG_SAFE_VERSION }}/g" docker-compose.yml
   sed {{ sed_vars }} "s/org.opencontainers.image.version='.*'/org.opencontainers.image.version='{{ RSW_VERSION }}'/g" workbench-for-microsoft-azure-ml/Dockerfile.bionic
+  sed {{ sed_vars }} "s/^RSW_VERSION := .*/RSW_VERSION := \"{{ RSW_VERSION }}\"/g" \
+    r-session-complete/Justfile \
+    workbench/Justfile \
+    workbench-for-microsoft-azure-ml/Justfile \
+    Justfile
 
 # just RSPM_VERSION=1.2.3 update-rspm-versions
 update-rspm-versions:
   #!/usr/bin/env bash
   set -euxo pipefail
-  sed {{ sed_vars }} "s/^RSPM_VERSION=.*/RSPM_VERSION={{ RSPM_VERSION }}/g" package-manager/.env
-  sed {{ sed_vars }} "s/^ARG RSPM_VERSION=.*/ARG RSPM_VERSION={{ RSPM_VERSION }}/g" package-manager/Dockerfile.bionic
-  sed {{ sed_vars }} "s/^RSPM_VERSION:.*/RSPM_VERSION: {{ RSPM_VERSION }}/g" docker-compose.yml
+  sed {{ sed_vars }} "s/RSPM_VERSION=.*/RSPM_VERSION={{ RSPM_VERSION }}/g" \
+    package-manager/.env \
+    package-manager/Dockerfile.bionic
   sed {{ sed_vars }} "s/RSPM_VERSION:.*/RSPM_VERSION: {{ RSPM_VERSION }}/g" docker-compose.yml
   sed {{ sed_vars }} "s/rstudio\/rstudio-package-manager:.*/rstudio\/rstudio-package-manager:{{ RSPM_VERSION }}/g" docker-compose.yml
+  sed {{ sed_vars }} "s/^RSPM_VERSION := .*/RSPM_VERSION := \"{{ RSPM_VERSION }}\"/g" \
+    package-manager/Justfile \
+    Justfile
 
 # just RSC_VERSION=1.2.3 update-rsc-versions
 update-rsc-versions:
   #!/usr/bin/env bash
   set -euxo pipefail
-  sed {{ sed_vars }} "s/^RSC_VERSION=.*/RSC_VERSION={{ RSC_VERSION }}/g" connect/.env
-  sed {{ sed_vars }} "s/^ARG RSC_VERSION=.*/ARG RSC_VERSION={{ RSC_VERSION }}/g" connect/Dockerfile.bionic
-  sed {{ sed_vars }} "s/^ARG RSC_VERSION=.*/ARG RSC_VERSION={{ RSC_VERSION }}/g" connect-content-init/Dockerfile.bionic
+  sed {{ sed_vars }} "s/RSC_VERSION=.*/RSC_VERSION={{ RSC_VERSION }}/g" \
+    connect/.env \
+    connect/Dockerfile.bionic \
+    connect-content-init/Dockerfile.bionic
   sed {{ sed_vars }} "s/RSC_VERSION:.*/RSC_VERSION: {{ RSC_VERSION }}/g" docker-compose.yml
   sed {{ sed_vars }} "s/rstudio\/rstudio-connect:.*/rstudio\/rstudio-connect:{{ RSC_VERSION }}/g" docker-compose.yml
-  sed {{ sed_vars }} "s/^RSC_VERSION?=.*/RSC_VERSION?={{ RSC_VERSION }}/g" connect-content-init/Makefile
+  sed {{ sed_vars }} "s/^RSC_VERSION := .*/RSC_VERSION := \"{{ RSC_VERSION }}\"/g" \
+    connect/Justfile \
+    Justfile
 
-# just R_VERSION=3.2.1 update-r-versions
+# just R_VERSION=3.2.1 R_VERSION_ALT=4.1.0 update-r-versions
 update-r-versions:
   #!/usr/bin/env bash
   set -euxo pipefail
-  sed {{ sed_vars }} "s/^R_VERSION:.*/R_VERSION={{ R_VERSION }}/g" workbench/Dockerfile.bionic
-  sed {{ sed_vars }} "s/^R_VERSION:.*/R_VERSION={{ R_VERSION }}/g" connect/Dockerfile.bionic
-  sed {{ sed_vars }} "s/^R_VERSION:.*/R_VERSION={{ R_VERSION }}/g" package-manager/Dockerfile.bionic
+  # Update primary R versions
+  sed {{ sed_vars }} "s/R_VERSION=.*/R_VERSION={{ R_VERSION }}/g" \
+    workbench/.env \
+    connect/.env \
+    package-manager/.env \
+    workbench/Dockerfile.bionic \
+    connect/Dockerfile.bionic \
+    package-manager/Dockerfile.bionic
   sed {{ sed_vars }} "s|^RVersion.*=.*|RVersion = /opt/R/{{ R_VERSION }}/|g" package-manager/rstudio-pm.gcfg
-
-build-release $PRODUCT $OS $VERSION $BRANCH=`git branch --show` $SHA_SHORT=`git rev-parse --short HEAD`:
+  sed {{ sed_vars }} "s/^R_VERSION := .*/R_VERSION := \"{{ R_VERSION }}\"/g" \
+    workbench/Justfile \
+    workbench-for-microsoft-azure-ml/Justfile \
+    connect/Justfile package-manager/Justfile \
+    Justfile
+
+  # Update alt R versions
+  sed {{ sed_vars }} "s/R_VERSION_ALT=.*/R_VERSION_ALT={{ R_VERSION_ALT }}/g" \
+    workbench/.env \
+    connect/.env \
+    workbench/Dockerfile.bionic \
+    connect/Dockerfile.bionic
+  sed {{ sed_vars }} "s/^R_VERSION_ALT := .*/R_VERSION_ALT := \"{{ R_VERSION_ALT }}\"/g" \
+    workbench/Justfile \
+    workbench-for-microsoft-azure-ml/Justfile \
+    connect/Justfile \
+    Justfile
+
+# just PYTHON_VERSION=3.9.5 PYTHON_VERSION_ALT=3.8.10 update-py-versions
+update-py-versions:
   #!/usr/bin/env bash
   set -euxo pipefail
-
-  # variable placeholders
-  RSW_DOWNLOAD_URL=`just _rsw-download-url release $OS`
-  BUILDX_ARGS=""
-  SHORT_NAME=""
-  TAG_VERSION=`just _tag_safe_version $VERSION`
-
-  # set short name
-  if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then
-    SHORT_NAME="RSW"
-  elif [[ $PRODUCT == "connect" ]]; then
-    SHORT_NAME="RSC"
-  elif [[ $PRODUCT == "package-manager" ]]; then
-    SHORT_NAME="RSPM"
-  fi
-
-  # set image prefix
-  if [[ $PRODUCT == "r-session-complete" ]]; then
-    IMAGE_PREFIX=""
-  else
-    IMAGE_PREFIX="rstudio-"
-  fi
-
-  # set buildx args
-  if [[ "{{BUILDX_PATH}}" != "" ]]; then
-    BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache"
-  fi
-
-  docker buildx --builder="{{BUILDX_PATH}}" build --load $BUILDX_ARGS \
-        -t rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS} \
-        -t rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION} \
-        -t rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION}--${SHA_SHORT} \
-        -t ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS} \
-        -t ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION} \
-        -t ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION}--${SHA_SHORT} \
-        --build-arg "$SHORT_NAME"_VERSION=$VERSION \
-        --build-arg RSW_DOWNLOAD_URL=$RSW_DOWNLOAD_URL \
-        --file=./${PRODUCT}/Dockerfile.${OS} ${PRODUCT}
-
-  echo rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS} \
-        rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION} \
-        rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION}--${SHA_SHORT} \
-        ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS} \
-        ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION} \
-        ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}:${OS}-${TAG_VERSION}--${SHA_SHORT}
-
-# just BUILDX_PATH=~/.buildx build-preview preview workbench bionic 12.0.11-11
-build-preview $TYPE $PRODUCT $OS $VERSION $BRANCH=`git branch --show`:
+  # Update primary Python versions
+  sed {{ sed_vars }} "s/PYTHON_VERSION=.*/PYTHON_VERSION={{ PYTHON_VERSION }}/g" \
+    workbench/Dockerfile.bionic \
+    workbench/.env \
+    connect/Dockerfile.bionic \
+    connect/.env \
+    package-manager/Dockerfile.bionic \
+    package-manager/.env
+  sed {{ sed_vars }} "s/^PYTHON_VERSION := .*/PYTHON_VERSION := \"{{ PYTHON_VERSION }}\"/g" \
+    workbench/Justfile \
+    workbench-for-microsoft-azure-ml/Justfile \
+    connect/Justfile \
+    Justfile
+
+  # Update alt Python versions
+  sed {{ sed_vars }} "s/PYTHON_VERSION_ALT=.*/PYTHON_VERSION_ALT={{ PYTHON_VERSION_ALT }}/g" \
+    workbench/Dockerfile.bionic \
+    workbench/.env \
+    connect/Dockerfile.bionic \
+    connect/.env \
+    workbench/Justfile \
+    workbench-for-microsoft-azure-ml/Justfile \
+    connect/Justfile \
+    Justfile
+
+# just DRIVERS_VERSION=2021.10.0 update-driver-versions
+update-drivers-versions:
   #!/usr/bin/env bash
   set -euxo pipefail
-
-  # variable placeholders
-  BRANCH_PREFIX=""
-  RSW_DOWNLOAD_URL=`just _rsw-download-url $TYPE $OS`
-  BUILDX_ARGS=""
-  SHORT_NAME=""
-  TAG_VERSION=`just _tag_safe_version $VERSION`
-
-  # set branch prefix
-  if [[ $BRANCH == "dev" ]]; then
-    BRANCH_PREFIX="dev-"
-  elif [[ $BRANCH == "dev-rspm" ]]; then
-    BRANCH_PREFIX="dev-rspm-"
-  fi
-
-  # set short name
-  if [[ $PRODUCT == "workbench" || $PRODUCT == "r-session-complete" || $PRODUCT == "workbench-for-microsoft-azure-ml" ]]; then
-    SHORT_NAME="RSW"
-  elif [[ $PRODUCT == "connect" || $PRODUCT == "connect-content-init" ]]; then
-    SHORT_NAME="RSC"
-  elif [[ $PRODUCT == "package-manager" ]]; then
-    SHORT_NAME="RSPM"
-  fi
-
-  # set image prefix
-  if [[ $PRODUCT == "r-session-complete" ]]; then
-    IMAGE_PREFIX=""
-  else
-    IMAGE_PREFIX="rstudio-"
-  fi
-
-  # set buildx args
-  if [[ "{{BUILDX_PATH}}" != "" ]]; then
-    BUILDX_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache"
-  fi
-
-  docker buildx --builder="{{BUILDX_PATH}}" build --load $BUILDX_ARGS \
-        -t rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TAG_VERSION} \
-        -t rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TYPE} \
-        -t ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TAG_VERSION} \
-        -t ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TYPE} \
-        --build-arg ${SHORT_NAME}_VERSION=$VERSION \
-        --build-arg RSW_DOWNLOAD_URL=$RSW_DOWNLOAD_URL \
-        --file=./${PRODUCT}/Dockerfile.${OS} ${PRODUCT}
-
-  # These tags are propogated forward to test-images and push-images in builds. It is important that these tags match the build tags above.
-  echo rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TAG_VERSION} \
-        rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TYPE} \
-        ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TAG_VERSION} \
-        ghcr.io/rstudio/${IMAGE_PREFIX}${PRODUCT}-preview:${BRANCH_PREFIX}${OS}-${TYPE}
-
-_rsw-download-url TYPE OS:
-  #!/usr/bin/env bash
-  if [[ "{{TYPE}}" == "release" ]]; then
-    echo "https://download2.rstudio.org/server/{{OS}}/{{ if OS == "centos7" { "x86_64"} else { "amd64" } }}"
-  else
-    echo "https://s3.amazonaws.com/rstudio-ide-build/server/{{OS}}/{{ if OS == "centos7" { "x86_64"} else { "amd64" } }}"
-  fi
-
-# just push-images tag1 tag2 ...
-push-images +IMAGES:
-  #!/usr/bin/env bash
-  set -euxo pipefail
-  for IMAGE in {{IMAGES}}
-  do
-    docker push $IMAGE
-  done
+  sed {{ sed_vars }} "s/\"drivers\": \".[^\,\}]*\"/\"drivers\": \"{{ DRIVERS_VERSION }}\"/g" content/matrix.json
+  sed {{ sed_vars }} "s/DRIVERS_VERSION=.*/DRIVERS_VERSION={{ DRIVERS_VERSION }}/g" workbench-for-microsoft-azure-ml/Dockerfile.bionic
+  sed {{ sed_vars }} "s/^DRIVERS_VERSION := .*/DRIVERS_VERSION := \"{{ DRIVERS_VERSION }}\"/g" \
+    content/pro/Justfile \
+    Justfile
 
 # just test-image preview workbench 12.0.11-8 tag1 tag2 tag3 ...
 test-image $PRODUCT $VERSION +IMAGES:
   #!/usr/bin/env bash
   set -euxo pipefail
   IMAGES="{{IMAGES}}"
   read -ra IMAGE_ARRAY <<<"$IMAGES"
-  cd ./"$PRODUCT" && \
-    IMAGE_NAME="${IMAGE_ARRAY[0]}" RSW_VERSION="$VERSION" RSC_VERSION="$VERSION" RSPM_VERSION="$VERSION" \
-    docker-compose -f docker-compose.test.yml run sut
-
-# just get-version workbench --type=preview --local
-get-version +NARGS:
-  ./get-version.py {{NARGS}}
-
-_tag_safe_version $VERSION:
-  #!/usr/bin/env bash
-  echo -n "$VERSION" | sed 's/+/-/g'
+  just $PRODUCT/test "${IMAGE_ARRAY[0]}" "$VERSION"
 
+# just lint workbench bionic
 lint $PRODUCT $OS:
   #!/usr/bin/env bash
   docker run --rm -i -v $PWD/hadolint.yaml:/.config/hadolint.yaml ghcr.io/hadolint/hadolint < $PRODUCT/Dockerfile.$OS