cleanup(gh-actions/build-debian): Unify steps common tasks and make apt faster

.github/workflows/test-build-deb.yaml

@@ -59,6 +59,7 @@ jobs:
         uses: ./.source/gh-actions/common/build-debian
         with:
           docker-image: ${{ env.UBUNTU_IMAGE }}
+          eatmydata: false
 
 
   run_autopkgtest_from_src_and_binaries:

gh-actions/common/build-debian/action.yml

@@ -5,37 +5,50 @@ inputs:
   source-dir:
     required: false
     description: Directory where the source is located
+
   docker-image:
     required: false
     default: ubuntu:rolling
     description: The docker image used to build the package
+
   token:
     required: false
     description: If provided, used for git authentication in the source build
+
   extra-source-build-deps:
     description: A list of extra build dependencies required during source build.
     required: false
     # FIXME: this should default to '', but we don't want to break job depending on us for now
     default: 'ca-certificates'
+
   extra-source-build-script:
     description: |
       A script to run to prepare the source build machine.
       This happens after the dependencies have been installed, but before
       running `dpkg-buildpackage -S`.
     required: false
     default: ''
+
   sources-only:
     description: Whether to build the package sources only.
     type: boolean
     required: false
     default: false
+
   from-sources-file:
     description: |
       The path to the source .dsc or .changes file.
       The the rest of the source files are expected to be in the same directory.
       If this is provided, then the source package won't be prepared from the repository code.
     required: false
     default: ''
+
+  eatmydata:
+    description: Use eatmydata as dpkg replacement to speedup file writes
+    required: false
+    type: boolean
+    default: true
+
   lintian:
     required: false
     description: Arguments to pass to lintian, if any. Set to `skip` to skip the lintian check.
@@ -85,17 +98,31 @@ runs:
         cd '${{ inputs.source-dir }}'
 
         # Short commit to avoid "package-has-long-file-name"
-        echo VERSION_REF=$(date +'%y%m%d')+${{ github.run_number }}+$(echo ${{ github.sha }} | cut -c1-8) >> $GITHUB_ENV
+        VERSION_REF=$(date +'%y%m%d')+${{ github.run_number }}+$(echo ${{ github.sha }} | cut -c1-8)
+        echo "::endgroup::"
 
-        echo DEBFULLNAME="GitHub actions runner" >> $GITHUB_ENV
-        echo DEBEMAIL="[email protected]" >> $GITHUB_ENV
+        echo "::group::Prepare source output directory"
+        src_output_dir="$(mktemp --directory --tmpdir="${PWD}" -t output-XXXXXX)"
+        echo SOURCE_OUTPUT_DIR="${src_output_dir}" >> ${GITHUB_ENV}
+        echo "::endgroup::"
+
+        echo "::group::Set build environment"
+        (
+          echo DEBFULLNAME="GitHub actions runner"
+          echo DEBEMAIL="[email protected]"
+          echo VERSION_REF="${VERSION_REF}"
+          echo "SOURCE_DIR=${{ github.workspace }}/${{ inputs.source-dir }}"
+          echo "GITHUB_ACTION_PATH=${GITHUB_ACTION_PATH}"
+        ) >> "${GITHUB_ENV}"
 
         if git status --porcelain &>/dev/null; then
-          echo DEBFULLNAME="$(git log -1 --format='%an' HEAD) - GH Action" >> $GITHUB_ENV
-          echo DEBEMAIL="$(git log -1 --format='%ae' HEAD)" >> $GITHUB_ENV
+          (
+            echo DEBFULLNAME="$(git log -1 --format='%an' HEAD) - GH Action"
+            echo DEBEMAIL="$(git log -1 --format='%ae' HEAD)"
+          ) >> "${GITHUB_ENV}"
         fi
 
-        echo "SOURCE_DIR=${{ github.workspace }}/${{ inputs.source-dir }}" >> "${GITHUB_ENV}"
+        echo "${{ inputs.eatmydata }}" >> "${GITHUB_ACTION_PATH}/.use-eatmydata"
         echo "::endgroup::"
 
     - name: Extract source file
@@ -122,18 +149,24 @@ runs:
       with:
         image: ${{ inputs.docker-image }}
         environment: |
+          TERM=dumb
           DEBIAN_FRONTEND=noninteractive
+          DEBCONF_NONINTERACTIVE_SEEN=true
           DEBFULLNAME=${{ env.DEBFULLNAME }}
           DEBEMAIL=${{ env.DEBEMAIL }}
-        volumes: ${{ github.workspace }}:${{ github.workspace }}
+        volumes: |
+          ${{ env.GITHUB_ACTION_PATH }}:${{ env.GITHUB_ACTION_PATH }}
+          ${{ github.workspace }}:${{ github.workspace }}
         workdir: ${{ env.SOURCE_DIR }}
         shell: bash
         run: |
-          echo "::group::Update builder instance and install dependencies"
+          echo "::group::Update source builder instance"
           set -eu
 
-          echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes
-          apt update
+          "${{ env.GITHUB_ACTION_PATH }}"/prepare-container.sh
+          echo "::endgroup::"
+
+          echo "::group::Install source build dependencies"
           apt install devscripts lsb-release
           echo "::endgroup::"
 
@@ -156,40 +189,32 @@ runs:
           ) >> "${GITHUB_ENV}"
           echo "::endgroup::"
 
-    - name: Parse package source info
-      shell: bash
-      run: |
-        echo "::group::Prepare source build"
-        set -eu
-
-        echo SOURCE_OUTPUT_DIR="$( mktemp --directory --tmpdir="${PWD}" )" >> $GITHUB_ENV
-        echo ACTION_BIN_PATH="$( mktemp --directory --tmpdir="${PWD}" )" >> $GITHUB_ENV
-        echo "::endgroup::"
-
     - name: Build source package
       uses: kohlerdominik/[email protected]
       with:
         image: ${{ inputs.docker-image }}
         environment: |
+          TERM=dumb
           DEBIAN_FRONTEND=noninteractive
-        volumes: ${{ github.workspace }}:${{ github.workspace }}
+          DEBCONF_NONINTERACTIVE_SEEN=true
+        volumes: |
+          ${{ env.GITHUB_ACTION_PATH }}:${{ env.GITHUB_ACTION_PATH }}
+          ${{ github.workspace }}:${{ github.workspace }}
         workdir: ${{ env.SOURCE_DIR }}
         shell: bash
         run: |
-          echo "::group::Update builder instance"
+          echo "::group::Update source builder instance"
           set -eu
 
-          echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes
-          apt update
-          apt dist-upgrade
+          "${{ env.GITHUB_ACTION_PATH }}"/prepare-container.sh
           echo "::endgroup::"
 
           echo "::group::Install build dependencies"
           apt build-dep .
 
           GITHUB_TOKEN="${{ inputs.token }}"
 
-          if [ -n "${{ inputs.extra-source-build-deps }}" ]; then
+          if [ -n "${{ inputs.extra-source-build-deps != '' && 'true' || '' }}" ]; then
             # Install extra packages for build-deps, to allow downloading vendored sources
             deps=(${{ inputs.extra-source-build-deps }})
 
@@ -233,18 +258,6 @@ runs:
         path: ${{ env.SOURCE_OUTPUT_DIR }}/
         if-no-files-found: error
 
-    - name: Install lintian result parser
-      if: inputs.lintian != 'skip'
-      shell: bash
-      run: |
-        echo "::group::Install lintian result parser"
-        set -eu
-
-        cp "${GITHUB_ACTION_PATH}/lintian-to-md.sh" \
-          "${{ env.ACTION_BIN_PATH }}"/lintian-to-md
-        chmod +x "${{ env.ACTION_BIN_PATH }}"/lintian-to-md
-        echo "::endgroup::"
-
     - name: Run lintian on sources
       # We do this in a different step for various reasons:
       # 1. To still be able to upload the source for manual inspection without
@@ -258,19 +271,18 @@ runs:
         environment:
           TERM=dumb
           DEBIAN_FRONTEND=noninteractive
+          DEBCONF_NONINTERACTIVE_SEEN=true
           SOURCE_OUTPUT_DIR=${{ env.SOURCE_OUTPUT_DIR }}
         volumes: |
+          ${{ env.GITHUB_ACTION_PATH }}:${{ env.GITHUB_ACTION_PATH }}
           ${{ env.SOURCE_OUTPUT_DIR }}:${{ env.SOURCE_OUTPUT_DIR }}
-          ${{ env.ACTION_BIN_PATH }}:${{ env.ACTION_BIN_PATH }}
         workdir: ${{ env.SOURCE_OUTPUT_DIR }}
         shell: bash
         run: |
-          echo "::group::Update tester instance"
+          echo "::group::Update lintian sources tester instance"
           set -eu
 
-          echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes
-          apt update
-          apt dist-upgrade
+          "${{ env.GITHUB_ACTION_PATH }}"/prepare-container.sh
           echo "::endgroup::"
 
           echo "::group::Install lintian"
@@ -288,7 +300,7 @@ runs:
             lintian --pedantic --fail-on error ${{ inputs.lintian }} -- \
               "${{ env.PKG_SOURCE_CHANGES }}" \
             | tee "${out_file}"
-          cat "${out_file}" | "${{ env.ACTION_BIN_PATH }}"/lintian-to-md \
+          cat "${out_file}" | "${{ env.GITHUB_ACTION_PATH }}"/lintian-to-md.sh \
             >> $GITHUB_STEP_SUMMARY
           echo "::endgroup::"
 
@@ -301,9 +313,11 @@ runs:
 
         # Appending /source because 'dpkg-source --extract' needs the output directory to be non-existent
         BUILD_INPUT_BASEDIR="$( mktemp --directory --tmpdir="${PWD}" )"
-        echo BUILD_INPUT_BASEDIR="${BUILD_INPUT_BASEDIR}" >> $GITHUB_ENV
         BUILD_INPUT_DIR="${BUILD_INPUT_BASEDIR}/source"
-        echo BUILD_INPUT_DIR="${BUILD_INPUT_DIR}" >> $GITHUB_ENV
+        (
+          echo BUILD_INPUT_BASEDIR="${BUILD_INPUT_BASEDIR}"
+          echo BUILD_INPUT_DIR="${BUILD_INPUT_DIR}"
+        ) >> "${GITHUB_ENV}"
         echo "::endgroup::"
 
         echo "::group::Create build output directory"
@@ -322,19 +336,20 @@ runs:
         image: ${{ inputs.docker-image }}
         options: --cap-add=NET_ADMIN
         environment: |
+          TERM=dumb
           DEBIAN_FRONTEND=noninteractive
+          DEBCONF_NONINTERACTIVE_SEEN=true
         workdir: ${{ env.BUILD_INPUT_DIR }}
         volumes: |
+          ${{ env.GITHUB_ACTION_PATH }}:${{ env.GITHUB_ACTION_PATH }}
           ${{ env.BUILD_INPUT_BASEDIR }}:${{ env.BUILD_INPUT_BASEDIR }}
           ${{ env.BUILD_OUTPUT_DIR }}:${{ env.BUILD_OUTPUT_DIR }}
         shell: bash
         run: |
           echo "::group::Update builder instance"
           set -eu
 
-          echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes
-          apt update
-          apt dist-upgrade
+          "${{ env.GITHUB_ACTION_PATH }}"/prepare-container.sh
           echo "::endgroup::"
 
           echo "::group::Create build user"
@@ -409,18 +424,17 @@ runs:
         environment:
           TERM=dumb
           DEBIAN_FRONTEND=noninteractive
+          DEBCONF_NONINTERACTIVE_SEEN=true
         volumes: |
+          ${{ env.GITHUB_ACTION_PATH }}:${{ env.GITHUB_ACTION_PATH }}
           ${{ env.BUILD_OUTPUT_DIR }}:${{ env.BUILD_OUTPUT_DIR }}
-          ${{ env.ACTION_BIN_PATH }}:${{ env.ACTION_BIN_PATH }}
         workdir: ${{ env.BUILD_OUTPUT_DIR }}
         shell: bash
         run: |
-          echo "::group::Update builder instance"
+          echo "::group::Update lintian binary tester instance"
           set -eu
 
-          echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes
-          apt update
-          apt dist-upgrade
+          ""${{ env.ACTION_BIN_PATH }}"/${{ env.GITHUB_ACTION_PATH }}"/prepare-container.sh
           echo "::endgroup::"
 
           echo "::group::Install lintian"
@@ -438,6 +452,6 @@ runs:
             lintian --pedantic --fail-on error ${{ inputs.lintian }} -- \
               *_"${{ env.PKG_VERSION }}"_*.deb \
             | tee "${out_file}"
-          cat "${out_file}" | "${{ env.ACTION_BIN_PATH }}"/lintian-to-md \
+          cat "${out_file}" | "${{ env.GITHUB_ACTION_PATH }}"/lintian-to-md.sh \
             >> $GITHUB_STEP_SUMMARY
           echo "::endgroup::"

gh-actions/common/build-debian/prepare-container.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -eux
+
+# Disable installing of manpages and docs
+cat <<"EOF" | tee /etc/dpkg/dpkg.cfg.d/01_nodoc
+# Delete man pages
+path-exclude=/usr/share/man/*
+
+# Delete docs
+path-exclude=/usr/share/doc/*
+path-include=/usr/share/doc/*/copyright
+EOF
+
+echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90aptyes
+apt update
+
+locales_required_check=$(dirname "$0")/.locales-required
+if ! [ -e "${locales_required_check}" ] && [ -f ./debian/control ]; then
+  apt install dctrl-tools
+  build_deps=$(grep-dctrl -s Build-Depends -n - ./debian/control)
+
+  locales_needed=false
+  if echo "${build_deps}" | grep -Fqs language-pack ||
+      echo "${build_deps}" | grep -Fqs locales; then
+      locales_needed=true
+  fi
+
+  # We can't use ${GITHUB_ENV} here, because the github ${{ env.XXX }} variables
+  # arent't visible here, and we don't want to expose each one at call time.
+  echo "${locales_needed}" > "${locales_required_check}"
+fi
+
+if [ "$(cat "${locales_required_check}" || true)" != "true" ]; then
+  # Disable installing of locale files
+  cat <<"EOF" | tee /etc/dpkg/dpkg.cfg.d/01_nolocales
+# Delete locales
+path-exclude=/usr/share/locale/*
+EOF
+fi
+
+apt dist-upgrade
+
+eatmydata_check=$(dirname "$0")/.use-eatmydata
+if [ "$(cat "${eatmydata_check}" || true)" == "true" ]; then
+  # Install and use eatmydata
+  apt install eatmydata
+
+  mkdir -p "/usr/local/libexec/dt-deb-action"
+  ln -s /usr/bin/eatmydata /usr/local/libexec/dt-deb-action/dpkg
+  echo 'Dir::Bin::dpkg /usr/local/libexec/dt-deb-action/dpkg;' > \
+    /etc/apt/apt.conf.d/00dt-deb-action-eatmydata
+fi