diff --git a/.cspell.json b/.cspell.json index 8f52faa0..dbe89e2c 100644 --- a/.cspell.json +++ b/.cspell.json @@ -89,6 +89,7 @@ "edkii", "elif", "emeraldlake", + "endgroup", "exitcode", "filenamify", "githubaction", @@ -119,6 +120,7 @@ "markdownlint", "megalinter", "menuconfig", + "mktemp", "modifyitems", "mountpoint", "multilib", @@ -138,6 +140,7 @@ "pylint", "pytest", "rdparty", + "readarray", "rootdir", "runslow", "rustup", diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml index fa202ffc..055546da 100644 --- a/.github/workflows/example.yml +++ b/.github/workflows/example.yml @@ -163,16 +163,13 @@ jobs: target: 'coreboot-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: COREBOOT_VERSION: ${{ matrix.coreboot-version }} - UROOT_VERSION: "dummy" - - - name: Get artifacts - uses: actions/upload-artifact@v4 - with: - name: coreboot-${{ matrix.coreboot-version }}-${{ matrix.arch }} - path: output-coreboot - retention-days: 14 + UROOT_VERSION: 'dummy' # ANCHOR_END: example_build_coreboot # Example of building Linux kernel @@ -237,17 +234,14 @@ jobs: target: 'linux-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: LINUX_VERSION: ${{ matrix.linux-version }} SYSTEM_ARCH: ${{ matrix.arch }} - UROOT_VERSION: "dummy" - - - name: Get artifacts - uses: actions/upload-artifact@v4 - with: - name: linux-${{ matrix.linux-version }}-${{ matrix.arch }} - path: output-linux - retention-days: 14 + UROOT_VERSION: 'dummy' # ANCHOR_END: example_build_linux_kernel # Example of building EDK2 @@ -317,16 +311,13 @@ jobs: target: 'edk2-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: EDK2_VERSION: ${{ matrix.edk2-version }} GCC_TOOLCHAIN_VERSION: ${{ steps.gcc_toolchain.outputs.gcc_toolchain_version }} - - - name: Get artifacts - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.edk2-version }} - path: output-edk2 - retention-days: 14 # ANCHOR_END: example_build_edk2 # Example of building Firmware Stitching @@ -376,15 +367,12 @@ jobs: target: 'stitching-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: COREBOOT_VERSION: ${{ matrix.coreboot-version }} - - - name: Get artifacts - uses: actions/upload-artifact@v4 - with: - name: stitch-${{ matrix.coreboot-version }}-${{ matrix.arch }} - path: output-stitch - retention-days: 14 # ANCHOR_END: example_build_stitch # Example of building u-root @@ -434,15 +422,12 @@ jobs: target: 'u-root-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: UROOT_VERSION: ${{ matrix.uroot-version }} - - - name: Get artifacts - uses: actions/upload-artifact@v4 - with: - name: uroot-${{ matrix.uroot-version }}-${{ matrix.arch }} - path: output-uroot - retention-days: 14 # ANCHOR_END: example_build_uroot # Example of building u-boot @@ -499,15 +484,12 @@ jobs: target: 'u-boot-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: UBOOT_VERSION: ${{ matrix.uboot-version }} - - - name: Get artifacts - uses: actions/upload-artifact@v4 - with: - name: uboot-${{ matrix.uboot-version }}-${{ matrix.arch }} - path: output-uboot - retention-days: 14 # ANCHOR_END: example_build_uboot # Example of using universal module @@ -634,6 +616,10 @@ jobs: target: 'u-root-example' recursive: 'false' compile: ${{ needs.changes.outputs.compile }} + debug: 'true' + enable-cache: 'true' + auto-download-artifacts: 'true' + auto-upload-artifacts: 'true' env: COREBOOT_VERSION: '4.19' LINUX_VERSION: '6.9.9' diff --git a/Taskfile.yml b/Taskfile.yml index 39fdc0e1..a19be23f 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -27,6 +27,7 @@ tasks: dir: '{{.GOLANG_CODE_PATH}}' cmds: - go build -ldflags="-s -w -X main.version={{.VERSION}} -X main.commit={{.COMMIT}} -X main.date={{.DATE}}" -o ../../bin/firmware-action-{{OS}}-{{ARCH}}-v{{.VERSION}} + - ln -sf ./firmware-action-{{OS}}-{{ARCH}}-v{{.VERSION}} ../../bin/firmware-action env: CGO_ENABLED: 0 sources: @@ -67,6 +68,7 @@ tasks: dir: docs cmds: - mdbook build + - echo "Visit file://{{.USER_WORKING_DIR}}/docs/public/index.html" build-mdbook-watch: desc: Build mdBook with watch diff --git a/action.yml b/action.yml index e8c5836a..bef8b9e7 100644 --- a/action.yml +++ b/action.yml @@ -23,12 +23,95 @@ inputs: Enable this when building complex firmware stack in single job recursively and you are running out of disk space. required: false default: 'false' + debug: + description: | + Run the action with increased verbosity. + required: false + default: 'false' compile: description: | Compile the action from source instead of downloading pre-compiled binary from releases. required: false default: 'false' + # Options for automatic caches and artifacts + enable-cache: + description: | + Automatically cache the '.firmware-action' directory along with all existing output directories. + Firmware-action can automatically cache the produced files, by iterating over all 'output_dir' + entries in the confirmation file. These files, if exists, can be cached. + Because firmware-action has multiple methods of detecting changes in the sources, it can then + rebuild the files if needed. But it there was no change, it can greatly speed up the CI run. + required: false + default: 'false' + auto-download-artifacts: + description: | + Automatically download all artifacts for the current workflow run. + There is no easy convenient way to download all required artifacts, so firmware-action offers the + option to just automatically download all available artifacts. This does not mean that they will + be used during the build, but it means that you do not need to handle the downloading of artifacts + yourself. + Internally a actions/download-artifact@v4 is used + required: false + default: 'false' + auto-upload-artifacts: + description: | + Automatically upload all artifacts.inputs.auto-upload-artifacts-if-no-files-found + Firmware action can automatically upload the artifacts for the current target (upload the 'output_dir' + directory), so that you don't have it. Combined with the option 'auto-download-artifacts' + firmware-action should be able to built entire firmware stack in CI even if it is split into multiple + jobs and even if there are multiple matrix builds running in parallel. + Internally a actions/upload-artifact@v4 is used + required: false + default: 'false' + + # Options passed over to artifact uploading + auto-upload-artifacts-if-no-files-found: + description: | + See 'if-no-files-found' at https://github.com/actions/upload-artifact + default: 'warn' + auto-upload-artifacts-retention-days: + description: | + See 'retention-days' at https://github.com/actions/upload-artifact + default: '0' + auto-upload-artifacts-compression-level: + description: | + See 'compression-level' at https://github.com/actions/upload-artifact + default: '6' + auto-upload-artifacts-overwrite: + description: | + See 'overwrite' at https://github.com/actions/upload-artifact + default: 'false' + # As for the only omitted input 'include-hidden-files', we do need this enabled + # because we are uploading the `./firmware-action` directory + # which would not get uploaded at all with this option set to 'false' + # We might change the behavior in the future if anyone needs it + # (maybe just renaming the directory before upload) + +outputs: + artifact-name: + description: | + Name of the artifact which will be uploaded to GitHub + value: ${{ steps.get_artifact_name.outputs.artifact_name }} + + # These outputs are just passing on outputs of 'upload-artifact' + # Docs: https://github.com/actions/upload-artifact + artifact-id: + description: | + GitHub ID of an Artifact, can be used by the REST API + value: ${{ steps.upload_artifact.outputs.artifact-id }} + artifact-url: + description: | + URL to download an Artifact. + Can be used in many scenarios such as linking to artifacts in issues or pull requests. + Users must be logged-in in order for this URL to work. + This URL is valid as long as the artifact has not expired or the artifact, run or repository have not been deleted. + value: ${{ steps.upload_artifact.outputs.artifact-url }} + artifact-digest: + description: | + SHA-256 digest of an Artifact + value: ${{ steps.upload_artifact.outputs.artifact-digest }} + runs: using: 'composite' steps: @@ -88,33 +171,151 @@ runs: with: go-version: stable + #============================================== + # Acquire firmware-action executable for Linux + #============================================== - id: fetch_unix if: ${{ ( runner.os == 'Linux' || runner.os == 'macOS' ) && inputs.compile == 'false' }} shell: bash run: | + echo "::group::curl"inputs.auto-upload-artifacts-if-no-files-found curl -L -o "${{ steps.filename.outputs.filename }}" "${{ steps.url.outputs.url }}" tar -xvzf "${{ steps.filename.outputs.filename }}" chmod +x firmware-action + echo "::endgroup::" # chmod should not be necessary, but just to be safe - name: compile_unix if: ${{ ( runner.os == 'Linux' || runner.os == 'macOS' ) && inputs.compile == 'true' }} shell: bash working-directory: ./cmd/firmware-action run: | + echo "::group::compile firmware-action" go build -ldflags="-s -w" -o ../../firmware-action + echo "::endgroup::" + + #================================================ + # Acquire firmware-action executable for Windows + #================================================ - id: fetch_windows if: ${{ runner.os == 'Windows' && inputs.compile == 'false' }} shell: pwsh run: | + echo ##[group]fetch Invoke-WebRequest -Uri "${{ steps.url.outputs.url }}" -OutFile "${{ steps.filename.outputs.filename }}" Expand-Archive -Path "${{ steps.filename.outputs.filename }}" -DestinationPath .\ + echo ##[endgroup] - name: compile_windows if: ${{ runner.os == 'Windows' && inputs.compile == 'true' }} shell: pwsh working-directory: ./cmd/firmware-action run: | + echo ##[group]compile firmware-action go build -ldflags="-s -w" -o ../../firmware-action.exe + echo ##[endgroup] + + #================ + # CACHE: Restore + #================ + # + # GitHub does not allow to run a single step in some sort of for-loop, nor does allow to call another + # reusable action to potentially use matrix strategy to achieve this. + # I also looked into GitHub CI commands (link to docs below), but there is no support for uploading + # cache or artifacts + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions + # The only way that I could come up with is to pack it (move all the files into single directory) and then upload. + # And upon download it is unpacked. + # One way might be to implement GitHub API interface right into the golang code, but that seems like too much + # overkill for what we want. + + - name: restore_cache + uses: actions/cache/restore@v4 + if: inputs.enable-cache == 'true' + id: cache + with: + path: .firmware-action/ + key: firmware-action-${{ inputs.target }}-${{ hashFiles(inputs.config) }}-${{ github.sha }}-${{ github.run_id }} + restore-keys: | + firmware-action-${{ inputs.target }}-${{ hashFiles(inputs.config) }}-${{ github.sha }}- + firmware-action-${{ inputs.target }}-${{ hashFiles(inputs.config) }}- + firmware-action-${{ inputs.target }}- + + - name: merge_config_files + shell: bash + id: merge_config + run: | + # Create temporary directory for merged config + TEMP_DIR=$(mktemp -d) + MERGED_CONFIG="${TEMP_DIR}/merged.json" + echo "merged_config=${MERGED_CONFIG}" >> "${GITHUB_OUTPUT}" + + # Initialize empty JSON object + echo '{}' > "${MERGED_CONFIG}" + + # Split config input into individual files + readarray -t CONFIG_FILES <<< "${{ inputs.config }}" + + # Process and merge each config file + for CONFIG_FILE in "${CONFIG_FILES[@]}"; do + # Trim whitespace + CONFIG_FILE=$(echo "${CONFIG_FILE}" | xargs) + + # Skip empty lines + [ -z "${CONFIG_FILE}" ] && continue + + # Merge this config with accumulated config + jq -s '.[0] * .[1]' "${MERGED_CONFIG}" "${CONFIG_FILE}" > "${TEMP_DIR}/temp.json" + mv "${TEMP_DIR}/temp.json" "${MERGED_CONFIG}" + done + + #===================== + # ARTIFACTS: Download + #===================== + # + # Download all artifacts for the current workflow run. + # Docs: https://github.com/actions/download-artifact?tab=readme-ov-file#download-all-artifacts + + - name: download_artifacts + uses: actions/download-artifact@v4 + if: inputs.auto-download-artifacts == 'true' + with: + path: .firmware-action2/ + merge-multiple: true + - name: merge_firmware_action_directory + if: inputs.auto-download-artifacts == 'true' + shell: bash + run: | + echo "::group::merge" + mkdir -p .firmware-action/ + # Check if firmware-action2 directory exists and has content + if [ -d ".firmware-action2/" ] && [ "$(ls -A .firmware-action2/ 2>/dev/null)" ]; then + # Copy all content from firmware-action2 to firmware-action, overwriting existing files + cp -rf .firmware-action2/. .firmware-action/ + echo "Merged downloaded artifacts into .firmware-action directory" + else + echo "No downloaded artifacts found in .firmware-action2/" + fi + echo "::endgroup::" + + - name: unpack_cached_files + shell: bash + if: ${{ steps.cache.outputs.cache-hit == 'true' || inputs.enable-cache == 'true' }} + run: | + echo "::group::unpack" + # Get output directories from merged config + while IFS= read -r DIR; do + # Expand any environment variables in DIR + EXPANDED_DIR=$(eval echo "${DIR}") + if [ -d ".firmware-action/artifacts/${EXPANDED_DIR}" ]; then + echo "moving ${EXPANDED_DIR}" + mv ".firmware-action/artifacts/${EXPANDED_DIR}" ./ + fi + done < <(jq -r '[.. | objects | ."output_dir"? | select(. != null)][]' "${{ steps.merge_config.outputs.merged_config }}") + echo "::endgroup::" + + #================================= + # RUN: firmware-action executable + #================================= - name: run_unix if: ${{ runner.os == 'Linux' || runner.os == 'macOS' }} @@ -137,3 +338,61 @@ runs: INPUT_TARGET: ${{ inputs.target }} INPUT_RECURSIVE: ${{ inputs.recursive }} INPUT_PRUNE: ${{ inputs.prune }} + + #=============== + # CACHE: Create + #=============== + + - name: copy_artifacts + if: ${{ always() && ( inputs.enable-cache == 'true' || inputs.auto-upload-artifacts == 'true' ) }} + shell: bash + run: | + echo "::group::pack" + mkdir -p .firmware-action/artifacts/ + + # Get output directories from merged config and copy them + while IFS= read -r DIR; do + # Expand any environment variables in DIR + EXPANDED_DIR=$(eval echo "${DIR}") + if [ -d "${EXPANDED_DIR}" ]; then + echo "copying ${EXPANDED_DIR}" + cp -r "${EXPANDED_DIR}" .firmware-action/artifacts/ + fi + done < <(jq -r '[.. | objects | ."output_dir"? | select(. != null)][]' "${{ steps.merge_config.outputs.merged_config }}") + echo "::endgroup::" + + - name: save_cache + if: ${{ always() && inputs.enable-cache == 'true' }} + uses: actions/cache/save@v4 + with: + path: .firmware-action/ + key: firmware-action-${{ inputs.target }}-${{ hashFiles(inputs.config) }}-${{ github.sha }}-${{ github.run_id }} + + #=================== + # ARTIFACTS: Upload + #=================== + + - name: get_artifact_name + if: ${{ always() && inputs.auto-upload-artifacts == 'true' }} + shell: bash + id: get_artifact_name + run: | + # Extract output_dir for the specific target from merged config + OUTPUT_DIR=$(jq -r --arg TARGET "${{ inputs.target }}" 'to_entries[] | select(.value | to_entries[].key == $TARGET) | .value[$TARGET].output_dir' "${{ steps.merge_config.outputs.merged_config }}" | sed -E 's/\/$//g') + # Expand any environment variables in OUTPUT_DIR + OUTPUT_DIR=$(eval echo "${OUTPUT_DIR}") + DATETIME=$(date "+%Y-%m-%d_%H-%M-%S.%N") + echo "artifact_name=artifacts--${{ inputs.target }}--${OUTPUT_DIR}--${DATETIME}" >> "${GITHUB_OUTPUT}" + + - name: upload_artifact + if: ${{ always() && inputs.auto-upload-artifacts == 'true' }} + uses: actions/upload-artifact@v4 + id: upload_artifact + with: + name: ${{ steps.get_artifact_name.outputs.artifact_name }} + path: .firmware-action/ + include-hidden-files: true + if-no-files-found: ${{ inputs.auto-upload-artifacts-if-no-files-found }} + retention-days: ${{ inputs.auto-upload-artifacts-retention-days }} + compression-level: ${{ inputs.auto-upload-artifacts-compression-level }} + overwrite: ${{ inputs.auto-upload-artifacts-overwrite }} diff --git a/cmd/firmware-action/filesystem/git.go b/cmd/firmware-action/filesystem/git.go index c57abc91..c98221cb 100644 --- a/cmd/firmware-action/filesystem/git.go +++ b/cmd/firmware-action/filesystem/git.go @@ -117,5 +117,14 @@ func gitDescribe(repoPath string, cfg *describe) (string, error) { // Remove trailing newline result := strings.TrimSpace(describe) + // Check validity of the returned string + pattern := regexp.MustCompile(`[\d\w]{13}(\-dirty)?`) + valid := pattern.MatchString(result) + if !valid { + slog.Warn( + fmt.Sprintf("Output of 'git describe' for '%s' seems to be invalid commit hash, output is '%s'. Will carry on.", repoPath, result), + ) + } + return result, err } diff --git a/cmd/firmware-action/main.go b/cmd/firmware-action/main.go index 433553f4..016c870f 100644 --- a/cmd/firmware-action/main.go +++ b/cmd/firmware-action/main.go @@ -296,6 +296,7 @@ func parseGithub() (string, error) { CLI.Build.Recursive = regexTrue.MatchString(action.GetInput("recursive")) CLI.Build.PruneDockerContainers = regexTrue.MatchString(action.GetInput("prune")) CLI.JSON = regexTrue.MatchString(action.GetInput("json")) + CLI.Debug = regexTrue.MatchString(action.GetInput("debug")) return "GitHub", nil } diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 0c0d5da5..734e1f12 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -28,6 +28,7 @@ - [Change detection](firmware-action/change_detection.md) - [Migration instructions]() - [Migration from v0.13.x to v0.14.0](firmware-action/migration/v0.13.x--v0.14.0/migrate.md) + - [Migration from v0.14.x to v0.15.0](firmware-action/migration/v0.14.x--v0.15.0/migrate.md) --- diff --git a/docs/src/firmware-action/config.md b/docs/src/firmware-action/config.md index a2f1d9c4..5bd89a60 100644 --- a/docs/src/firmware-action/config.md +++ b/docs/src/firmware-action/config.md @@ -9,10 +9,34 @@ ```admonish tip Multiple configuration files can be supplied to `firmware-action`. Dependencies also work across files. +Multiple configs in local build: ~~~ firmware-action build --config=config-01.json --config=config-02.json ... ~~~ +Multiple configs in GitHub CI: +~~~ +- name: firmware-action + uses: ./ + with: + config: |- + config-01.json + config-02.json +~~~ +``` + +```admonish tip +For YAML multi-line string please refer to documentation [Block Style Productions in YAML](https://yaml.org/spec/1.2.2/#block-style-productions) + +| Indicator | Name | Behavior | +|-----------|---------------------|---------------------------------------------------| +| `\|` | Literal Block | Preserves newlines, strips final newline | +| `\|+` | Literal Block Keep | Preserves all newlines including final | +| `\|-` | Literal Block Strip | Strips all trailing newlines | +| `>` | Folded Block | Folds newlines to spaces, keeps paragraph breaks | +| `>+` | Folded Block Keep | Like > but keeps final newline | +| `>-` | Folded Block Strip | Like > but strips final newline | + Beware that modules with identical names are permitted, as long as they are not in the same configuration file. `firmware-action` processes the files in order in which they were supplied and in case of name-collision, the configuration in last file takes precedence. diff --git a/docs/src/firmware-action/get_started/06_run_in_ci.md b/docs/src/firmware-action/get_started/06_run_in_ci.md index 89ff6ae4..bfb7191f 100644 --- a/docs/src/firmware-action/get_started/06_run_in_ci.md +++ b/docs/src/firmware-action/get_started/06_run_in_ci.md @@ -10,5 +10,7 @@ Now that we have `firmware-action` working on local system. Let's set up CI. Commit, push and watch. And that is it. +The action automatically handles artifact uploading and caching. See [GitHub CI Usage](../usage_github.md) for details about artifacts, outputs and caching. + Now you should be able to build coreboot in CI and on your local machine. diff --git a/docs/src/firmware-action/migration/v0.14.x--v0.15.0/migrate.md b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/migrate.md new file mode 100644 index 00000000..f18edec1 --- /dev/null +++ b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/migrate.md @@ -0,0 +1,9 @@ +# Migration guide from v0.14.x to v0.15.0 + +Drop-in replacement, should work out of the box. But there are many simplifications and quality of life improvements you might want to check out. + +Most important being the optional automatic handling of caches and artifacts. This can greatly simplify your workflows. + +```patch +{{#include ./workflow.yml.patch}} +``` diff --git a/docs/src/firmware-action/migration/v0.14.x--v0.15.0/new.yml b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/new.yml new file mode 100644 index 00000000..1ee777f1 --- /dev/null +++ b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/new.yml @@ -0,0 +1,30 @@ +--- +name: linuxboot build +on: + push: + +permissions: + contents: read + +jobs: + build-coreboot-linuxboot-example: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Fetch few missing submodules + run: | + git submodule update --depth 1 --init --recursive --checkout + + - name: firmware-action + uses: 9elements/firmware-action@v0.15.0 + with: + config: 'coreboot-linuxboot-example.json' + target: 'coreboot-example-with-linuxboot' + recursive: 'true' + debug: 'true' + auto-cache: 'true' + auto-artifact-download: 'true' + auto-artifact-upload: 'true' diff --git a/docs/src/firmware-action/migration/v0.14.x--v0.15.0/old.yml b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/old.yml new file mode 100644 index 00000000..24dc50cf --- /dev/null +++ b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/old.yml @@ -0,0 +1,128 @@ +--- +name: linuxboot build +on: + push: + +permissions: + contents: read + +jobs: + build-coreboot-linuxboot-example: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Fetch few missing submodules + run: | + git submodule update --depth 1 --init --recursive --checkout + + #================================== + # Get commit hashes for submodules + #================================== + + - name: Extract uroot commit sha + id: uroot_commit + run: | + echo "uroot_commit=$( git rev-parse HEAD:coreboot-linuxboot-example/u-root )" >> "${GITHUB_OUTPUT}" + + - name: Extract Linux commit sha + id: linux_commit + run: | + echo "linux_commit=$( git rev-parse HEAD:coreboot-linuxboot-example/linux )" >> "${GITHUB_OUTPUT}" + + - name: Extract Coreboot commit sha + id: coreboot_commit + run: | + echo "coreboot_commit=$( git rev-parse HEAD:coreboot-linuxboot-example/coreboot )" >> "${GITHUB_OUTPUT}" + + #=============== + # Restore cache + #=============== + + - name: Restore cached u-root artefact + uses: actions/cache/restore@v4 + id: cache-uroot + with: + path: output-linuxboot-uroot + key: uroot-${{ steps.uroot_commit.outputs.uroot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json') }} + + - name: Restore cached Linux artefact + uses: actions/cache/restore@v4 + id: cache-linux + with: + path: output-linuxboot-linux + key: linux-${{ steps.linux_commit.outputs.linux_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/linux_defconfig', 'output-linuxboot-uroot/*') }} + + - name: Restore cached coreboot artefact + uses: actions/cache/restore@v4 + id: cache-coreboot + with: + path: output-linuxboot-coreboot + key: coreboot-${{ steps.coreboot_commit.outputs.coreboot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/coreboot_linuxboot_defconfig', 'output-linuxboot-linux/*') }} + + #============================ + # Build with firmware-action + #============================ + + - name: firmware-action + uses: 9elements/firmware-action@v0.14.1 + with: + config: 'coreboot-linuxboot-example.json' + target: 'coreboot-example-with-linuxboot' + recursive: 'true' + + #========================== + # Upload artifacts - uroot + #========================== + + - name: Cache u-root + uses: actions/cache/save@v4 + if: always() + with: + key: uroot-${{ steps.uroot_commit.outputs.uroot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json') }} + path: output-linuxboot-uroot + + - name: Upload artifacts for uroot + uses: actions/upload-artifact@v4 + if: always() + with: + name: linuxboot-uroot + path: output-linuxboot-uroot + + #========================== + # Upload artifacts - Linux + #========================== + + - name: Cache Linux + uses: actions/cache/save@v4 + if: always() + with: + key: linux-${{ steps.linux_commit.outputs.linux_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/linux_defconfig', 'output-linuxboot-uroot/*') }} + path: output-linuxboot-linux + + - name: Upload artifacts for Linux + uses: actions/upload-artifact@v4 + if: always() + with: + name: linuxboot-linux + path: output-linuxboot-linux + + #============================= + # Upload artifacts - coreboot + #============================= + + - name: Cache coreboot + uses: actions/cache/save@v4 + if: always() + with: + key: coreboot-${{ steps.coreboot_commit.outputs.coreboot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/coreboot_linuxboot_defconfig', 'output-linuxboot-linux/*') }} + path: output-linuxboot-coreboot + + - name: Upload artifacts for coreboot + uses: actions/upload-artifact@v4 + if: always() + with: + name: linuxboot-coreboot + path: output-linuxboot-coreboot diff --git a/docs/src/firmware-action/migration/v0.14.x--v0.15.0/workflow.yml.patch b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/workflow.yml.patch new file mode 100644 index 00000000..35f07e5d --- /dev/null +++ b/docs/src/firmware-action/migration/v0.14.x--v0.15.0/workflow.yml.patch @@ -0,0 +1,132 @@ + --- + name: linuxboot build + on: + push: + + permissions: + contents: read + + jobs: + build-coreboot-linuxboot-example: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Fetch few missing submodules + run: | + git submodule update --depth 1 --init --recursive --checkout + +- #================================== +- # Get commit hashes for submodules +- #================================== +- +- - name: Extract uroot commit sha +- id: uroot_commit +- run: | +- echo "uroot_commit=$( git rev-parse HEAD:coreboot-linuxboot-example/u-root )" >> "${GITHUB_OUTPUT}" +- +- - name: Extract Linux commit sha +- id: linux_commit +- run: | +- echo "linux_commit=$( git rev-parse HEAD:coreboot-linuxboot-example/linux )" >> "${GITHUB_OUTPUT}" +- +- - name: Extract Coreboot commit sha +- id: coreboot_commit +- run: | +- echo "coreboot_commit=$( git rev-parse HEAD:coreboot-linuxboot-example/coreboot )" >> "${GITHUB_OUTPUT}" +- +- #=============== +- # Restore cache +- #=============== +- +- - name: Restore cached u-root artefact +- uses: actions/cache/restore@v4 +- id: cache-uroot +- with: +- path: output-linuxboot-uroot +- key: uroot-${{ steps.uroot_commit.outputs.uroot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json') }} +- +- - name: Restore cached Linux artefact +- uses: actions/cache/restore@v4 +- id: cache-linux +- with: +- path: output-linuxboot-linux +- key: linux-${{ steps.linux_commit.outputs.linux_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/linux_defconfig', 'output-linuxboot-uroot/*') }} +- +- - name: Restore cached coreboot artefact +- uses: actions/cache/restore@v4 +- id: cache-coreboot +- with: +- path: output-linuxboot-coreboot +- key: coreboot-${{ steps.coreboot_commit.outputs.coreboot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/coreboot_linuxboot_defconfig', 'output-linuxboot-linux/*') }} +- +- #============================ +- # Build with firmware-action +- #============================ +- + - name: firmware-action +- uses: 9elements/firmware-action@v0.14.1 ++ uses: 9elements/firmware-action@v0.15.0 + with: + config: 'coreboot-linuxboot-example.json' + target: 'coreboot-example-with-linuxboot' + recursive: 'true' ++ auto-cache: 'true' ++ auto-artifact-download: 'true' ++ auto-artifact-upload: 'true' +- +- #========================== +- # Upload artifacts - uroot +- #========================== +- +- - name: Cache u-root +- uses: actions/cache/save@v4 +- if: always() +- with: +- key: uroot-${{ steps.uroot_commit.outputs.uroot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json') }} +- path: output-linuxboot-uroot +- +- - name: Upload artifacts for uroot +- uses: actions/upload-artifact@v4 +- if: always() +- with: +- name: linuxboot-uroot +- path: output-linuxboot-uroot +- +- #========================== +- # Upload artifacts - Linux +- #========================== +- +- - name: Cache Linux +- uses: actions/cache/save@v4 +- if: always() +- with: +- key: linux-${{ steps.linux_commit.outputs.linux_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/linux_defconfig', 'output-linuxboot-uroot/*') }} +- path: output-linuxboot-linux +- +- - name: Upload artifacts for Linux +- uses: actions/upload-artifact@v4 +- if: always() +- with: +- name: linuxboot-linux +- path: output-linuxboot-linux +- +- #============================= +- # Upload artifacts - coreboot +- #============================= +- +- - name: Cache coreboot +- uses: actions/cache/save@v4 +- if: always() +- with: +- key: coreboot-${{ steps.coreboot_commit.outputs.coreboot_commit }}-${{ hashFiles('coreboot-linuxboot-example.json', 'coreboot-linuxboot-example/coreboot_linuxboot_defconfig', 'output-linuxboot-linux/*') }} +- path: output-linuxboot-coreboot +- +- - name: Upload artifacts for coreboot +- uses: actions/upload-artifact@v4 +- if: always() +- with: +- name: linuxboot-coreboot +- path: output-linuxboot-coreboot diff --git a/docs/src/firmware-action/usage_github.md b/docs/src/firmware-action/usage_github.md index 715c5274..1326cab8 100644 --- a/docs/src/firmware-action/usage_github.md +++ b/docs/src/firmware-action/usage_github.md @@ -1,57 +1,89 @@ # Github CI -You can use `firmware-action` as any other action. +You can use `firmware-action` as any other GitHub action. The action requires a configuration file (`config`) and target (`target`) to build, with additional options for artifact handling and caching to improve build times. + +## Artifacts and Outputs + +The action can automatically upload build artifacts when the `auto-artifact-upload` option is enabled (disabled by default). When enabled, the action provides the following outputs: + +- `artifact-name`: Name of the uploaded artifact +- `artifact-id`: GitHub ID of the artifact (useful for REST API) +- `artifact-url`: Direct download URL for the artifact (requires GitHub login) +- `artifact-digest`: SHA-256 digest of the artifact + +Internally, `firmware-action` uses the [actions/upload-artifact@v4](https://github.com/actions/upload-artifact)@v4 action to handle artifact uploads and [actions/download-artifact@v4](https://github.com/actions/download-artifact) to handle artifact downloads. ```admonish example +You can use these outputs in subsequent steps: ~~~yaml -name: Firmware example build -jobs: - firmware_build: - runs-on: ubuntu-latest - steps: - - name: Build coreboot with firmware-action - uses: 9elements/firmware-action@main - with: - config: '' - target: '' - recursive: 'false' +{{#include ../firmware-action-example/.github/workflows/coreboot-example.yml:CorebootExample}} ~~~ ``` +You can configure artifact handling with these options: +- `artifact-if-no-files-found`: Behavior if no files are found ('warn', 'error', or 'ignore', default: 'warn') +- `artifact-compression-level`: Compression level for artifacts (0-9, default: '6') +- `artifact-overwrite`: Whether to overwrite existing artifacts with the same name (default: 'false') + +## Build Caching + +The action can cache build artifacts between runs to speed up builds, but this feature must be explicitly enabled with the `auto-cache` input (disabled by default). + +When enabled, the cache is: +- Keyed by the config file contents and commit SHA +- Restored at the start of each run +- Saved after each run, even if the build fails + +You can also use the `recursive` option to build a target with all its dependencies, and the `debug` option for increased verbosity when troubleshooting. + +## Artifact Downloading + +When `auto-artifact-download` is enabled (disabled by default), the action automatically downloads all artifacts from the current workflow run. This feature is particularly useful in workflows with multiple jobs that depend on each other, as it can save a lot of copy-pasting between workflow steps. + +```admonish note +Due to limitations in GitHub Actions, it's not possible for the action to download only relevant artifacts. When enabled, it downloads all artifacts from the current workflow run regardless of whether they're needed for the current build. +``` + +```admonish warning +When using `auto-artifact-download`, be careful to avoid naming conflicts in the `output_dir` values across different targets. Since all artifacts are downloaded and merged into the same workspace, files with the same paths will overwrite each other. +``` + +```admonish warning +At the time of writing, GitHub-hosted runners have only 14GB of disk space. If your artifacts are large and/or you have many matrix combinations, the workflow could run out of disk space. Monitor your disk usage when using this feature with large builds. +``` + +## Complete Configuration Example + +For a complete example with all options: + +```admonish example +~~~yaml +{{#include ../firmware-action-example/.github/workflows/linuxboot-example.yml:AllFeatures}} +~~~ +``` ## Parametric builds with environment variables To take advantage of matrix builds in GitHub, it is possible to use environment variables inside the JSON configuration file. ```admonish example -For example let's make `COREBOOT_VERSION` environment variable which will hold version of coreboot. +For example let's make `RELEASE_TYPE` environment variable which will hold either `DEBUG` or `RELEASE`. JSON would look like this: ~~~json -... -"sdk_url": "ghcr.io/9elements/firmware-action/coreboot_${COREBOOT_VERSION}:main", -... -"defconfig_path": "tests/coreboot_${COREBOOT_VERSION}/seabios.defconfig", -... +{ + "u-root": { + "uroot-example-${RELEASE_TYPE}": { + ... + "output_dir": "output-linuxboot-uroot-${RELEASE_TYPE}/", + ... + } + } +} ~~~ YAML would look like this: ~~~yaml -name: Firmware example build -jobs: - firmware_build: - runs-on: ubuntu-latest - strategy: - matrix: - coreboot_version: ["4.19", "4.20"] - steps: - - name: Build coreboot with firmware-action - uses: 9elements/firmware-action@main - with: - config: '' - target: '' - recursive: 'false' - env: - COREBOOT_VERSION: ${{ matrix.coreboot_version }} +{{#include ../firmware-action-example/.github/workflows/linuxboot-example-separate-jobs.yml:Parametric}} ~~~ ``` diff --git a/tests/Taskfile.yml b/tests/Taskfile.yml index cbf02061..bda04fe3 100644 --- a/tests/Taskfile.yml +++ b/tests/Taskfile.yml @@ -43,15 +43,37 @@ tasks: internal: true cmds: - cp -f "../tests/linux_{{.LINUX_VERSION}}/linux.defconfig" "ci_defconfig" - - ../bin/firmware-action-linux-amd64-v{{.VERSION}} build --config="../tests/example_config__linux.json" --target=linux-example + - ../bin/firmware-action build --config="../tests/example_config__linux.json" --target=linux-example env: LINUX_VERSION: '{{.LINUX_VERSION}}' SYSTEM_ARCH: 'amd64' + UROOT_VERSION: "" requires: vars: [LINUX_VERSION] - linux-*: - desc: Run example + linux-run-github: + desc: Run firmware-action just like in GitHub runner + dir: '{{.WORKDIR}}' + internal: true + cmds: + - cp -f "../tests/linux_{{.LINUX_VERSION}}/linux.defconfig" "ci_defconfig" + - ../bin/firmware-action + env: + LINUX_VERSION: '{{.LINUX_VERSION}}' + SYSTEM_ARCH: 'amd64' + UROOT_VERSION: "" + # Fake GitHub runner + GITHUB_ACTIONS: "yes" + INPUT_TARGET: "linux-example" + INPUT_RECURSIVE: "false" + # For testing multiple config inputs + #INPUT_CONFIG: " ../tests/example_config__linux.json\n ../tests/example_config__uroot.json \n " + INPUT_CONFIG: "../tests/example_config__linux.json" + requires: + vars: [LINUX_VERSION] + + linux-local-*: + desc: Run example as it would run locally deps: - task: mkdir - task: :build-go-binary @@ -66,6 +88,22 @@ tasks: vars: LINUX_VERSION: '{{.VARIANT}}' + linux-github-*: + desc: Run example as it would run in GitHub CI + deps: + - task: mkdir + - task: :build-go-binary + vars: + VARIANT: '{{index .MATCH 0}}' + cmds: + - task: linux-fetch + vars: + LINUX_VERSION: '{{.VARIANT}}' + - task: linux-run-github + vars: + LINUX_VERSION: '{{.VARIANT}}' + + #======== # U-BOOT #======== @@ -88,7 +126,7 @@ tasks: cmds: - cp -f "../tests/uboot_{{.UBOOT_VERSION}}/uboot.defconfig" "uboot_defconfig" - ln -sf "uboot-{{.UBOOT_VERSION}}" u-boot - - ../bin/firmware-action-linux-amd64-v{{.VERSION}} build --config="../tests/example_config__uboot.json" --target=u-boot-example + - ../bin/firmware-action build --config="../tests/example_config__uboot.json" --target=u-boot-example env: UBOOT_VERSION: '{{.UBOOT_VERSION}}' SYSTEM_ARCH: 'arm64'