diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml
index 1ba9e7db..fa202ffc 100644
--- a/.github/workflows/example.yml
+++ b/.github/workflows/example.yml
@@ -544,6 +544,45 @@ jobs:
retention-days: 14
# ANCHOR_END: example_build_universal
+ # Example of building with pruning enabled
+ # ANCHOR: example_build_prune
+ build-with-pruning:
+ needs:
+ - changes
+ - skip-check
+ runs-on: ubuntu-latest
+ strategy:
+ fail-fast: false
+ matrix:
+ prune: ['true', 'false']
+ if: ${{ ! (github.event_name == 'pull_request_review' && github.actor != 'github-actions[bot]') && needs.skip-check.outputs.changes == 'true' }}
+ # Skip if pull_request_review on PR not made by a bot
+ steps:
+ - name: Cleanup
+ run: |
+ rm -rf ./* || true
+ rm -rf ./.??* || true
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: firmware-action
+ uses: ./
+ # uses: 9elements/firmware-action
+ with:
+ config: 'tests/example_config__depends.json'
+ target: 'universal-example-B'
+ recursive: 'true'
+ prune: ${{ matrix.prune }}
+ compile: ${{ needs.changes.outputs.compile }}
+
+ - name: Get artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: prune
+ path: output-universal-example-B-${{ matrix.prune }}
+ retention-days: 14
+ # ANCHOR_END: example_build_prune
+
# Example of running on non-Linux systems
test-operating-systems:
strategy:
diff --git a/action.yml b/action.yml
index fed3e739..e8c5836a 100644
--- a/action.yml
+++ b/action.yml
@@ -17,6 +17,12 @@ inputs:
Build target recursively, with all of its dependencies.
required: false
default: 'false'
+ prune:
+ description: |
+ Remove Dagger container and its volumes after each module (only in recursive mode).
+ Enable this when building complex firmware stack in single job recursively and you are running out of disk space.
+ required: false
+ default: 'false'
compile:
description: |
Compile the action from source instead of downloading pre-compiled binary from releases.
@@ -119,6 +125,7 @@ runs:
INPUT_CONFIG: ${{ inputs.config }}
INPUT_TARGET: ${{ inputs.target }}
INPUT_RECURSIVE: ${{ inputs.recursive }}
+ INPUT_PRUNE: ${{ inputs.prune }}
- name: run_windows
if: ${{ runner.os == 'Windows' }}
@@ -129,3 +136,4 @@ runs:
INPUT_CONFIG: ${{ inputs.config }}
INPUT_TARGET: ${{ inputs.target }}
INPUT_RECURSIVE: ${{ inputs.recursive }}
+ INPUT_PRUNE: ${{ inputs.prune }}
diff --git a/cmd/firmware-action/container/container.go b/cmd/firmware-action/container/container.go
index cc06f0c6..7d704cfc 100644
--- a/cmd/firmware-action/container/container.go
+++ b/cmd/firmware-action/container/container.go
@@ -9,6 +9,7 @@ import (
"fmt"
"log/slog"
"os"
+ "os/exec"
"path/filepath"
"regexp"
"runtime"
@@ -312,3 +313,123 @@ func GetArtifacts(ctx context.Context, container *dagger.Container, artifacts *[
return nil
}
+
+// CleanupAfterContainer performs cleanup operations after container use
+func CleanupAfterContainer(ctx context.Context) error {
+ // Unfortunately it is not possible to only remove the container used for building the module.
+ // Dagger Engine somehow absorbs the other containers into itself (possibly into it's volume, not sure).
+ // So to actually free up a disk space by deleting a container we have to delete the whole dagger engine
+ // container and it's volume.
+ //
+ // This function is used to free up disk space on constrained environments like GitHub Actions.
+ // GitHub-hosted public runners have only 14GB of disk space available.
+ // If user wants to build complex firmware stacks in single job recursively, they will easily run
+ // out of disk space.
+ //
+ // WARNING: This will completely stop the Dagger engine. Any subsequent Dagger
+ // operations will need to reinitialize the Dagger client.
+
+ slog.Info("Cleaning up Dagger container resources")
+
+ // Step 1: Find the Dagger engine container
+ findCmd := exec.CommandContext(ctx, "docker", "container", "ls", "--filter", "name=dagger-engine", "--format", "{{.ID}}")
+ containerID, err := findCmd.Output()
+ if err != nil {
+ slog.Error(
+ "Failed to find Dagger engine container",
+ slog.Any("error", err),
+ )
+ return err
+ }
+
+ containerIDStr := strings.TrimSpace(string(containerID))
+ if containerIDStr == "" {
+ slog.Info("No Dagger engine container found to clean up")
+ return nil
+ }
+
+ // Step 2: Stop the Dagger engine container
+ slog.Debug(
+ "Stopping Dagger engine container",
+ slog.String("containerID", containerIDStr),
+ )
+ stopCmd := exec.CommandContext(ctx, "docker", "container", "stop", containerIDStr)
+ stopOutput, err := stopCmd.CombinedOutput()
+ if err != nil {
+ slog.Error(
+ "Failed to stop Dagger engine container",
+ slog.String("output", strings.TrimSpace(string(stopOutput))),
+ slog.Any("error", err),
+ )
+ return err
+ }
+
+ // Step 3: Remove the Dagger engine container
+ slog.Debug(
+ "Removing Dagger engine container",
+ slog.String("containerID", containerIDStr),
+ )
+ rmCmd := exec.CommandContext(ctx, "docker", "container", "rm", containerIDStr)
+ rmOutput, err := rmCmd.CombinedOutput()
+ if err != nil {
+ slog.Error(
+ "Failed to remove Dagger engine container",
+ slog.String("output", strings.TrimSpace(string(rmOutput))),
+ slog.Any("error", err),
+ )
+ return err
+ }
+
+ // Step 4: Find and remove Dagger volumes
+ volCmd := exec.CommandContext(ctx, "docker", "volume", "ls", "--filter", "dangling=true", "--format", "{{.Name}}")
+ volumes, err := volCmd.Output()
+ if err != nil {
+ slog.Warn(
+ "Failed to list Docker volumes",
+ slog.Any("error", err),
+ )
+ // Continue even if this fails
+ } else {
+ volumeList := strings.Split(strings.TrimSpace(string(volumes)), "\n")
+ for _, vol := range volumeList {
+ if vol == "" {
+ continue
+ }
+ slog.Debug(
+ "Removing Docker volume",
+ slog.String("volume", vol),
+ )
+ rmVolCmd := exec.CommandContext(ctx, "docker", "volume", "rm", vol)
+ rmVolOutput, err := rmVolCmd.CombinedOutput()
+ if err != nil {
+ slog.Warn(
+ "Failed to remove Docker volume",
+ slog.String("volume", vol),
+ slog.String("output", strings.TrimSpace(string(rmVolOutput))),
+ slog.Any("error", err),
+ )
+ // Continue with other volumes even if one fails
+ }
+ }
+ }
+
+ // Step 5: Run system prune to clean up any remaining resources
+ pruneCmd := exec.CommandContext(ctx, "docker", "system", "prune", "-f")
+ pruneOutput, err := pruneCmd.CombinedOutput()
+ slog.Debug(
+ "Docker system prune output",
+ slog.String("command", "docker system prune -f"),
+ slog.String("output", strings.TrimSpace(string(pruneOutput))),
+ )
+ if err != nil {
+ slog.Error(
+ "Failed to prune Docker system",
+ slog.String("output", strings.TrimSpace(string(pruneOutput))),
+ slog.Any("error", err),
+ )
+ return err
+ }
+
+ slog.Info("Dagger container resources cleaned up successfully")
+ return nil
+}
diff --git a/cmd/firmware-action/environment/environment.go b/cmd/firmware-action/environment/environment.go
index 61219afe..28b33ab4 100644
--- a/cmd/firmware-action/environment/environment.go
+++ b/cmd/firmware-action/environment/environment.go
@@ -21,3 +21,11 @@ func FetchEnvVars(variables []string) map[string]string {
return result
}
+
+// DetectGithub function returns True when the execution environment is detected to be GitHub CI
+func DetectGithub() bool {
+ // Check for GitHub
+ // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+ _, exists := os.LookupEnv("GITHUB_ACTIONS")
+ return exists
+}
diff --git a/cmd/firmware-action/main.go b/cmd/firmware-action/main.go
index 2eda5600..433553f4 100644
--- a/cmd/firmware-action/main.go
+++ b/cmd/firmware-action/main.go
@@ -14,6 +14,7 @@ import (
"regexp"
"strings"
+ "github.com/9elements/firmware-action/cmd/firmware-action/environment"
"github.com/9elements/firmware-action/cmd/firmware-action/filesystem"
"github.com/9elements/firmware-action/cmd/firmware-action/logging"
"github.com/9elements/firmware-action/cmd/firmware-action/recipes"
@@ -50,8 +51,9 @@ var CLI struct {
Config []string `type:"path" required:"" default:"${config_file}" help:"Path to configuration file, supports multiple flags to use multiple configuration files"`
Build struct {
- Target string `required:"" help:"Select which target to build, use ID from configuration file"`
- Recursive bool `help:"Build recursively with all dependencies and payloads"`
+ Target string `required:"" help:"Select which target to build, use ID from configuration file"`
+ Recursive bool `help:"Build recursively with all dependencies and payloads"`
+ PruneDockerContainers bool `help:"Remove Dagger container and its volumes after each module (only in recursive mode)"`
} `cmd:"build" help:"Build a target defined in configuration file. For interactive debugging preface the command with 'dagger run --interactive', for example 'dagger run --interactive $(which firmware-action) build --config=...'. To install dagger follow instructions at https://dagger.io/"`
GenerateConfig struct{} `cmd:"generate-config" help:"Generate empty configuration file"`
@@ -85,6 +87,7 @@ func run(ctx context.Context) error {
slog.Any("input/config", CLI.Config),
slog.String("input/target", CLI.Build.Target),
slog.Bool("input/recursive", CLI.Build.Recursive),
+ slog.Bool("input/prune", CLI.Build.PruneDockerContainers),
)
// Check if submodules were initialized
@@ -133,6 +136,7 @@ submodule_out:
ctx,
CLI.Build.Target,
CLI.Build.Recursive,
+ CLI.Build.PruneDockerContainers,
myConfig,
recipes.Execute,
)
@@ -163,9 +167,7 @@ submodule_out:
func getInputsFromEnvironment() (string, error) {
// Check for GitHub
- // https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
- _, exists := os.LookupEnv("GITHUB_ACTIONS")
- if exists {
+ if environment.DetectGithub() {
return parseGithub()
}
@@ -292,6 +294,7 @@ func parseGithub() (string, error) {
CLI.Config = strings.Split(action.GetInput("config"), "\n")
CLI.Build.Target = action.GetInput("target")
CLI.Build.Recursive = regexTrue.MatchString(action.GetInput("recursive"))
+ CLI.Build.PruneDockerContainers = regexTrue.MatchString(action.GetInput("prune"))
CLI.JSON = regexTrue.MatchString(action.GetInput("json"))
return "GitHub", nil
diff --git a/cmd/firmware-action/recipes/recipes.go b/cmd/firmware-action/recipes/recipes.go
index 51885410..ae152107 100644
--- a/cmd/firmware-action/recipes/recipes.go
+++ b/cmd/firmware-action/recipes/recipes.go
@@ -15,6 +15,7 @@ import (
"sync"
"dagger.io/dagger"
+ "github.com/9elements/firmware-action/cmd/firmware-action/container"
"github.com/9elements/firmware-action/cmd/firmware-action/filesystem"
"github.com/heimdalr/dag"
)
@@ -67,6 +68,7 @@ func Build(
ctx context.Context,
target string,
recursive bool,
+ pruneDocker bool,
config *Config,
executor func(context.Context, string, *Config) error,
) ([]BuildResults, error) {
@@ -139,6 +141,16 @@ func Build(
if err != nil && !errors.Is(err, ErrBuildUpToDate) {
break
}
+
+ // Prune the Dagger Engine to free disk space
+ // It is fine to do here because the `executor` function connects and disconnects
+ // to Dagger Engine (it is self-contained)
+ if pruneDocker {
+ err = container.CleanupAfterContainer(ctx)
+ if err != nil {
+ break
+ }
+ }
}
} else {
// else build only the target
diff --git a/cmd/firmware-action/recipes/recipes_test.go b/cmd/firmware-action/recipes/recipes_test.go
index ec6a44b4..c6f23fc3 100644
--- a/cmd/firmware-action/recipes/recipes_test.go
+++ b/cmd/firmware-action/recipes/recipes_test.go
@@ -250,6 +250,7 @@ func TestBuild(t *testing.T) {
ctx,
tc.target,
tc.recursive,
+ false, // do not prune
&tc.config,
executeDummy,
)
@@ -262,6 +263,7 @@ func TestBuild(t *testing.T) {
ctx,
"pizza",
recursive,
+ false, // do not prune
&testConfigDependencyHell,
executeDummy,
)
diff --git a/tests/example_config.json b/tests/example_config.json
index ac5bc731..15e713f7 100644
--- a/tests/example_config.json
+++ b/tests/example_config.json
@@ -17,6 +17,44 @@
"input_files": null
}
},
+ "universal": {
+ "universal-example-A": {
+ "depends": null,
+ "sdk_url": "golang:latest",
+ "repo_path": "./",
+ "container_output_dirs": null,
+ "container_output_files": [
+ "test.txt"
+ ],
+ "output_dir": "output-universal-example-A/",
+ "input_dirs": null,
+ "input_files": null,
+ "container_input_dir": "inputs/",
+ "build_commands": [
+ "echo 'hello world'",
+ "touch test.txt"
+ ]
+ },
+ "universal-example-B": {
+ "depends": [
+ "universal-example-A"
+ ],
+ "sdk_url": "golang:latest",
+ "repo_path": "./",
+ "container_output_dirs": null,
+ "container_output_files": [
+ "test.txt"
+ ],
+ "output_dir": "output-universal-example-B/",
+ "input_dirs": null,
+ "input_files": null,
+ "container_input_dir": "inputs/",
+ "build_commands": [
+ "echo 'hello world'",
+ "touch test.txt"
+ ]
+ }
+ },
"edk2": {
"edk2-example": {
"depends": null,
diff --git a/tests/example_config__depends.json b/tests/example_config__depends.json
new file mode 100644
index 00000000..437b5851
--- /dev/null
+++ b/tests/example_config__depends.json
@@ -0,0 +1,40 @@
+{
+ "universal": {
+ "universal-example-A": {
+ "depends": null,
+ "sdk_url": "golang:latest",
+ "repo_path": "./",
+ "container_output_dirs": null,
+ "container_output_files": [
+ "test.txt"
+ ],
+ "output_dir": "output-universal-example-A/",
+ "input_dirs": null,
+ "input_files": null,
+ "container_input_dir": "inputs/",
+ "build_commands": [
+ "echo 'hello world'",
+ "touch test.txt"
+ ]
+ },
+ "universal-example-B": {
+ "depends": [
+ "universal-example-A"
+ ],
+ "sdk_url": "golang:latest",
+ "repo_path": "./",
+ "container_output_dirs": null,
+ "container_output_files": [
+ "test.txt"
+ ],
+ "output_dir": "output-universal-example-B/",
+ "input_dirs": null,
+ "input_files": null,
+ "container_input_dir": "inputs/",
+ "build_commands": [
+ "echo 'hello world'",
+ "touch test.txt"
+ ]
+ }
+ }
+}