feat: delete container after use

.github/workflows/example.yml

@@ -544,6 +544,45 @@ jobs:
           retention-days: 14
   # ANCHOR_END: example_build_universal
 
+  # Example of building with pruning enabled
+  # ANCHOR: example_build_prune
+  build-with-pruning:
+    needs:
+      - changes
+      - skip-check
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        prune: ['true', 'false']
+    if: ${{ ! (github.event_name == 'pull_request_review' && github.actor != 'github-actions[bot]') && needs.skip-check.outputs.changes == 'true' }}
+    # Skip if pull_request_review on PR not made by a bot
+    steps:
+      - name: Cleanup
+        run: |
+          rm -rf ./* || true
+          rm -rf ./.??* || true
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: firmware-action
+        uses: ./
+        # uses: 9elements/firmware-action
+        with:
+          config: 'tests/example_config__depends.json'
+          target: 'universal-example-B'
+          recursive: 'true'
+          prune: ${{ matrix.prune }}
+          compile: ${{ needs.changes.outputs.compile }}
+
+      - name: Get artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: prune
+          path: output-universal-example-B-${{ matrix.prune }}
+          retention-days: 14
+  # ANCHOR_END: example_build_prune
+
   # Example of running on non-Linux systems
   test-operating-systems:
     strategy:

action.yml

@@ -17,6 +17,12 @@ inputs:
       Build target recursively, with all of its dependencies.
     required: false
     default: 'false'
+  prune:
+    description: |
+      Remove Dagger container and its volumes after each module (only in recursive mode).
+      Enable this when building complex firmware stack in single job recursively and you are running out of disk space.
+    required: false
+    default: 'false'
   compile:
     description: |
       Compile the action from source instead of downloading pre-compiled binary from releases.
@@ -119,6 +125,7 @@ runs:
         INPUT_CONFIG: ${{ inputs.config }}
         INPUT_TARGET: ${{ inputs.target }}
         INPUT_RECURSIVE: ${{ inputs.recursive }}
+        INPUT_PRUNE: ${{ inputs.prune }}
 
     - name: run_windows
       if: ${{ runner.os == 'Windows' }}
@@ -129,3 +136,4 @@ runs:
         INPUT_CONFIG: ${{ inputs.config }}
         INPUT_TARGET: ${{ inputs.target }}
         INPUT_RECURSIVE: ${{ inputs.recursive }}
+        INPUT_PRUNE: ${{ inputs.prune }}

cmd/firmware-action/container/container.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -312,3 +313,123 @@ func GetArtifacts(ctx context.Context, container *dagger.Container, artifacts *[
 
 	return nil
 }
+
+// CleanupAfterContainer performs cleanup operations after container use
+func CleanupAfterContainer(ctx context.Context) error {
+	// Unfortunately it is not possible to only remove the container used for building the module.
+	// Dagger Engine somehow absorbs the other containers into itself (possibly into it's volume, not sure).
+	// So to actually free up a disk space by deleting a container we have to delete the whole dagger engine
+	//   container and it's volume.
+	//
+	// This function is used to free up disk space on constrained environments like GitHub Actions.
+	//   GitHub-hosted public runners have only 14GB of disk space available.
+	// If user wants to build complex firmware stacks in single job recursively, they will easily run
+	//   out of disk space.
+	//
+	// WARNING: This will completely stop the Dagger engine. Any subsequent Dagger
+	//   operations will need to reinitialize the Dagger client.
+
+	slog.Info("Cleaning up Dagger container resources")
+
+	// Step 1: Find the Dagger engine container
+	findCmd := exec.CommandContext(ctx, "docker", "container", "ls", "--filter", "name=dagger-engine", "--format", "{{.ID}}")
+	containerID, err := findCmd.Output()
+	if err != nil {
+		slog.Error(
+			"Failed to find Dagger engine container",
+			slog.Any("error", err),
+		)
+		return err
+	}
+
+	containerIDStr := strings.TrimSpace(string(containerID))
+	if containerIDStr == "" {
+		slog.Info("No Dagger engine container found to clean up")
+		return nil
+	}
+
+	// Step 2: Stop the Dagger engine container
+	slog.Debug(
+		"Stopping Dagger engine container",
+		slog.String("containerID", containerIDStr),
+	)
+	stopCmd := exec.CommandContext(ctx, "docker", "container", "stop", containerIDStr)
+	stopOutput, err := stopCmd.CombinedOutput()
+	if err != nil {
+		slog.Error(
+			"Failed to stop Dagger engine container",
+			slog.String("output", strings.TrimSpace(string(stopOutput))),
+			slog.Any("error", err),
+		)
+		return err
+	}
+
+	// Step 3: Remove the Dagger engine container
+	slog.Debug(
+		"Removing Dagger engine container",
+		slog.String("containerID", containerIDStr),
+	)
+	rmCmd := exec.CommandContext(ctx, "docker", "container", "rm", containerIDStr)
+	rmOutput, err := rmCmd.CombinedOutput()
+	if err != nil {
+		slog.Error(
+			"Failed to remove Dagger engine container",
+			slog.String("output", strings.TrimSpace(string(rmOutput))),
+			slog.Any("error", err),
+		)
+		return err
+	}
+
+	// Step 4: Find and remove Dagger volumes
+	volCmd := exec.CommandContext(ctx, "docker", "volume", "ls", "--filter", "dangling=true", "--format", "{{.Name}}")
+	volumes, err := volCmd.Output()
+	if err != nil {
+		slog.Warn(
+			"Failed to list Docker volumes",
+			slog.Any("error", err),
+		)
+		// Continue even if this fails
+	} else {
+		volumeList := strings.Split(strings.TrimSpace(string(volumes)), "\n")
+		for _, vol := range volumeList {
+			if vol == "" {
+				continue
+			}
+			slog.Debug(
+				"Removing Docker volume",
+				slog.String("volume", vol),
+			)
+			rmVolCmd := exec.CommandContext(ctx, "docker", "volume", "rm", vol)
+			rmVolOutput, err := rmVolCmd.CombinedOutput()
+			if err != nil {
+				slog.Warn(
+					"Failed to remove Docker volume",
+					slog.String("volume", vol),
+					slog.String("output", strings.TrimSpace(string(rmVolOutput))),
+					slog.Any("error", err),
+				)
+				// Continue with other volumes even if one fails
+			}
+		}
+	}
+
+	// Step 5: Run system prune to clean up any remaining resources
+	pruneCmd := exec.CommandContext(ctx, "docker", "system", "prune", "-f")
+	pruneOutput, err := pruneCmd.CombinedOutput()
+	slog.Debug(
+		"Docker system prune output",
+		slog.String("command", "docker system prune -f"),
+		slog.String("output", strings.TrimSpace(string(pruneOutput))),
+	)
+	if err != nil {
+		slog.Error(
+			"Failed to prune Docker system",
+			slog.String("output", strings.TrimSpace(string(pruneOutput))),
+			slog.Any("error", err),
+		)
+		return err
+	}
+
+	slog.Info("Dagger container resources cleaned up successfully")
+	return nil
+}

cmd/firmware-action/environment/environment.go

@@ -21,3 +21,11 @@ func FetchEnvVars(variables []string) map[string]string {
 
 	return result
 }
+
+// DetectGithub function returns True when the execution environment is detected to be GitHub CI
+func DetectGithub() bool {
+	// Check for GitHub
+	// https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
+	_, exists := os.LookupEnv("GITHUB_ACTIONS")
+	return exists
+}

cmd/firmware-action/main.go

@@ -14,6 +14,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/9elements/firmware-action/cmd/firmware-action/environment"
 	"github.com/9elements/firmware-action/cmd/firmware-action/filesystem"
 	"github.com/9elements/firmware-action/cmd/firmware-action/logging"
 	"github.com/9elements/firmware-action/cmd/firmware-action/recipes"
@@ -50,8 +51,9 @@ var CLI struct {
 	Config []string `type:"path" required:"" default:"${config_file}" help:"Path to configuration file, supports multiple flags to use multiple configuration files"`
 
 	Build struct {
-		Target    string `required:"" help:"Select which target to build, use ID from configuration file"`
-		Recursive bool   `help:"Build recursively with all dependencies and payloads"`
+		Target                string `required:"" help:"Select which target to build, use ID from configuration file"`
+		Recursive             bool   `help:"Build recursively with all dependencies and payloads"`
+		PruneDockerContainers bool   `help:"Remove Dagger container and its volumes after each module (only in recursive mode)"`
 	} `cmd:"build" help:"Build a target defined in configuration file. For interactive debugging preface the command with 'dagger run --interactive', for example 'dagger run --interactive $(which firmware-action) build --config=...'. To install dagger follow instructions at https://dagger.io/"`
 
 	GenerateConfig struct{} `cmd:"generate-config" help:"Generate empty configuration file"`
@@ -85,6 +87,7 @@ func run(ctx context.Context) error {
 		slog.Any("input/config", CLI.Config),
 		slog.String("input/target", CLI.Build.Target),
 		slog.Bool("input/recursive", CLI.Build.Recursive),
+		slog.Bool("input/prune", CLI.Build.PruneDockerContainers),
 	)
 
 	// Check if submodules were initialized
@@ -133,6 +136,7 @@ submodule_out:
 		ctx,
 		CLI.Build.Target,
 		CLI.Build.Recursive,
+		CLI.Build.PruneDockerContainers,
 		myConfig,
 		recipes.Execute,
 	)
@@ -163,9 +167,7 @@ submodule_out:
 
 func getInputsFromEnvironment() (string, error) {
 	// Check for GitHub
-	// https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
-	_, exists := os.LookupEnv("GITHUB_ACTIONS")
-	if exists {
+	if environment.DetectGithub() {
 		return parseGithub()
 	}
 
@@ -292,6 +294,7 @@ func parseGithub() (string, error) {
 	CLI.Config = strings.Split(action.GetInput("config"), "\n")
 	CLI.Build.Target = action.GetInput("target")
 	CLI.Build.Recursive = regexTrue.MatchString(action.GetInput("recursive"))
+	CLI.Build.PruneDockerContainers = regexTrue.MatchString(action.GetInput("prune"))
 	CLI.JSON = regexTrue.MatchString(action.GetInput("json"))
 
 	return "GitHub", nil

cmd/firmware-action/recipes/recipes.go

@@ -15,6 +15,7 @@ import (
 	"sync"
 
 	"dagger.io/dagger"
+	"github.com/9elements/firmware-action/cmd/firmware-action/container"
 	"github.com/9elements/firmware-action/cmd/firmware-action/filesystem"
 	"github.com/heimdalr/dag"
 )
@@ -67,6 +68,7 @@ func Build(
 	ctx context.Context,
 	target string,
 	recursive bool,
+	pruneDocker bool,
 	config *Config,
 	executor func(context.Context, string, *Config) error,
 ) ([]BuildResults, error) {
@@ -139,6 +141,16 @@ func Build(
 			if err != nil && !errors.Is(err, ErrBuildUpToDate) {
 				break
 			}
+
+			// Prune the Dagger Engine to free disk space
+			// It is fine to do here because the `executor` function connects and disconnects
+			//   to Dagger Engine (it is self-contained)
+			if pruneDocker {
+				err = container.CleanupAfterContainer(ctx)
+				if err != nil {
+					break
+				}
+			}
 		}
 	} else {
 		// else build only the target

cmd/firmware-action/recipes/recipes_test.go

@@ -250,6 +250,7 @@ func TestBuild(t *testing.T) {
 				ctx,
 				tc.target,
 				tc.recursive,
+				false, // do not prune
 				&tc.config,
 				executeDummy,
 			)
@@ -262,6 +263,7 @@ func TestBuild(t *testing.T) {
 			ctx,
 			"pizza",
 			recursive,
+			false, // do not prune
 			&testConfigDependencyHell,
 			executeDummy,
 		)

tests/example_config.json

@@ -17,6 +17,44 @@
       "input_files": null
     }
   },
+  "universal": {
+    "universal-example-A": {
+      "depends": null,
+      "sdk_url": "golang:latest",
+      "repo_path": "./",
+      "container_output_dirs": null,
+      "container_output_files": [
+        "test.txt"
+      ],
+      "output_dir": "output-universal-example-A/",
+      "input_dirs": null,
+      "input_files": null,
+      "container_input_dir": "inputs/",
+      "build_commands": [
+        "echo 'hello world'",
+        "touch test.txt"
+      ]
+    },
+    "universal-example-B": {
+      "depends": [
+        "universal-example-A"
+      ],
+      "sdk_url": "golang:latest",
+      "repo_path": "./",
+      "container_output_dirs": null,
+      "container_output_files": [
+        "test.txt"
+      ],
+      "output_dir": "output-universal-example-B/",
+      "input_dirs": null,
+      "input_files": null,
+      "container_input_dir": "inputs/",
+      "build_commands": [
+        "echo 'hello world'",
+        "touch test.txt"
+      ]
+    }
+  },
   "edk2": {
     "edk2-example": {
       "depends": null,