add support for pack to extend base images using daemon

internal/build/docker.go

@@ -10,6 +10,8 @@ import (
 	specs "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
+//go:generate mockgen -package mockdocker -destination ./mockdocker/mockDockerClient.go github.com/buildpacks/pack/internal/build DockerClient
+
 type DockerClient interface {
 	ImageRemove(ctx context.Context, image string, options types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error)
 	VolumeRemove(ctx context.Context, volumeID string, force bool) error
@@ -21,4 +23,7 @@ type DockerClient interface {
 	ContainerInspect(ctx context.Context, container string) (types.ContainerJSON, error)
 	ContainerRemove(ctx context.Context, container string, options types.ContainerRemoveOptions) error
 	CopyToContainer(ctx context.Context, container, path string, content io.Reader, options types.CopyToContainerOptions) error
+	ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error)
+	ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error)
+	ImageSave(ctx context.Context, imageIDs []string) (io.ReadCloser, error)
 }

internal/build/extend_build_test.go

@@ -0,0 +1,168 @@
+package build_test
+
+import (
+	"bytes"
+	"context"
+	"io"
+
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/apex/log"
+
+	"github.com/buildpacks/lifecycle/buildpack"
+
+	"github.com/buildpacks/pack/internal/build"
+	"github.com/buildpacks/pack/internal/build/fakes"
+
+	"github.com/docker/docker/api/types"
+	"github.com/golang/mock/gomock"
+	"github.com/heroku/color"
+	"github.com/sclevine/spec"
+	"github.com/sclevine/spec/report"
+
+	"github.com/buildpacks/pack/pkg/logging"
+	h "github.com/buildpacks/pack/testhelpers"
+
+	mockdocker "github.com/buildpacks/pack/internal/build/mockdocker"
+)
+
+func TestBuildDockerfiles(t *testing.T) {
+	color.Disable(true)
+	defer color.Disable(false)
+	spec.Run(t, "buildExtendByDocker", testBuildDockerfiles, spec.Report(report.Terminal{}), spec.Sequential())
+}
+
+const (
+	argUserID  = "user_id"
+	argGroupID = "group_id"
+)
+
+func testBuildDockerfiles(t *testing.T, when spec.G, it spec.S) {
+	var (
+		mockDockerClient *mockdocker.MockDockerClient
+		mockController   *gomock.Controller
+		lifecycle        *build.LifecycleExecution
+		tmpDir           string
+
+		// lifecycle options
+		providedClearCache            bool
+		providedPublish               bool
+		providedBuilderImage          = "some-registry.com/some-namespace/some-builder-name"
+		extendedBuilderImage          = "some-registry.com/some-namespace/some-builder-name-extended"
+		configureDefaultTestLifecycle func(opts *build.LifecycleOptions)
+		lifecycleOps                  []func(*build.LifecycleOptions)
+	)
+
+	it.Before(func() {
+		h.SkipIf(t, runtime.GOOS == "windows", "extensions not supported on windows")
+		var err error
+		mockController = gomock.NewController(t)
+		mockDockerClient = mockdocker.NewMockDockerClient(mockController)
+		h.AssertNil(t, err)
+
+		configureDefaultTestLifecycle = func(opts *build.LifecycleOptions) {
+			opts.BuilderImage = providedBuilderImage
+			opts.ClearCache = providedClearCache
+			opts.Publish = providedPublish
+		}
+
+		lifecycleOps = []func(*build.LifecycleOptions){configureDefaultTestLifecycle}
+	})
+
+	when("Extend Build Image By Docker", func() {
+		it("should extend build image using 1 extension", func() {
+			// set tmp directory
+			tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single")
+			lifecycle = getTestLifecycleExec(t, true, tmpDir, mockDockerClient, lifecycleOps...)
+			dockerfile := build.DockerfileInfo{
+				Info: &buildpack.DockerfileInfo{
+					Path: filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single", "build", "samples_test", "Dockerfile"),
+				},
+			}
+			expectedBuilder := lifecycle.Builder()
+			expectedBuildContext, _ := dockerfile.CreateBuildContext(lifecycle.AppPath())
+			// Set up expected Build Args
+			UID := strconv.Itoa(expectedBuilder.UID())
+			GID := strconv.Itoa(expectedBuilder.GID())
+			expectedbuildArguments := map[string]*string{}
+			expectedbuildArguments["base_image"] = &providedBuilderImage
+			expectedbuildArguments[argUserID] = &UID
+			expectedbuildArguments[argGroupID] = &GID
+			expectedBuildOptions := types.ImageBuildOptions{
+				Context:    expectedBuildContext,
+				Dockerfile: "Dockerfile",
+				Tags:       []string{extendedBuilderImage},
+				Remove:     true,
+				BuildArgs:  expectedbuildArguments,
+			}
+			mockResponse := types.ImageBuildResponse{
+				Body:   io.NopCloser(strings.NewReader("mock-build-response-body")),
+				OSType: "linux",
+			}
+			mockDockerClient.EXPECT().ImageBuild(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_ context.Context, buildContext io.Reader, buildOptions types.ImageBuildOptions) {
+				compBuildOptions(t, expectedBuildOptions, buildOptions)
+			}).Return(mockResponse, nil).Times(1)
+			err := lifecycle.ExtendBuildByDaemon(context.Background())
+			h.AssertNil(t, err)
+		})
+
+		it("should extend build image using multiple extension", func() {
+			// set tmp directory
+			tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "multi")
+			lifecycle = getTestLifecycleExec(t, true, tmpDir, mockDockerClient, lifecycleOps...)
+			mockResponse := types.ImageBuildResponse{
+				Body:   io.NopCloser(strings.NewReader("mock-build-response-body")),
+				OSType: "linux",
+			}
+			mockDockerClient.EXPECT().ImageBuild(gomock.Any(), gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(2)
+			err := lifecycle.ExtendBuildByDaemon(context.Background())
+			h.AssertNil(t, err)
+		})
+	})
+}
+func GetTestLifecycleExecErr(t *testing.T, logVerbose bool, tmpDir string, mockDockerClient *mockdocker.MockDockerClient, ops ...func(*build.LifecycleOptions)) (*build.LifecycleExecution, error) {
+	var outBuf bytes.Buffer
+	logger := logging.NewLogWithWriters(&outBuf, &outBuf)
+	if logVerbose {
+		logger.Level = log.DebugLevel
+	}
+	defaultBuilder, err := fakes.NewFakeBuilder()
+	h.AssertNil(t, err)
+
+	opts := build.LifecycleOptions{
+		AppPath:    "some-app-path",
+		Builder:    defaultBuilder,
+		HTTPProxy:  "some-http-proxy",
+		HTTPSProxy: "some-https-proxy",
+		NoProxy:    "some-no-proxy",
+		Termui:     &fakes.FakeTermui{},
+	}
+
+	for _, op := range ops {
+		op(&opts)
+	}
+
+	return build.NewLifecycleExecution(logger, mockDockerClient, tmpDir, opts)
+}
+
+func getTestLifecycleExec(t *testing.T, logVerbose bool, tmpDir string, mockDockerClient *mockdocker.MockDockerClient, ops ...func(*build.LifecycleOptions)) *build.LifecycleExecution {
+	t.Helper()
+
+	lifecycleExec, err := GetTestLifecycleExecErr(t, logVerbose, tmpDir, mockDockerClient, ops...)
+	h.AssertNil(t, err)
+	return lifecycleExec
+}
+
+func compBuildOptions(t *testing.T, expectedBuildOptions types.ImageBuildOptions, actualBuildOptions types.ImageBuildOptions) {
+	t.Helper()
+	h.AssertEq(t, expectedBuildOptions.Dockerfile, actualBuildOptions.Dockerfile)
+	h.AssertEq(t, expectedBuildOptions.Tags, actualBuildOptions.Tags)
+	h.AssertEq(t, expectedBuildOptions.Remove, actualBuildOptions.Remove)
+	h.AssertEq(t, expectedBuildOptions.BuildArgs["base_image"], actualBuildOptions.BuildArgs["base_image"])
+	h.AssertEq(t, expectedBuildOptions.BuildArgs[argUserID], actualBuildOptions.BuildArgs[argUserID])
+	h.AssertEq(t, expectedBuildOptions.BuildArgs[argGroupID], actualBuildOptions.BuildArgs[argGroupID])
+}

internal/build/helper.go

@@ -0,0 +1,158 @@
+package build
+
+import (
+	"archive/tar"
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/BurntSushi/toml"
+	"github.com/buildpacks/lifecycle/buildpack"
+	"github.com/buildpacks/lifecycle/cmd"
+
+	"github.com/buildpacks/pack/pkg/archive"
+
+	"github.com/buildpacks/pack/pkg/logging"
+)
+
+const (
+	DockerfileKindBuild = "build"
+	DockerfileKindRun   = "run"
+)
+
+type Extensions struct {
+	Extensions []buildpack.GroupElement
+}
+
+type DockerfileInfo struct {
+	Info *buildpack.DockerfileInfo
+	Args []Arg
+}
+
+type Arg struct {
+	Name  string `toml:"name"`
+	Value string `toml:"value"`
+}
+
+type Config struct {
+	Build BuildConfig `toml:"build"`
+	Run   BuildConfig `toml:"run"`
+}
+
+type BuildConfig struct {
+	Args []Arg `toml:"args"`
+}
+
+func (extensions *Extensions) DockerFiles(kind string, path string, logger logging.Logger) ([]DockerfileInfo, error) {
+	var dockerfiles []DockerfileInfo
+	for _, ext := range extensions.Extensions {
+		dockerfile, err := extensions.ReadDockerFile(path, kind, ext.ID)
+		if err != nil {
+			return nil, err
+		}
+		if dockerfile != nil {
+			logger.Debugf("Found %s Dockerfile for extension '%s'", kind, ext.ID)
+			switch kind {
+			case DockerfileKindBuild:
+				break
+			case DockerfileKindRun:
+				buildpack.ValidateRunDockerfile(dockerfile.Info, logger)
+			default:
+				return nil, fmt.Errorf("unknown dockerfile kind: %s", kind)
+			}
+			dockerfiles = append(dockerfiles, *dockerfile)
+		}
+	}
+	return dockerfiles, nil
+}
+
+func (extensions *Extensions) ReadDockerFile(path string, kind string, extID string) (*DockerfileInfo, error) {
+	dockerfilePath := filepath.Join(path, kind, escapeID(extID), "Dockerfile")
+	if _, err := os.Stat(dockerfilePath); err != nil {
+		return nil, nil
+	}
+	configPath := filepath.Join(path, kind, escapeID(extID), "extend-config.toml")
+	var config Config
+	_, err := toml.DecodeFile(configPath, &config)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, err
+		}
+	}
+
+	var args []Arg
+	if kind == buildpack.DockerfileKindBuild {
+		args = config.Build.Args
+	} else {
+		args = config.Run.Args
+	}
+	return &DockerfileInfo{
+		Info: &buildpack.DockerfileInfo{
+			ExtensionID: extID,
+			Kind:        kind,
+			Path:        dockerfilePath,
+		},
+		Args: args,
+	}, nil
+}
+
+func (extensions *Extensions) SetExtensions(path string, logger logging.Logger) error {
+	groupExt, err := readExtensionsGroup(path)
+	if err != nil {
+		return fmt.Errorf("reading group: %w", err)
+	}
+	for i := range groupExt {
+		groupExt[i].Extension = true
+	}
+	for _, groupEl := range groupExt {
+		if err = cmd.VerifyBuildpackAPI(groupEl.Kind(), groupEl.String(), groupEl.API, logger); err != nil {
+			return err
+		}
+	}
+	extensions.Extensions = groupExt
+	return nil
+}
+
+func readExtensionsGroup(path string) ([]buildpack.GroupElement, error) {
+	var group buildpack.Group
+	_, err := toml.DecodeFile(filepath.Join(path, "group.toml"), &group)
+	for e := range group.GroupExtensions {
+		group.GroupExtensions[e].Extension = true
+		group.GroupExtensions[e].Optional = true
+	}
+	return group.GroupExtensions, err
+}
+
+func escapeID(id string) string {
+	return strings.ReplaceAll(id, "/", "_")
+}
+
+func (dockerfile *DockerfileInfo) CreateBuildContext(path string) (io.Reader, error) {
+	defaultFilterFunc := func(file string) bool { return true }
+	buf := new(bytes.Buffer)
+	tarWriter := tar.NewWriter(buf)
+	var completeErr error
+
+	defer func() {
+		if err := tarWriter.Close(); err != nil {
+			fmt.Println("Error closing tar writer:", err)
+			completeErr = archive.AggregateError(completeErr, err)
+		}
+	}()
+	if err := archive.WriteDirToTar(tarWriter, path, "/workspace", 0, 0, -1, true, false, defaultFilterFunc); err != nil {
+		tarWriter.Close()
+		fmt.Println("Error adding workspace:", err)
+		completeErr = archive.AggregateError(completeErr, err)
+	}
+
+	if err := archive.WriteFileToTar(tarWriter, dockerfile.Info.Path, filepath.Join(".", "Dockerfile"), 0, 0, -1, true); err != nil {
+		tarWriter.Close()
+		fmt.Println("Error adding dockerfile:", err)
+		completeErr = archive.AggregateError(completeErr, err)
+	}
+
+	return buf, completeErr
+}