Merge pull request buildpacks#549 from buildpacks/feature/511-add-exc…

…lude-and-include-support

add project descriptor exclude and include support for build

acceptance/acceptance_test.go

@@ -882,6 +882,119 @@ func testAcceptance(t *testing.T, when spec.G, it spec.S, packFixturesDir, packP
 							h.AssertNotContains(t, buf.String(), "Successfully built image")
 						})
 					})
+
+					when("--descriptor", func() {
+
+						when("exclude and include", func() {
+							var buildpackTgz, tempAppDir string
+
+							it.Before(func() {
+								var err error
+
+								packVer, err := packVersion(packPath)
+								h.AssertNil(t, err)
+								packSemver := semver.MustParse(strings.TrimPrefix(strings.Split(packVer, " ")[0], "v"))
+								supported := packSemver.GreaterThan(semver.MustParse("0.9.0")) || packSemver.Equal(semver.MustParse("0.0.0"))
+								h.SkipIf(t, !supported, "pack --descriptor does NOT support 'exclude' and 'include' feature")
+
+								buildpackTgz = h.CreateTGZ(t, filepath.Join(bpDir, "descriptor-buildpack"), "./", 0755)
+
+								tempAppDir, err = ioutil.TempDir("", "descriptor-app")
+								h.AssertNil(t, err)
+
+								// Create test directories and files:
+								//
+								// ├── cookie.jar
+								// ├── secrets
+								// │   ├── api_keys.json
+								// |   |── user_token
+								// ├── media
+								// │   ├── mountain.jpg
+								// │   └── person.png
+								// └── test.sh
+								err = os.Mkdir(filepath.Join(tempAppDir, "secrets"), 0755)
+								h.AssertNil(t, err)
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "secrets", "api_keys.json"), []byte("{}"), 0755)
+								h.AssertNil(t, err)
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "secrets", "user_token"), []byte("token"), 0755)
+								h.AssertNil(t, err)
+
+								err = os.Mkdir(filepath.Join(tempAppDir, "media"), 0755)
+								h.AssertNil(t, err)
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "media", "mountain.jpg"), []byte("fake image bytes"), 0755)
+								h.AssertNil(t, err)
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "media", "person.png"), []byte("fake image bytes"), 0755)
+								h.AssertNil(t, err)
+
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "cookie.jar"), []byte("chocolate chip"), 0755)
+								h.AssertNil(t, err)
+								err = ioutil.WriteFile(filepath.Join(tempAppDir, "test.sh"), []byte("echo test"), 0755)
+								h.AssertNil(t, err)
+							})
+
+							it.After(func() {
+								h.AssertNil(t, os.RemoveAll(tempAppDir))
+							})
+
+							it("should exclude ALL specified files and directories", func() {
+								projectToml := `
+[project]
+name = "exclude test"
+[[project.licenses]]
+type = "MIT"
+[build]
+exclude = [ "*.sh", "secrets/", "media/metadata" ]
+`
+								excludeDescriptorPath := filepath.Join(tempAppDir, "exclude.toml")
+								err := ioutil.WriteFile(excludeDescriptorPath, []byte(projectToml), 0755)
+								h.AssertNil(t, err)
+
+								output := h.Run(t, subjectPack(
+									"build",
+									repoName,
+									"-p", tempAppDir,
+									"--buildpack", buildpackTgz,
+									"--descriptor", excludeDescriptorPath,
+								))
+								h.AssertNotContains(t, output, "api_keys.json")
+								h.AssertNotContains(t, output, "user_token")
+								h.AssertNotContains(t, output, "test.sh")
+
+								h.AssertContains(t, output, "cookie.jar")
+								h.AssertContains(t, output, "mountain.jpg")
+								h.AssertContains(t, output, "person.png")
+							})
+
+							it("should ONLY include specified files and directories", func() {
+								projectToml := `
+[project]
+name = "include test"
+[[project.licenses]]
+type = "MIT"
+[build]
+include = [ "*.jar", "media/mountain.jpg", "media/person.png" ]
+`
+								includeDescriptorPath := filepath.Join(tempAppDir, "include.toml")
+								err := ioutil.WriteFile(includeDescriptorPath, []byte(projectToml), 0755)
+								h.AssertNil(t, err)
+
+								output := h.Run(t, subjectPack(
+									"build",
+									repoName,
+									"-p", tempAppDir,
+									"--buildpack", buildpackTgz,
+									"--descriptor", includeDescriptorPath,
+								))
+								h.AssertNotContains(t, output, "api_keys.json")
+								h.AssertNotContains(t, output, "user_token")
+								h.AssertNotContains(t, output, "test.sh")
+
+								h.AssertContains(t, output, "cookie.jar")
+								h.AssertContains(t, output, "mountain.jpg")
+								h.AssertContains(t, output, "person.png")
+							})
+						})
+					})
 				})
 
 				when("default builder is not set", func() {
@@ -1671,8 +1784,10 @@ func createStack(t *testing.T, dockerCli client.CommonAPIClient, runImageMirror
 }
 
 func createStackImage(dockerCli client.CommonAPIClient, repoName string, dir string) error {
+	defaultFilterFunc := func(file string) bool { return true }
+
 	ctx := context.Background()
-	buildContext := archive.ReadDirAsTar(dir, "/", 0, 0, -1, true)
+	buildContext := archive.ReadDirAsTar(dir, "/", 0, 0, -1, true, defaultFilterFunc)
 
 	res, err := dockerCli.ImageBuild(ctx, buildContext, dockertypes.ImageBuildOptions{
 		Tags:        []string{repoName},

acceptance/testdata/mock_buildpacks/0.2/descriptor-buildpack/bin/build

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+echo "---> Build: Descriptor Buildpack"
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ls -laR
+
+echo "---> Done"

acceptance/testdata/mock_buildpacks/0.2/descriptor-buildpack/bin/detect

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+echo "---> Detect: Descriptor Buildpack"
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+echo "---> Done"

acceptance/testdata/mock_buildpacks/0.2/descriptor-buildpack/buildpack.toml

@@ -0,0 +1,9 @@
+api = "0.2"
+
+[buildpack]
+  id = "descriptor/bp"
+  version = "descriptor-bp-version"
+  name = "Descriptor Buildpack"
+
+[[stacks]]
+  id = "pack.test.stack"

build.go

@@ -49,6 +49,7 @@ type BuildOptions struct {
 	ProxyConfig        *ProxyConfig // defaults to  environment proxy vars
 	ContainerConfig    ContainerConfig
 	DefaultProcessType string
+	FileFilter         func(string) bool
 }
 
 type ProxyConfig struct {
@@ -148,6 +149,7 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error {
 		Network:            opts.ContainerConfig.Network,
 		Volumes:            platformVolumes,
 		DefaultProcessType: opts.DefaultProcessType,
+		FileFilter:         opts.FileFilter,
 	})
 }
 

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/opencontainers/selinux v1.4.0 // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94
 	github.com/sclevine/spec v1.4.0
 	github.com/spf13/cobra v0.0.6
 	golang.org/x/crypto v0.0.0-20200311171314-f7b00557c8c4

go.sum

@@ -156,6 +156,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -339,6 +340,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94 h1:G04eS0JkAIVZfaJLjla9dNxkJCPiKIGZlw9AfOhzOD0=
+github.com/sabhiram/go-gitignore v0.0.0-20180611051255-d3107576ba94/go.mod h1:b18R55ulyQ/h3RaWyloPyER7fWQVZvimKKhnI5OfrJQ=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=

internal/archive/archive.go

@@ -21,15 +21,15 @@ func init() {
 	NormalizedDateTime = time.Date(1980, time.January, 1, 0, 0, 1, 0, time.UTC)
 }
 
-func ReadDirAsTar(srcDir, basePath string, uid, gid int, mode int64, normalizeModTime bool) io.ReadCloser {
+func ReadDirAsTar(srcDir, basePath string, uid, gid int, mode int64, normalizeModTime bool, fileFilter func(string) bool) io.ReadCloser {
 	return GenerateTar(func(tw *tar.Writer) error {
-		return WriteDirToTar(tw, srcDir, basePath, uid, gid, mode, normalizeModTime)
+		return WriteDirToTar(tw, srcDir, basePath, uid, gid, mode, normalizeModTime, fileFilter)
 	})
 }
 
-func ReadZipAsTar(srcPath, basePath string, uid, gid int, mode int64, normalizeModTime bool) io.ReadCloser {
+func ReadZipAsTar(srcPath, basePath string, uid, gid int, mode int64, normalizeModTime bool, fileFilter func(string) bool) io.ReadCloser {
 	return GenerateTar(func(tw *tar.Writer) error {
-		return WriteZipToTar(tw, srcPath, basePath, uid, gid, mode, normalizeModTime)
+		return WriteZipToTar(tw, srcPath, basePath, uid, gid, mode, normalizeModTime, fileFilter)
 	})
 }
 
@@ -148,8 +148,11 @@ func ReadTarEntry(rc io.Reader, entryPath string) (*tar.Header, []byte, error) {
 
 // WriteDirToTar writes the contents of a directory to a tar writer. `basePath` is the "location" in the tar the
 // contents will be places.
-func WriteDirToTar(tw *tar.Writer, srcDir, basePath string, uid, gid int, mode int64, normalizeModTime bool) error {
+func WriteDirToTar(tw *tar.Writer, srcDir, basePath string, uid, gid int, mode int64, normalizeModTime bool, fileFilter func(string) bool) error {
 	return filepath.Walk(srcDir, func(file string, fi os.FileInfo, err error) error {
+		if fileFilter != nil && !fileFilter(file) {
+			return nil
+		}
 		if err != nil {
 			return err
 		}
@@ -206,14 +209,17 @@ func WriteDirToTar(tw *tar.Writer, srcDir, basePath string, uid, gid int, mode i
 	})
 }
 
-func WriteZipToTar(tw *tar.Writer, srcZip, basePath string, uid, gid int, mode int64, normalizeModTime bool) error {
+func WriteZipToTar(tw *tar.Writer, srcZip, basePath string, uid, gid int, mode int64, normalizeModTime bool, fileFilter func(string) bool) error {
 	zipReader, err := zip.OpenReader(srcZip)
 	if err != nil {
 		return err
 	}
 	defer zipReader.Close()
 
 	for _, f := range zipReader.File {
+		if fileFilter != nil && !fileFilter(f.Name) {
+			continue
+		}
 		var header *tar.Header
 		if f.Mode()&os.ModeSymlink != 0 {
 			target, err := func() (string, error) {

internal/archive/archive_test.go

@@ -100,7 +100,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteDirToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, 0777, true)
+				err = archive.WriteDirToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, 0777, true, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -127,7 +127,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteDirToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, -1, true)
+				err = archive.WriteDirToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, -1, true, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -155,7 +155,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteDirToTar(tw, src, "/foo", 1234, 2345, 0777, false)
+				err = archive.WriteDirToTar(tw, src, "/foo", 1234, 2345, 0777, false, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -174,7 +174,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteDirToTar(tw, src, "/foo", 1234, 2345, 0777, true)
+				err = archive.WriteDirToTar(tw, src, "/foo", 1234, 2345, 0777, true, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -221,7 +221,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 					tw := tar.NewWriter(fh)
 
-					err = archive.WriteDirToTar(tw, tmpSrcDir, "/nested/dir/dir-in-archive", 1234, 2345, 0777, true)
+					err = archive.WriteDirToTar(tw, tmpSrcDir, "/nested/dir/dir-in-archive", 1234, 2345, 0777, true, nil)
 					h.AssertNil(t, err)
 					h.AssertNil(t, tw.Close())
 					h.AssertNil(t, fh.Close())
@@ -257,7 +257,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteZipToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, 0777, true)
+				err = archive.WriteZipToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, 0777, true, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -284,7 +284,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteZipToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, -1, true)
+				err = archive.WriteZipToTar(tw, src, "/nested/dir/dir-in-archive", 1234, 2345, -1, true, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -312,7 +312,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteZipToTar(tw, src, "/foo", 1234, 2345, 0777, false)
+				err = archive.WriteZipToTar(tw, src, "/foo", 1234, 2345, 0777, false, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())
@@ -331,7 +331,7 @@ func testArchive(t *testing.T, when spec.G, it spec.S) {
 
 				tw := tar.NewWriter(fh)
 
-				err = archive.WriteZipToTar(tw, src, "/foo", 1234, 2345, 0777, true)
+				err = archive.WriteZipToTar(tw, src, "/foo", 1234, 2345, 0777, true, nil)
 				h.AssertNil(t, err)
 				h.AssertNil(t, tw.Close())
 				h.AssertNil(t, fh.Close())

internal/blob/blob.go

@@ -31,7 +31,7 @@ func (b blob) Open() (r io.ReadCloser, err error) {
 		return nil, errors.Wrapf(err, "read blob at path '%s'", b.path)
 	}
 	if fi.IsDir() {
-		return archive.ReadDirAsTar(b.path, ".", 0, 0, -1, true), nil
+		return archive.ReadDirAsTar(b.path, ".", 0, 0, -1, true, nil), nil
 	}
 
 	fh, err := os.Open(b.path)

internal/build/build.go

@@ -36,6 +36,7 @@ type Lifecycle struct {
 	AppVolume          string
 	Volumes            []string
 	DefaultProcessType string
+	fileFilter         func(string) bool
 }
 
 type Cache interface {
@@ -66,6 +67,7 @@ type LifecycleOptions struct {
 	Network            string
 	Volumes            []string
 	DefaultProcessType string
+	FileFilter         func(string) bool
 }
 
 func (l *Lifecycle) Execute(ctx context.Context, opts LifecycleOptions) error {
@@ -128,6 +130,7 @@ func (l *Lifecycle) Setup(opts LifecycleOptions) {
 	l.version = opts.Builder.LifecycleDescriptor().Info.Version.String()
 	l.platformAPIVersion = opts.Builder.LifecycleDescriptor().API.PlatformVersion.String()
 	l.DefaultProcessType = opts.DefaultProcessType
+	l.fileFilter = opts.FileFilter
 }
 
 func (l *Lifecycle) Cleanup() error {