From 1b452bdee22d8111a693eba56b62f908025b9d7f Mon Sep 17 00:00:00 2001 From: ashpect Date: Sat, 21 Oct 2023 09:29:56 +0530 Subject: [PATCH 01/20] Saving images to disk and adding flag --to-oci-tar Signed-off-by: ashpect --- pkg/imgpkg/bundle/contents.go | 7 ++++--- pkg/imgpkg/bundle/contents_test.go | 4 ++-- pkg/imgpkg/bundle/locations_configs.go | 2 +- pkg/imgpkg/cmd/oci_flags.go | 17 +++++++++++++++++ pkg/imgpkg/cmd/push.go | 11 ++++++++--- pkg/imgpkg/plainimage/contents.go | 12 ++++++++++-- 6 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 pkg/imgpkg/cmd/oci_flags.go diff --git a/pkg/imgpkg/bundle/contents.go b/pkg/imgpkg/bundle/contents.go index d7cdbc03f..e43e39b8d 100644 --- a/pkg/imgpkg/bundle/contents.go +++ b/pkg/imgpkg/bundle/contents.go @@ -27,6 +27,7 @@ type Contents struct { paths []string excludedPaths []string preservePermissions bool + ociTarPath string } //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . ImagesMetadataWriter @@ -38,8 +39,8 @@ type ImagesMetadataWriter interface { } // NewContents creates Contents struct -func NewContents(paths []string, excludedPaths []string, preservePermissions bool) Contents { - return Contents{paths: paths, excludedPaths: excludedPaths, preservePermissions: preservePermissions} +func NewContents(paths []string, excludedPaths []string, preservePermissions bool, ociTarPath string) Contents { + return Contents{paths: paths, excludedPaths: excludedPaths, preservePermissions: preservePermissions, ociTarPath: ociTarPath} } // Push the contents of the bundle to the registry as an OCI Image @@ -54,7 +55,7 @@ func (b Contents) Push(uploadRef regname.Tag, labels map[string]string, registry } labels[BundleConfigLabel] = "true" - return plainimage.NewContents(b.paths, b.excludedPaths, b.preservePermissions).Push(uploadRef, labels, registry, logger) + return plainimage.NewContents(b.paths, b.excludedPaths, b.preservePermissions, b.ociTarPath).Push(uploadRef, labels, registry, logger) } // PresentsAsBundle checks if the provided folders have the needed structure to be a bundle diff --git a/pkg/imgpkg/bundle/contents_test.go b/pkg/imgpkg/bundle/contents_test.go index a5bc69f61..eac2377ca 100644 --- a/pkg/imgpkg/bundle/contents_test.go +++ b/pkg/imgpkg/bundle/contents_test.go @@ -38,7 +38,7 @@ images: fakeRegistry.ImageReturns(bundleImg, nil) t.Run("push is successful", func(t *testing.T) { - subject := bundle.NewContents([]string{bundleDir}, nil, false) + subject := bundle.NewContents([]string{bundleDir}, nil, false, "") imgTag, err := name.NewTag("my.registry.io/new-bundle:tag") if err != nil { t.Fatalf("failed to read tag: %s", err) @@ -72,7 +72,7 @@ images: fakeRegistry.ImageReturns(bundleImg, nil) t.Run("push is successful", func(t *testing.T) { - subject := bundle.NewContents([]string{bundleDir}, nil, false) + subject := bundle.NewContents([]string{bundleDir}, nil, false, "") imgTag, err := name.NewTag("my.registry.io/new-bundle:tag") if err != nil { t.Fatalf("failed to read tag: %s", err) diff --git a/pkg/imgpkg/bundle/locations_configs.go b/pkg/imgpkg/bundle/locations_configs.go index a6912bbd8..6ec7bc6f8 100644 --- a/pkg/imgpkg/bundle/locations_configs.go +++ b/pkg/imgpkg/bundle/locations_configs.go @@ -132,7 +132,7 @@ func (r LocationsConfigs) Save(reg ImagesMetadataWriter, bundleRef name.Digest, r.ui.Tracef("Pushing image\n") - _, err = plainimage.NewContents([]string{tmpDir}, nil, false).Push(locRef, nil, reg.CloneWithLogger(util.NewNoopProgressBar()), logger) + _, err = plainimage.NewContents([]string{tmpDir}, nil, false, "").Push(locRef, nil, reg.CloneWithLogger(util.NewNoopProgressBar()), logger) if err != nil { // Immutable tag errors within registries are not standardized. // Assume word "immutable" would be present in most cases. diff --git a/pkg/imgpkg/cmd/oci_flags.go b/pkg/imgpkg/cmd/oci_flags.go new file mode 100644 index 000000000..b80a36f0b --- /dev/null +++ b/pkg/imgpkg/cmd/oci_flags.go @@ -0,0 +1,17 @@ +// Copyright 2020 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "github.com/spf13/cobra" +) + +type OciFlags struct { + OciImg string + OciTar string +} + +func (o *OciFlags) Set(cmd *cobra.Command) { + cmd.Flags().StringVar(&o.OciTar, "to-oci-tar", "", "Set OciTarPath to be saved to disk (example: /path/file.tar)") +} diff --git a/pkg/imgpkg/cmd/push.go b/pkg/imgpkg/cmd/push.go index 0602389cc..e05eec215 100644 --- a/pkg/imgpkg/cmd/push.go +++ b/pkg/imgpkg/cmd/push.go @@ -20,6 +20,7 @@ type PushOptions struct { ui ui.UI ImageFlags ImageFlags + OciFlags OciFlags BundleFlags BundleFlags LockOutputFlags LockOutputFlags FileFlags FileFlags @@ -40,10 +41,14 @@ func NewPushCmd(o *PushOptions) *cobra.Command { # Push bundle repo/app1-config with contents of config/ directory imgpkg push -b repo/app1-config -f config/ + #Push bundle saving the tar as OCI tar + imgpkg push -b repo/app1-config -f config/ --to-oci-tar /path/to/file.tar + # Push image repo/app1-config with contents from multiple locations imgpkg push -i repo/app1-config -f config/ -f additional-config.yml`, } o.ImageFlags.Set(cmd) + o.OciFlags.Set(cmd) o.BundleFlags.Set(cmd) o.LockOutputFlags.SetOnPush(cmd) o.FileFlags.Set(cmd) @@ -104,7 +109,7 @@ func (po *PushOptions) pushBundle(registry registry.Registry) (string, error) { } logger := util.NewUILevelLogger(util.LogWarn, util.NewLogger(po.ui)) - imageURL, err := bundle.NewContents(po.FileFlags.Files, po.FileFlags.ExcludedFilePaths, po.FileFlags.PreservePermissions).Push(uploadRef, po.LabelFlags.Labels, registry, logger) + imageURL, err := bundle.NewContents(po.FileFlags.Files, po.FileFlags.ExcludedFilePaths, po.FileFlags.PreservePermissions, po.OciFlags.OciTar).Push(uploadRef, po.LabelFlags.Labels, registry, logger) if err != nil { return "", err } @@ -140,7 +145,7 @@ func (po *PushOptions) pushImage(registry registry.Registry) (string, error) { return "", fmt.Errorf("Parsing '%s': %s", po.ImageFlags.Image, err) } - isBundle, err := bundle.NewContents(po.FileFlags.Files, po.FileFlags.ExcludedFilePaths, po.FileFlags.PreservePermissions).PresentsAsBundle() + isBundle, err := bundle.NewContents(po.FileFlags.Files, po.FileFlags.ExcludedFilePaths, po.FileFlags.PreservePermissions, po.OciFlags.OciTar).PresentsAsBundle() if err != nil { return "", err } @@ -149,7 +154,7 @@ func (po *PushOptions) pushImage(registry registry.Registry) (string, error) { } logger := util.NewUILevelLogger(util.LogWarn, util.NewLogger(po.ui)) - return plainimage.NewContents(po.FileFlags.Files, po.FileFlags.ExcludedFilePaths, po.FileFlags.PreservePermissions).Push(uploadRef, po.LabelFlags.Labels, registry, logger) + return plainimage.NewContents(po.FileFlags.Files, po.FileFlags.ExcludedFilePaths, po.FileFlags.PreservePermissions, po.OciFlags.OciTar).Push(uploadRef, po.LabelFlags.Labels, registry, logger) } // validateFlags checks if the provided flags are valid diff --git a/pkg/imgpkg/plainimage/contents.go b/pkg/imgpkg/plainimage/contents.go index ff3754635..255b83989 100644 --- a/pkg/imgpkg/plainimage/contents.go +++ b/pkg/imgpkg/plainimage/contents.go @@ -21,6 +21,7 @@ type Contents struct { paths []string excludedPaths []string preservePermissions bool + ociTarPath string } // ImagesWriter defines the needed functions to write to the registry @@ -30,8 +31,8 @@ type ImagesWriter interface { } // NewContents creates the struct that represent an OCI Image based on the provided paths -func NewContents(paths []string, excludedPaths []string, preservePermissions bool) Contents { - return Contents{paths: paths, excludedPaths: excludedPaths, preservePermissions: preservePermissions} +func NewContents(paths []string, excludedPaths []string, preservePermissions bool, ociTarPath string) Contents { + return Contents{paths: paths, excludedPaths: excludedPaths, preservePermissions: preservePermissions, ociTarPath: ociTarPath} } // Push the OCI Image to the registry @@ -48,6 +49,13 @@ func (i Contents) Push(uploadRef regname.Tag, labels map[string]string, writer I return "", err } + if i.ociTarPath != "" { + err = crane.SaveOCI(img, i.ociTarPath) + if err != nil { + return "", err + } + } + defer img.Remove() err = writer.WriteImage(uploadRef, img, nil) From a0ee159aacbce0956ac93d5f7a3bcfe765078f15 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 1 Nov 2023 07:51:20 +0530 Subject: [PATCH 02/20] WIP Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy.go | 6 + pkg/imgpkg/cmd/copy_repo_src.go | 10 +- pkg/imgpkg/cmd/oci_flags.go | 7 +- pkg/imgpkg/imageset/image_set.go | 1 + pkg/imgpkg/imageset/tar_image_set.go | 63 ++- pkg/imgpkg/imagetar/tar_oci_writer.go | 66 +++ pkg/imgpkg/plainimage/contents.go | 2 +- pkg/imgpkg/registry/registry.go | 2 + .../pkg/v1/layout/README.md | 5 + .../pkg/v1/layout/blob.go | 37 ++ .../go-containerregistry/pkg/v1/layout/doc.go | 19 + .../pkg/v1/layout/image.go | 139 +++++ .../pkg/v1/layout/index.go | 161 ++++++ .../pkg/v1/layout/layoutpath.go | 25 + .../pkg/v1/layout/options.go | 71 +++ .../pkg/v1/layout/read.go | 32 ++ .../pkg/v1/layout/write.go | 482 ++++++++++++++++++ 17 files changed, 1119 insertions(+), 9 deletions(-) create mode 100644 pkg/imgpkg/imagetar/tar_oci_writer.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/README.md create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go diff --git a/pkg/imgpkg/cmd/copy.go b/pkg/imgpkg/cmd/copy.go index ae3db0d2c..5ab4bcadd 100644 --- a/pkg/imgpkg/cmd/copy.go +++ b/pkg/imgpkg/cmd/copy.go @@ -23,6 +23,7 @@ type CopyOptions struct { ui ui.UI ImageFlags ImageFlags + OciFlags OciFlags BundleFlags BundleFlags LockInputFlags LockInputFlags LockOutputFlags LockOutputFlags @@ -72,6 +73,7 @@ func NewCopyCmd(o *CopyOptions) *cobra.Command { } o.ImageFlags.SetCopy(cmd) + o.OciFlags.Set(cmd) o.BundleFlags.SetCopy(cmd) o.LockInputFlags.Set(cmd) o.LockOutputFlags.SetOnCopy(cmd) @@ -125,6 +127,7 @@ func (c *CopyOptions) Run() error { repoSrc := CopyRepoSrc{ ImageFlags: c.ImageFlags, + OciFlags: c.OciFlags, BundleFlags: c.BundleFlags, LockInputFlags: c.LockInputFlags, TarFlags: c.TarFlags, @@ -261,6 +264,9 @@ func (c *CopyOptions) hasOneSrc() bool { seen = true } } + if c.OciFlags.IsOci() { + seen = true + } return seen } diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index 00a2b0a7f..29bc16985 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -23,6 +23,7 @@ type SignatureRetriever interface { type CopyRepoSrc struct { ImageFlags ImageFlags + OciFlags OciFlags BundleFlags BundleFlags LockInputFlags LockInputFlags TarFlags TarFlags @@ -66,12 +67,17 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) return nil, fmt.Errorf("Building import repository ref: %s", err) } - if c.TarFlags.IsSrc() { + if c.TarFlags.IsSrc() || c.OciFlags.IsOci() { if c.TarFlags.IsDst() { return nil, fmt.Errorf("Cannot use tar source (--tar) with tar destination (--to-tar)") } - processedImages, err = c.tarImageSet.Import(c.TarFlags.TarSrc, importRepo, c.registry) + if c.OciFlags.IsOci() { + processedImages, err = c.tarImageSet.Import(c.OciFlags.OcitoReg, importRepo, c.registry, true) + } else { + processedImages, err = c.tarImageSet.Import(c.TarFlags.TarSrc, importRepo, c.registry, false) + } + if err != nil { return nil, err } diff --git a/pkg/imgpkg/cmd/oci_flags.go b/pkg/imgpkg/cmd/oci_flags.go index b80a36f0b..3c8baa0d5 100644 --- a/pkg/imgpkg/cmd/oci_flags.go +++ b/pkg/imgpkg/cmd/oci_flags.go @@ -8,10 +8,13 @@ import ( ) type OciFlags struct { - OciImg string - OciTar string + OcitoReg string + OciTar string } func (o *OciFlags) Set(cmd *cobra.Command) { cmd.Flags().StringVar(&o.OciTar, "to-oci-tar", "", "Set OciTarPath to be saved to disk (example: /path/file.tar)") + cmd.Flags().StringVar(&o.OcitoReg, "oci-tar", "", "Give path to OCI tar file (example: /path/file.tar)") } + +func (o OciFlags) IsOci() bool { return o.OcitoReg != "" } diff --git a/pkg/imgpkg/imageset/image_set.go b/pkg/imgpkg/imageset/image_set.go index c3faa3d75..09ac5c5ae 100644 --- a/pkg/imgpkg/imageset/image_set.go +++ b/pkg/imgpkg/imageset/image_set.go @@ -71,6 +71,7 @@ func (i ImageSet) Export(foundImages *UnprocessedImageRefs, return ids, nil } +// this pushes probably to a registry func (i *ImageSet) Import(imgOrIndexes []imagedesc.ImageOrIndex, importRepo regname.Repository, registry registry.ImagesReaderWriter) (*ProcessedImages, error) { diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index 2cbca60dd..ac937c02b 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -123,10 +123,18 @@ func (i TarImageSet) Export(foundImages *UnprocessedImageRefs, outputPath string } // Import Copy tar with Images to the Registry -func (i *TarImageSet) Import(path string, importRepo regname.Repository, registry registry.ImagesReaderWriter) (*ProcessedImages, error) { - imgOrIndexes, err := imagetar.NewTarReader(path).Read() - if err != nil { - return nil, err +func (i *TarImageSet) Import(path string, importRepo regname.Repository, registry registry.ImagesReaderWriter, tarisoci bool) (*ProcessedImages, error) { + + var imgOrIndexes []imagedesc.ImageOrIndex + var err error + + if tarisoci { + imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci() + } else { + imgOrIndexes, err = imagetar.NewTarReader(path).Read() + //for testing + //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") + //crane.SaveOCI(*imgOrIndexes[1].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a1") } processedImages, err := i.imageSet.Import(imgOrIndexes, importRepo, registry) @@ -136,3 +144,50 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr return processedImages, err } + +// change function to use accordingly +func loadImage(path string, index bool) (partial.WithRawManifest, error) { + // to check if path is a tarball or a directory + stat, err := os.Stat(path) + if err != nil { + return nil, err + } + + if !stat.IsDir() { + img, err := crane.Load(path) + if err != nil { + return nil, fmt.Errorf("loading %s as tarball: %w", path, err) + } + return img, nil + } + + //if the path is a oci layout directory + l, err := layout.ImageIndexFromPath(path) + if err != nil { + return nil, fmt.Errorf("loading %s as OCI layout: %w", path, err) + } + + if index { + return l, nil + } + + //append l.ImageIndex() to imgOrIndexes + + m, err := l.IndexManifest() + + if err != nil { + return nil, err + } + if len(m.Manifests) != 1 { + return nil, fmt.Errorf("layout contains %d entries, consider --index", len(m.Manifests)) + } + + desc := m.Manifests[0] + if desc.MediaType.IsImage() { + return l.Image(desc.Digest) + } else if desc.MediaType.IsIndex() { + return l.ImageIndex(desc.Digest) + } + + return nil, fmt.Errorf("layout contains non-image (mediaType: %q), consider --index", desc.MediaType) +} diff --git a/pkg/imgpkg/imagetar/tar_oci_writer.go b/pkg/imgpkg/imagetar/tar_oci_writer.go new file mode 100644 index 000000000..abc57209f --- /dev/null +++ b/pkg/imgpkg/imagetar/tar_oci_writer.go @@ -0,0 +1,66 @@ +// Copyright 2020 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package imagetar + +import ( + "fmt" + "os" + + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/imagedesc" +) + +func lmao() { + fmt.Println("jojop") +} + +func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { + + //file := tarFile{r.path} + + //Check if the path is a OCI layout directory + stat, err := os.Stat(r.path) + if err != nil { + return nil, err + } + + if !stat.IsDir() { + //give error "not a directory" + return nil, err + } + + //FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. + + //Get the oci layout rooted in the file system at path + l, err := layout.FromPath(r.path) + if err != nil { + return nil, err + } + + //Conversion from layout.ImageIndex to imageOrIndex + var imgOrIndexes []imagedesc.ImageOrIndex + ImageIndex, err := l.ImageIndex() + if err != nil { + return nil, err + } + + imgOrIndexes[0].Index = ImageIndex + + //imgOrIndexes = append(imgOrIndexes, imagedesc.ImageOrIndex{Index: ImageIndex}) + //Handle multiples cases when manifests in index.json are >1 + + //IMP + //img, err := loadImage(path, false) + //ok is the bool that tells us if the image is an image or an index + //t is the v1.Image or v1.ImageIndex + // t, _ := img.(v1.Image) + + // if err != nil { + // return nil, err + // } + + // crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") + + return result, nil +} diff --git a/pkg/imgpkg/plainimage/contents.go b/pkg/imgpkg/plainimage/contents.go index 255b83989..9858db46b 100644 --- a/pkg/imgpkg/plainimage/contents.go +++ b/pkg/imgpkg/plainimage/contents.go @@ -50,7 +50,7 @@ func (i Contents) Push(uploadRef regname.Tag, labels map[string]string, writer I } if i.ociTarPath != "" { - err = crane.SaveOCI(img, i.ociTarPath) + err = crane.SaveOCI(img.Image, i.ociTarPath) if err != nil { return "", err } diff --git a/pkg/imgpkg/registry/registry.go b/pkg/imgpkg/registry/registry.go index 9a10033b4..2a1e320cd 100644 --- a/pkg/imgpkg/registry/registry.go +++ b/pkg/imgpkg/registry/registry.go @@ -410,6 +410,8 @@ func (r *SimpleRegistry) MultiWrite(imageOrIndexesToUpload map[regname.Reference if updatesCh != nil { rOpts = append(rOpts, regremote.WithProgress(updatesCh)) } + + //THIS HERE FINAL PUSHES return regremote.MultiWrite(overriddenImageOrIndexesToUploadRef, rOpts...) } diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/README.md b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/README.md new file mode 100644 index 000000000..54bee6d9f --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/README.md @@ -0,0 +1,5 @@ +# `layout` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/layout) + +The `layout` package implements support for interacting with an [OCI Image Layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md). diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go new file mode 100644 index 000000000..2e5f4358d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go @@ -0,0 +1,37 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Blob returns a blob with the given hash from the Path. +func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) { + return os.Open(l.blobPath(h)) +} + +// Bytes is a convenience function to return a blob from the Path as +// a byte slice. +func (l Path) Bytes(h v1.Hash) ([]byte, error) { + return os.ReadFile(l.blobPath(h)) +} + +func (l Path) blobPath(h v1.Hash) string { + return l.path("blobs", h.Algorithm, h.Hex) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go new file mode 100644 index 000000000..d80d27363 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go @@ -0,0 +1,19 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout provides facilities for reading/writing artifacts from/to +// an OCI image layout on disk, see: +// +// https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go new file mode 100644 index 000000000..c9ae9665c --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go @@ -0,0 +1,139 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "fmt" + "io" + "os" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layoutImage struct { + path Path + desc v1.Descriptor + manifestLock sync.Mutex // Protects rawManifest + rawManifest []byte +} + +var _ partial.CompressedImageCore = (*layoutImage)(nil) + +// Image reads a v1.Image with digest h from the Path. +func (l Path) Image(h v1.Hash) (v1.Image, error) { + ii, err := l.ImageIndex() + if err != nil { + return nil, err + } + + return ii.Image(h) +} + +func (li *layoutImage) MediaType() (types.MediaType, error) { + return li.desc.MediaType, nil +} + +// Implements WithManifest for partial.Blobset. +func (li *layoutImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(li) +} + +func (li *layoutImage) RawManifest() ([]byte, error) { + li.manifestLock.Lock() + defer li.manifestLock.Unlock() + if li.rawManifest != nil { + return li.rawManifest, nil + } + + b, err := li.path.Bytes(li.desc.Digest) + if err != nil { + return nil, err + } + + li.rawManifest = b + return li.rawManifest, nil +} + +func (li *layoutImage) RawConfigFile() ([]byte, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + return li.path.Bytes(manifest.Config.Digest) +} + +func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + if h == manifest.Config.Digest { + return &compressedBlob{ + path: li.path, + desc: manifest.Config, + }, nil + } + + for _, desc := range manifest.Layers { + if h == desc.Digest { + return &compressedBlob{ + path: li.path, + desc: desc, + }, nil + } + } + + return nil, fmt.Errorf("could not find layer in image: %s", h) +} + +type compressedBlob struct { + path Path + desc v1.Descriptor +} + +func (b *compressedBlob) Digest() (v1.Hash, error) { + return b.desc.Digest, nil +} + +func (b *compressedBlob) Compressed() (io.ReadCloser, error) { + return b.path.Blob(b.desc.Digest) +} + +func (b *compressedBlob) Size() (int64, error) { + return b.desc.Size, nil +} + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} + +// Descriptor implements partial.withDescriptor. +func (b *compressedBlob) Descriptor() (*v1.Descriptor, error) { + return &b.desc, nil +} + +// See partial.Exists. +func (b *compressedBlob) Exists() (bool, error) { + _, err := os.Stat(b.path.blobPath(b.desc.Digest)) + if os.IsNotExist(err) { + return false, nil + } + return err == nil, err +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go new file mode 100644 index 000000000..7404f186b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go @@ -0,0 +1,161 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var _ v1.ImageIndex = (*layoutIndex)(nil) + +type layoutIndex struct { + mediaType types.MediaType + path Path + rawIndex []byte +} + +// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex. +func ImageIndexFromPath(path string) (v1.ImageIndex, error) { + lp, err := FromPath(path) + if err != nil { + return nil, err + } + return lp.ImageIndex() +} + +// ImageIndex returns a v1.ImageIndex for the Path. +func (l Path) ImageIndex() (v1.ImageIndex, error) { + rawIndex, err := os.ReadFile(l.path("index.json")) + if err != nil { + return nil, err + } + + idx := &layoutIndex{ + mediaType: types.OCIImageIndex, + path: l, + rawIndex: rawIndex, + } + + return idx, nil +} + +func (i *layoutIndex) MediaType() (types.MediaType, error) { + return i.mediaType, nil +} + +func (i *layoutIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *layoutIndex) Size() (int64, error) { + return partial.Size(i) +} + +func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) { + var index v1.IndexManifest + err := json.Unmarshal(i.rawIndex, &index) + return &index, err +} + +func (i *layoutIndex) RawManifest() ([]byte, error) { + return i.rawIndex, nil +} + +func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + img := &layoutImage{ + path: i.path, + desc: *desc, + } + return partial.CompressedToImage(img) +} + +func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + rawIndex, err := i.path.Bytes(h) + if err != nil { + return nil, err + } + + return &layoutIndex{ + mediaType: desc.MediaType, + path: i.path, + rawIndex: rawIndex, + }, nil +} + +func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) { + return i.path.Blob(h) +} + +func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) { + im, err := i.IndexManifest() + if err != nil { + return nil, err + } + + if h == (v1.Hash{}) { + if len(im.Manifests) != 1 { + return nil, errors.New("oci layout must contain only a single image to be used with layout.Image") + } + return &(im.Manifests)[0], nil + } + + for _, desc := range im.Manifests { + if desc.Digest == h { + return &desc, nil + } + } + + return nil, fmt.Errorf("could not find descriptor in index: %s", h) +} + +// TODO: Pull this out into methods on types.MediaType? e.g. instead, have: +// * mt.IsIndex() +// * mt.IsImage() +func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool { + for _, allowed := range expected { + if mt == allowed { + return true + } + } + return false +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go new file mode 100644 index 000000000..a031ff5ae --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go @@ -0,0 +1,25 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import "path/filepath" + +// Path represents an OCI image layout rooted in a file system path +type Path string + +func (l Path) path(elem ...string) string { + complete := []string{string(l)} + return filepath.Join(append(complete, elem...)...) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go new file mode 100644 index 000000000..a26f9f371 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -0,0 +1,71 @@ +// Copyright 2019 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// Option is a functional option for Layout. +type Option func(*options) + +type options struct { + descOpts []descriptorOption +} + +func makeOptions(opts ...Option) *options { + o := &options{ + descOpts: []descriptorOption{}, + } + for _, apply := range opts { + apply(o) + } + return o +} + +type descriptorOption func(*v1.Descriptor) + +// WithAnnotations adds annotations to the artifact descriptor. +func WithAnnotations(annotations map[string]string) Option { + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + }) + } +} + +// WithURLs adds urls to the artifact descriptor. +func WithURLs(urls []string) Option { + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + }) + } +} + +// WithPlatform sets the platform of the artifact descriptor. +func WithPlatform(platform v1.Platform) Option { + return func(o *options) { + o.descOpts = append(o.descOpts, func(desc *v1.Descriptor) { + desc.Platform = &platform + }) + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go new file mode 100644 index 000000000..796abc7dd --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go @@ -0,0 +1,32 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "os" + "path/filepath" +) + +// FromPath reads an OCI image layout at path and constructs a layout.Path. +func FromPath(path string) (Path, error) { + // TODO: check oci-layout exists + + _, err := os.Stat(filepath.Join(path, "index.json")) + if err != nil { + return "", err + } + + return Path(path), nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go new file mode 100644 index 000000000..d6e35c391 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go @@ -0,0 +1,482 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/match" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/stream" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +var layoutFile = `{ + "imageLayoutVersion": "1.0.0" +}` + +// AppendImage writes a v1.Image to the Path and updates +// the index.json to reference it. +func (l Path) AppendImage(img v1.Image, options ...Option) error { + if err := l.WriteImage(img); err != nil { + return err + } + + desc, err := partial.Descriptor(img) + if err != nil { + return err + } + + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) + } + + return l.AppendDescriptor(*desc) +} + +// AppendIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it. +func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { + if err := l.WriteIndex(ii); err != nil { + return err + } + + desc, err := partial.Descriptor(ii) + if err != nil { + return err + } + + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) + } + + return l.AppendDescriptor(*desc) +} + +// AppendDescriptor adds a descriptor to the index.json of the Path. +func (l Path) AppendDescriptor(desc v1.Descriptor) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.WriteFile("index.json", rawIndex, os.ModePerm) +} + +// ReplaceImage writes a v1.Image to the Path and updates +// the index.json to reference it, replacing any existing one that matches matcher, if found. +func (l Path) ReplaceImage(img v1.Image, matcher match.Matcher, options ...Option) error { + if err := l.WriteImage(img); err != nil { + return err + } + + return l.replaceDescriptor(img, matcher, options...) +} + +// ReplaceIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it, replacing any existing one that matches matcher, if found. +func (l Path) ReplaceIndex(ii v1.ImageIndex, matcher match.Matcher, options ...Option) error { + if err := l.WriteIndex(ii); err != nil { + return err + } + + return l.replaceDescriptor(ii, matcher, options...) +} + +// replaceDescriptor adds a descriptor to the index.json of the Path, replacing +// any one matching matcher, if found. +func (l Path) replaceDescriptor(append mutate.Appendable, matcher match.Matcher, options ...Option) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + desc, err := partial.Descriptor(append) + if err != nil { + return err + } + + o := makeOptions(options...) + for _, opt := range o.descOpts { + opt(desc) + } + + add := mutate.IndexAddendum{ + Add: append, + Descriptor: *desc, + } + ii = mutate.AppendManifests(mutate.RemoveManifests(ii, matcher), add) + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.WriteFile("index.json", rawIndex, os.ModePerm) +} + +// RemoveDescriptors removes any descriptors that match the match.Matcher from the index.json of the Path. +func (l Path) RemoveDescriptors(matcher match.Matcher) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + ii = mutate.RemoveManifests(ii, matcher) + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.WriteFile("index.json", rawIndex, os.ModePerm) +} + +// WriteFile write a file with arbitrary data at an arbitrary location in a v1 +// layout. Used mostly internally to write files like "oci-layout" and +// "index.json", also can be used to write other arbitrary files. Do *not* use +// this to write blobs. Use only WriteBlob() for that. +func (l Path) WriteFile(name string, data []byte, perm os.FileMode) error { + if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + return os.WriteFile(l.path(name), data, perm) +} + +// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at +// blobs/{hash.Algorithm}/{hash.Hex}. +func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error { + return l.writeBlob(hash, -1, r, nil) +} + +func (l Path) writeBlob(hash v1.Hash, size int64, rc io.ReadCloser, renamer func() (v1.Hash, error)) error { + defer rc.Close() + if hash.Hex == "" && renamer == nil { + panic("writeBlob called an invalid hash and no renamer") + } + + dir := l.path("blobs", hash.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + // Check if blob already exists and is the correct size + file := filepath.Join(dir, hash.Hex) + if s, err := os.Stat(file); err == nil && !s.IsDir() && (s.Size() == size || size == -1) { + return nil + } + + // If a renamer func was provided write to a temporary file + open := func() (*os.File, error) { return os.Create(file) } + if renamer != nil { + open = func() (*os.File, error) { return os.CreateTemp(dir, hash.Hex) } + } + w, err := open() + if err != nil { + return err + } + if renamer != nil { + // Delete temp file if an error is encountered before renaming + defer func() { + if err := os.Remove(w.Name()); err != nil && !errors.Is(err, os.ErrNotExist) { + logs.Warn.Printf("error removing temporary file after encountering an error while writing blob: %v", err) + } + }() + } + defer w.Close() + + // Write to file and exit if not renaming + if n, err := io.Copy(w, rc); err != nil || renamer == nil { + return err + } else if size != -1 && n != size { + return fmt.Errorf("expected blob size %d, but only wrote %d", size, n) + } + + // Always close reader before renaming, since Close computes the digest in + // the case of streaming layers. If Close is not called explicitly, it will + // occur in a goroutine that is not guaranteed to succeed before renamer is + // called. When renamer is the layer's Digest method, it can return + // ErrNotComputed. + if err := rc.Close(); err != nil { + return err + } + + // Always close file before renaming + if err := w.Close(); err != nil { + return err + } + + // Rename file based on the final hash + finalHash, err := renamer() + if err != nil { + return fmt.Errorf("error getting final digest of layer: %w", err) + } + + renamePath := l.path("blobs", finalHash.Algorithm, finalHash.Hex) + return os.Rename(w.Name(), renamePath) +} + +// writeLayer writes the compressed layer to a blob. Unlike WriteBlob it will +// write to a temporary file (suffixed with .tmp) within the layout until the +// compressed reader is fully consumed and written to disk. Also unlike +// WriteBlob, it will not skip writing and exit without error when a blob file +// exists, but does not have the correct size. (The blob hash is not +// considered, because it may be expensive to compute.) +func (l Path) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if errors.Is(err, stream.ErrNotComputed) { + // Allow digest errors, since streams may not have calculated the hash + // yet. Instead, use an empty value, which will be transformed into a + // random file name with `os.CreateTemp` and the final digest will be + // calculated after writing to a temp file and before renaming to the + // final path. + d = v1.Hash{Algorithm: "sha256", Hex: ""} + } else if err != nil { + return err + } + + s, err := layer.Size() + if errors.Is(err, stream.ErrNotComputed) { + // Allow size errors, since streams may not have calculated the size + // yet. Instead, use zero as a sentinel value meaning that no size + // comparison can be done and any sized blob file should be considered + // valid and not overwritten. + // + // TODO: Provide an option to always overwrite blobs. + s = -1 + } else if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + if err := l.writeBlob(d, s, r, layer.Digest); err != nil { + return fmt.Errorf("error writing layer: %w", err) + } + return nil +} + +// RemoveBlob removes a file from the blobs directory in the Path +// at blobs/{hash.Algorithm}/{hash.Hex} +// It does *not* remove any reference to it from other manifests or indexes, or +// from the root index.json. +func (l Path) RemoveBlob(hash v1.Hash) error { + dir := l.path("blobs", hash.Algorithm) + err := os.Remove(filepath.Join(dir, hash.Hex)) + if err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// WriteImage writes an image, including its manifest, config and all of its +// layers, to the blobs directory. If any blob already exists, as determined by +// the hash filename, does not write it. +// This function does *not* update the `index.json` file. If you want to write the +// image and also update the `index.json`, call AppendImage(), which wraps this +// and also updates the `index.json`. +func (l Path) WriteImage(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + // Write the layers concurrently. + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return err + } + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := l.WriteBlob(cfgName, io.NopCloser(bytes.NewReader(cfgBlob))); err != nil { + return err + } + + // Write the img manifest. + d, err := img.Digest() + if err != nil { + return err + } + manifest, err := img.RawManifest() + if err != nil { + return err + } + + return l.WriteBlob(d, io.NopCloser(bytes.NewReader(manifest))) +} + +type withLayer interface { + Layer(v1.Hash) (v1.Layer, error) +} + +type withBlob interface { + Blob(v1.Hash) (io.ReadCloser, error) +} + +func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + // Walk the descriptors and write any v1.Image or v1.ImageIndex that we find. + // If we come across something we don't expect, just write it as a blob. + for _, desc := range index.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := l.WriteIndex(ii); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := l.WriteImage(img); err != nil { + return err + } + default: + // TODO: The layout could reference arbitrary things, which we should + // probably just pass through. + + var blob io.ReadCloser + // Workaround for #819. + if wl, ok := ii.(withLayer); ok { + layer, lerr := wl.Layer(desc.Digest) + if lerr != nil { + return lerr + } + blob, err = layer.Compressed() + } else if wb, ok := ii.(withBlob); ok { + blob, err = wb.Blob(desc.Digest) + } + if err != nil { + return err + } + if err := l.WriteBlob(desc.Digest, blob); err != nil { + return err + } + } + } + + rawIndex, err := ii.RawManifest() + if err != nil { + return err + } + + return l.WriteFile(indexFile, rawIndex, os.ModePerm) +} + +// WriteIndex writes an index to the blobs directory. Walks down the children, +// including its children manifests and/or indexes, and down the tree until all of +// config and all layers, have been written. If any blob already exists, as determined by +// the hash filename, does not write it. +// This function does *not* update the `index.json` file. If you want to write the +// index and also update the `index.json`, call AppendIndex(), which wraps this +// and also updates the `index.json`. +func (l Path) WriteIndex(ii v1.ImageIndex) error { + // Always just write oci-layout file, since it's small. + if err := l.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil { + return err + } + + h, err := ii.Digest() + if err != nil { + return err + } + + indexFile := filepath.Join("blobs", h.Algorithm, h.Hex) + return l.writeIndexToFile(indexFile, ii) +} + +// Write constructs a Path at path from an ImageIndex. +// +// The contents are written in the following format: +// At the top level, there is: +// +// One oci-layout file containing the version of this image-layout. +// One index.json file listing descriptors for the contained images. +// +// Under blobs/, there is, for each image: +// +// One file for each layer, named after the layer's SHA. +// One file for each config blob, named after its SHA. +// One file for each manifest blob, named after its SHA. +func Write(path string, ii v1.ImageIndex) (Path, error) { + lp := Path(path) + // Always just write oci-layout file, since it's small. + if err := lp.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil { + return "", err + } + + // TODO create blobs/ in case there is a blobs file which would prevent the directory from being created + + return lp, lp.writeIndexToFile("index.json", ii) +} From 5e8114767859b9e98a63eb5e7db151a1794447a4 Mon Sep 17 00:00:00 2001 From: ashpect Date: Thu, 2 Nov 2023 11:17:31 +0530 Subject: [PATCH 03/20] wip Signed-off-by: ashpect --- pkg/imgpkg/imagetar/tar_oci_writer.go | 49 +++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/pkg/imgpkg/imagetar/tar_oci_writer.go b/pkg/imgpkg/imagetar/tar_oci_writer.go index abc57209f..d72fd6526 100644 --- a/pkg/imgpkg/imagetar/tar_oci_writer.go +++ b/pkg/imgpkg/imagetar/tar_oci_writer.go @@ -7,10 +7,25 @@ import ( "fmt" "os" + regv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/imagedesc" ) +type MyImageIndex struct { + Index regv1.ImageIndex + Reference string + ImageTag string +} + +func (mi MyImageIndex) Ref() string { + return "my-image-ref" +} + +func (mi MyImageIndex) Tag() string { + return "latest" +} + func lmao() { fmt.Println("jojop") } @@ -37,15 +52,43 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { if err != nil { return nil, err } + //a, err := l.ImageIndex() + + myImageIndex := MyImageIndex{ + Index: nil, + Reference: "", + ImageTag: "", + } + + imageOrIndex := imagedesc.ImageOrIndex{ + Image: nil, + Index: &myImageIndex, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + OrigRef: "original-reference", + } + + ref := imageOrIndex.Ref() + fmt.Println("Ref:", ref) //Conversion from layout.ImageIndex to imageOrIndex - var imgOrIndexes []imagedesc.ImageOrIndex + //var imgOrIndexes imagedesc.ImageOrIndex ImageIndex, err := l.ImageIndex() if err != nil { return nil, err } + fmt.Println("ImageIndex:", ImageIndex) + + // var myImageIndex imagedesc.ImageIndexWithRef = &MyImageIndex{ + // Image: someImage, // Replace with an actual ImageIndex instance + // Index: someIndex, // Replace with an actual ImageIndex instance + // SomeRef: "defaultRef", // Set the default reference string + // SomeTag: "defaultTag", // Set the default tag string + // } - imgOrIndexes[0].Index = ImageIndex + //imgOrIndexes.Index = &ImageIndex //imgOrIndexes = append(imgOrIndexes, imagedesc.ImageOrIndex{Index: ImageIndex}) //Handle multiples cases when manifests in index.json are >1 @@ -62,5 +105,5 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { // crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") - return result, nil + return nil, nil } From 7bf3e603e44a44e890e23ff434a99b36ad55a532 Mon Sep 17 00:00:00 2001 From: ashpect Date: Tue, 7 Nov 2023 07:38:43 +0530 Subject: [PATCH 04/20] updates Signed-off-by: ashpect --- pkg/imgpkg/imagetar/tar_oci_writer.go | 38 +++++++++++++++------------ 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/pkg/imgpkg/imagetar/tar_oci_writer.go b/pkg/imgpkg/imagetar/tar_oci_writer.go index d72fd6526..d293b6f62 100644 --- a/pkg/imgpkg/imagetar/tar_oci_writer.go +++ b/pkg/imgpkg/imagetar/tar_oci_writer.go @@ -13,19 +13,21 @@ import ( ) type MyImageIndex struct { - Index regv1.ImageIndex - Reference string - ImageTag string + Index regv1.ImageIndex } -func (mi MyImageIndex) Ref() string { +func (mi *MyImageIndex) Ref() string { return "my-image-ref" } -func (mi MyImageIndex) Tag() string { +func (mi *MyImageIndex) Tag() string { return "latest" } +func (mi *MyImageIndex) Digest() (regv1.Hash, error) { + return regv1.Hash{}, nil +} + func lmao() { fmt.Println("jojop") } @@ -55,20 +57,22 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { //a, err := l.ImageIndex() myImageIndex := MyImageIndex{ - Index: nil, - Reference: "", - ImageTag: "", + Index: nil, } - imageOrIndex := imagedesc.ImageOrIndex{ - Image: nil, - Index: &myImageIndex, - Labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, - OrigRef: "original-reference", - } + var i imagedesc.ImageIndexWithRef + + i = &myImageIndex + + // imageOrIndex := imagedesc.ImageOrIndex{ + // Image: nil, + // Index: i, + // Labels: map[string]string{ + // "label1": "value1", + // "label2": "value2", + // }, + // OrigRef: "original-reference", + // } ref := imageOrIndex.Ref() fmt.Println("Ref:", ref) From e78123749675ed2cdfb8128f731c30b992354ca6 Mon Sep 17 00:00:00 2001 From: ashpect Date: Thu, 9 Nov 2023 17:00:19 +0530 Subject: [PATCH 05/20] Parsed ociformat to ImageOrIndex struct Signed-off-by: ashpect --- pkg/imgpkg/imageset/tar_image_set.go | 3 +- pkg/imgpkg/imagetar/tar_oci_writer.go | 94 +++++++++++++++------------ 2 files changed, 54 insertions(+), 43 deletions(-) diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index ac937c02b..4e9ab743c 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -130,10 +130,11 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr if tarisoci { imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci() + //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") } else { imgOrIndexes, err = imagetar.NewTarReader(path).Read() //for testing - //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") + crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") //crane.SaveOCI(*imgOrIndexes[1].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a1") } diff --git a/pkg/imgpkg/imagetar/tar_oci_writer.go b/pkg/imgpkg/imagetar/tar_oci_writer.go index d293b6f62..cdff97617 100644 --- a/pkg/imgpkg/imagetar/tar_oci_writer.go +++ b/pkg/imgpkg/imagetar/tar_oci_writer.go @@ -9,6 +9,7 @@ import ( regv1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/google/go-containerregistry/pkg/v1/types" "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/imagedesc" ) @@ -16,22 +17,45 @@ type MyImageIndex struct { Index regv1.ImageIndex } -func (mi *MyImageIndex) Ref() string { +func (mi MyImageIndex) Ref() string { return "my-image-ref" } -func (mi *MyImageIndex) Tag() string { +func (mi MyImageIndex) Tag() string { return "latest" } -func (mi *MyImageIndex) Digest() (regv1.Hash, error) { - return regv1.Hash{}, nil +func (mi MyImageIndex) MediaType() (types.MediaType, error) { + return mi.Index.MediaType() } -func lmao() { - fmt.Println("jojop") +func (mi MyImageIndex) Digest() (regv1.Hash, error) { + return mi.Index.Digest() } +func (mi MyImageIndex) Size() (int64, error) { + return mi.Index.Size() +} + +func (mi MyImageIndex) IndexManifest() (*regv1.IndexManifest, error) { + return mi.Index.IndexManifest() +} + +func (mi MyImageIndex) RawManifest() ([]byte, error) { + return mi.Index.RawManifest() +} + +func (mi MyImageIndex) Image(h regv1.Hash) (regv1.Image, error) { + return mi.Index.Image(h) +} + +func (mi MyImageIndex) ImageIndex(h regv1.Hash) (regv1.ImageIndex, error) { + return mi.Index.ImageIndex(h) +} + +// To explicitely implement the ImageIndex interface +var _ regv1.ImageIndex = MyImageIndex{} + func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { //file := tarFile{r.path} @@ -49,65 +73,51 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { //FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. - //Get the oci layout rooted in the file system at path + //Get the oci layout rooted in the file system at path, layout index struct l, err := layout.FromPath(r.path) if err != nil { return nil, err } - //a, err := l.ImageIndex() + ImageIndex, err := l.ImageIndex() myImageIndex := MyImageIndex{ - Index: nil, + Index: ImageIndex, } - var i imagedesc.ImageIndexWithRef + var i imagedesc.ImageIndexWithRef = myImageIndex - i = &myImageIndex - - // imageOrIndex := imagedesc.ImageOrIndex{ - // Image: nil, - // Index: i, - // Labels: map[string]string{ - // "label1": "value1", - // "label2": "value2", - // }, - // OrigRef: "original-reference", - // } + imageOrIndex := imagedesc.ImageOrIndex{ + Image: nil, + Index: &i, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + OrigRef: "original-reference", + } ref := imageOrIndex.Ref() fmt.Println("Ref:", ref) - //Conversion from layout.ImageIndex to imageOrIndex - //var imgOrIndexes imagedesc.ImageOrIndex - ImageIndex, err := l.ImageIndex() - if err != nil { - return nil, err - } - fmt.Println("ImageIndex:", ImageIndex) - - // var myImageIndex imagedesc.ImageIndexWithRef = &MyImageIndex{ - // Image: someImage, // Replace with an actual ImageIndex instance - // Index: someIndex, // Replace with an actual ImageIndex instance - // SomeRef: "defaultRef", // Set the default reference string - // SomeTag: "defaultTag", // Set the default tag string - // } - - //imgOrIndexes.Index = &ImageIndex + //add imageOrIndex to the slice of imageOrIndex + var imageOrIndexSlice []imagedesc.ImageOrIndex + imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) //imgOrIndexes = append(imgOrIndexes, imagedesc.ImageOrIndex{Index: ImageIndex}) //Handle multiples cases when manifests in index.json are >1 //IMP - //img, err := loadImage(path, false) //ok is the bool that tells us if the image is an image or an index //t is the v1.Image or v1.ImageIndex - // t, _ := img.(v1.Image) - + // t, _ := ImageIndex.(v1.Image) // if err != nil { // return nil, err // } - // crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") + //print the slice + fmt.Println(imageOrIndexSlice) + + //crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") - return nil, nil + return imageOrIndexSlice, nil } From b228a42a687056681f11c4a11c6e22326fa60ca4 Mon Sep 17 00:00:00 2001 From: ashpect Date: Fri, 10 Nov 2023 09:14:27 +0530 Subject: [PATCH 06/20] Update wip Signed-off-by: ashpect --- pkg/imgpkg/imagetar/tar_oci_writer.go | 42 +++++++++++++++++++-------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/imgpkg/imagetar/tar_oci_writer.go b/pkg/imgpkg/imagetar/tar_oci_writer.go index cdff97617..63834bd1f 100644 --- a/pkg/imgpkg/imagetar/tar_oci_writer.go +++ b/pkg/imgpkg/imagetar/tar_oci_writer.go @@ -4,7 +4,6 @@ package imagetar import ( - "fmt" "os" regv1 "github.com/google/go-containerregistry/pkg/v1" @@ -15,14 +14,24 @@ import ( type MyImageIndex struct { Index regv1.ImageIndex + ref string + tag string } func (mi MyImageIndex) Ref() string { - return "my-image-ref" + return mi.ref +} + +func (mi *MyImageIndex) SetRef(ref string) { + mi.ref = ref } func (mi MyImageIndex) Tag() string { - return "latest" + return mi.tag +} + +func (mi *MyImageIndex) SetTag(tag string) { + mi.tag = tag } func (mi MyImageIndex) MediaType() (types.MediaType, error) { @@ -58,8 +67,6 @@ var _ regv1.ImageIndex = MyImageIndex{} func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { - //file := tarFile{r.path} - //Check if the path is a OCI layout directory stat, err := os.Stat(r.path) if err != nil { @@ -71,19 +78,35 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { return nil, err } - //FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. + //TODO : FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. //Get the oci layout rooted in the file system at path, layout index struct l, err := layout.FromPath(r.path) if err != nil { return nil, err } + ImageIndex, err := l.ImageIndex() + digest, err := ImageIndex.Digest() + + //fmt.Println("ImageIndex's digest :", digest) + myImageIndex := MyImageIndex{ Index: ImageIndex, + ref: "", + tag: "latest", } + //convert digest into a string + digestStr := digest.String() + + ref := "index.docker.io/" + "ashpect/testrepo22@" + digestStr + + //fmt.Println("Ref to be updated :", ref) + + myImageIndex.SetRef(ref) + var i imagedesc.ImageIndexWithRef = myImageIndex imageOrIndex := imagedesc.ImageOrIndex{ @@ -96,9 +119,6 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { OrigRef: "original-reference", } - ref := imageOrIndex.Ref() - fmt.Println("Ref:", ref) - //add imageOrIndex to the slice of imageOrIndex var imageOrIndexSlice []imagedesc.ImageOrIndex imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) @@ -113,9 +133,7 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { // if err != nil { // return nil, err // } - - //print the slice - fmt.Println(imageOrIndexSlice) + // ----> file := tarFile{r.path} //crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") From 4e1dbc2f17b4fde9f53704b1e13d868e3e14a426 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 15 Nov 2023 00:07:07 +0530 Subject: [PATCH 07/20] cleanup Signed-off-by: ashpect --- pkg/imgpkg/imagedesc/image_intermediate.go | 61 +++++++++ pkg/imgpkg/imageset/tar_image_set.go | 1 + pkg/imgpkg/imagetar/tar_oci_writer.go | 141 --------------------- pkg/imgpkg/imagetar/tar_reader.go | 78 ++++++++++++ 4 files changed, 140 insertions(+), 141 deletions(-) create mode 100644 pkg/imgpkg/imagedesc/image_intermediate.go delete mode 100644 pkg/imgpkg/imagetar/tar_oci_writer.go diff --git a/pkg/imgpkg/imagedesc/image_intermediate.go b/pkg/imgpkg/imagedesc/image_intermediate.go new file mode 100644 index 000000000..be3732aaf --- /dev/null +++ b/pkg/imgpkg/imagedesc/image_intermediate.go @@ -0,0 +1,61 @@ +// Copyright 2020 VMware, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package imagedesc + +import ( + regv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type ImageIndexIntermediate struct { + Index regv1.ImageIndex + ref string + tag string +} + +func (mi ImageIndexIntermediate) Ref() string { + return mi.ref +} + +func (mi *ImageIndexIntermediate) SetRef(ref string) { + mi.ref = ref +} + +func (mi ImageIndexIntermediate) Tag() string { + return mi.tag +} + +func (mi *ImageIndexIntermediate) SetTag(tag string) { + mi.tag = tag +} + +func (mi ImageIndexIntermediate) MediaType() (types.MediaType, error) { + return mi.Index.MediaType() +} + +func (mi ImageIndexIntermediate) Digest() (regv1.Hash, error) { + return mi.Index.Digest() +} + +func (mi ImageIndexIntermediate) Size() (int64, error) { + return mi.Index.Size() +} + +func (mi ImageIndexIntermediate) IndexManifest() (*regv1.IndexManifest, error) { + return mi.Index.IndexManifest() +} + +func (mi ImageIndexIntermediate) RawManifest() ([]byte, error) { + return mi.Index.RawManifest() +} + +func (mi ImageIndexIntermediate) Image(h regv1.Hash) (regv1.Image, error) { + return mi.Index.Image(h) +} + +func (mi ImageIndexIntermediate) ImageIndex(h regv1.Hash) (regv1.ImageIndex, error) { + return mi.Index.ImageIndex(h) +} + +var _ regv1.ImageIndex = ImageIndexIntermediate{} diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index 4e9ab743c..246b8227c 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -133,6 +133,7 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") } else { imgOrIndexes, err = imagetar.NewTarReader(path).Read() + fmt.Println(imgOrIndexes[0].Ref()) //for testing crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") //crane.SaveOCI(*imgOrIndexes[1].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a1") diff --git a/pkg/imgpkg/imagetar/tar_oci_writer.go b/pkg/imgpkg/imagetar/tar_oci_writer.go deleted file mode 100644 index 63834bd1f..000000000 --- a/pkg/imgpkg/imagetar/tar_oci_writer.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2020 VMware, Inc. -// SPDX-License-Identifier: Apache-2.0 - -package imagetar - -import ( - "os" - - regv1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" - "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/vmware-tanzu/carvel-imgpkg/pkg/imgpkg/imagedesc" -) - -type MyImageIndex struct { - Index regv1.ImageIndex - ref string - tag string -} - -func (mi MyImageIndex) Ref() string { - return mi.ref -} - -func (mi *MyImageIndex) SetRef(ref string) { - mi.ref = ref -} - -func (mi MyImageIndex) Tag() string { - return mi.tag -} - -func (mi *MyImageIndex) SetTag(tag string) { - mi.tag = tag -} - -func (mi MyImageIndex) MediaType() (types.MediaType, error) { - return mi.Index.MediaType() -} - -func (mi MyImageIndex) Digest() (regv1.Hash, error) { - return mi.Index.Digest() -} - -func (mi MyImageIndex) Size() (int64, error) { - return mi.Index.Size() -} - -func (mi MyImageIndex) IndexManifest() (*regv1.IndexManifest, error) { - return mi.Index.IndexManifest() -} - -func (mi MyImageIndex) RawManifest() ([]byte, error) { - return mi.Index.RawManifest() -} - -func (mi MyImageIndex) Image(h regv1.Hash) (regv1.Image, error) { - return mi.Index.Image(h) -} - -func (mi MyImageIndex) ImageIndex(h regv1.Hash) (regv1.ImageIndex, error) { - return mi.Index.ImageIndex(h) -} - -// To explicitely implement the ImageIndex interface -var _ regv1.ImageIndex = MyImageIndex{} - -func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { - - //Check if the path is a OCI layout directory - stat, err := os.Stat(r.path) - if err != nil { - return nil, err - } - - if !stat.IsDir() { - //give error "not a directory" - return nil, err - } - - //TODO : FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. - - //Get the oci layout rooted in the file system at path, layout index struct - l, err := layout.FromPath(r.path) - if err != nil { - return nil, err - } - - ImageIndex, err := l.ImageIndex() - - digest, err := ImageIndex.Digest() - - //fmt.Println("ImageIndex's digest :", digest) - - myImageIndex := MyImageIndex{ - Index: ImageIndex, - ref: "", - tag: "latest", - } - - //convert digest into a string - digestStr := digest.String() - - ref := "index.docker.io/" + "ashpect/testrepo22@" + digestStr - - //fmt.Println("Ref to be updated :", ref) - - myImageIndex.SetRef(ref) - - var i imagedesc.ImageIndexWithRef = myImageIndex - - imageOrIndex := imagedesc.ImageOrIndex{ - Image: nil, - Index: &i, - Labels: map[string]string{ - "label1": "value1", - "label2": "value2", - }, - OrigRef: "original-reference", - } - - //add imageOrIndex to the slice of imageOrIndex - var imageOrIndexSlice []imagedesc.ImageOrIndex - imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) - - //imgOrIndexes = append(imgOrIndexes, imagedesc.ImageOrIndex{Index: ImageIndex}) - //Handle multiples cases when manifests in index.json are >1 - - //IMP - //ok is the bool that tells us if the image is an image or an index - //t is the v1.Image or v1.ImageIndex - // t, _ := ImageIndex.(v1.Image) - // if err != nil { - // return nil, err - // } - // ----> file := tarFile{r.path} - - //crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") - - return imageOrIndexSlice, nil -} diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index 964e24cd8..239932c3b 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -6,11 +6,13 @@ package imagetar import ( "fmt" "io" + "os" "carvel.dev/imgpkg/pkg/imgpkg/imagedesc" "carvel.dev/imgpkg/pkg/imgpkg/imageutils/verify" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" ) type TarReader struct { @@ -148,3 +150,79 @@ func (r TarReader) getIdsFromManifest(file tarFile) (*imagedesc.ImageRefDescript } return ids, nil } + +func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { + + //Check if the path is a OCI layout directory + stat, err := os.Stat(r.path) + if err != nil { + return nil, err + } + + if !stat.IsDir() { + //give error "not a directory" + return nil, err + } + + //TODO : FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. + + //Get the oci layout rooted in the file system at path, layout index struct + l, err := layout.FromPath(r.path) + if err != nil { + return nil, err + } + + ImageIndex, err := l.ImageIndex() + + digest, err := ImageIndex.Digest() + + //fmt.Println("ImageIndex's digest :", digest) + + myImageIndex := imagedesc.ImageIndexIntermediate{ + Index: ImageIndex, + } + + fmt.Println("ImageIndex's tag :", myImageIndex.Tag()) + fmt.Println("ImageIndex's ref :", myImageIndex.Ref()) + + //convert digest into a string + digestStr := digest.String() + + ref := "index.docker.io/" + "ashpect/testrepo22@" + digestStr + + //fmt.Println("Ref to be updated :", ref) + + myImageIndex.SetRef(ref) + + var i imagedesc.ImageIndexWithRef = myImageIndex + + imageOrIndex := imagedesc.ImageOrIndex{ + Image: nil, + Index: &i, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + OrigRef: "original-reference", + } + + //add imageOrIndex to the slice of imageOrIndex + var imageOrIndexSlice []imagedesc.ImageOrIndex + imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) + + //imgOrIndexes = append(imgOrIndexes, imagedesc.ImageOrIndex{Index: ImageIndex}) + //Handle multiples cases when manifests in index.json are >1 + + //IMP + //ok is the bool that tells us if the image is an image or an index + //t is the v1.Image or v1.ImageIndex + // t, _ := ImageIndex.(v1.Image) + // if err != nil { + // return nil, err + // } + // ----> file := tarFile{r.path} + + //crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") + + return imageOrIndexSlice, nil +} From 7d432d756558f3b07a22b2211a46595faa7b0281 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 15 Nov 2023 00:15:23 +0530 Subject: [PATCH 08/20] Use layout pkg instead of crane Signed-off-by: ashpect --- pkg/imgpkg/plainimage/contents.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/imgpkg/plainimage/contents.go b/pkg/imgpkg/plainimage/contents.go index 9858db46b..c65197fc3 100644 --- a/pkg/imgpkg/plainimage/contents.go +++ b/pkg/imgpkg/plainimage/contents.go @@ -13,6 +13,8 @@ import ( "carvel.dev/imgpkg/pkg/imgpkg/internal/util" regname "github.com/google/go-containerregistry/pkg/name" regv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" regremote "github.com/google/go-containerregistry/pkg/v1/remote" ) @@ -50,8 +52,14 @@ func (i Contents) Push(uploadRef regname.Tag, labels map[string]string, writer I } if i.ociTarPath != "" { - err = crane.SaveOCI(img.Image, i.ociTarPath) + p, err := layout.FromPath(i.ociTarPath) if err != nil { + p, err = layout.Write(i.ociTarPath, empty.Index) + if err != nil { + return "", err + } + } + if err = p.AppendImage(img); err != nil { return "", err } } From ad32912d97be2c4d880110486b0e294ade5874d6 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 15 Nov 2023 00:50:15 +0530 Subject: [PATCH 09/20] cleanup Signed-off-by: ashpect --- pkg/imgpkg/imageset/tar_image_set.go | 57 +++------------------------- pkg/imgpkg/imagetar/tar_reader.go | 42 +++++--------------- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index 246b8227c..893abc1ed 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -129,14 +129,14 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr var err error if tarisoci { - imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci() + imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci(importRepo.Name()) + // ---- FOR TESTING PORPOSES ---- //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") } else { imgOrIndexes, err = imagetar.NewTarReader(path).Read() - fmt.Println(imgOrIndexes[0].Ref()) - //for testing - crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") - //crane.SaveOCI(*imgOrIndexes[1].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a1") + // fmt.Println(imgOrIndexes[0].Ref()) + // ---- FOR TESTING PORPOSES ---- + //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") } processedImages, err := i.imageSet.Import(imgOrIndexes, importRepo, registry) @@ -146,50 +146,3 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr return processedImages, err } - -// change function to use accordingly -func loadImage(path string, index bool) (partial.WithRawManifest, error) { - // to check if path is a tarball or a directory - stat, err := os.Stat(path) - if err != nil { - return nil, err - } - - if !stat.IsDir() { - img, err := crane.Load(path) - if err != nil { - return nil, fmt.Errorf("loading %s as tarball: %w", path, err) - } - return img, nil - } - - //if the path is a oci layout directory - l, err := layout.ImageIndexFromPath(path) - if err != nil { - return nil, fmt.Errorf("loading %s as OCI layout: %w", path, err) - } - - if index { - return l, nil - } - - //append l.ImageIndex() to imgOrIndexes - - m, err := l.IndexManifest() - - if err != nil { - return nil, err - } - if len(m.Manifests) != 1 { - return nil, fmt.Errorf("layout contains %d entries, consider --index", len(m.Manifests)) - } - - desc := m.Manifests[0] - if desc.MediaType.IsImage() { - return l.Image(desc.Digest) - } else if desc.MediaType.IsIndex() { - return l.ImageIndex(desc.Digest) - } - - return nil, fmt.Errorf("layout contains non-image (mediaType: %q), consider --index", desc.MediaType) -} diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index 239932c3b..9bd376ddd 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -151,7 +151,7 @@ func (r TarReader) getIdsFromManifest(file tarFile) (*imagedesc.ImageRefDescript return ids, nil } -func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { +func (r TarReader) ReadOci(reponame string) ([]imagedesc.ImageOrIndex, error) { //Check if the path is a OCI layout directory stat, err := os.Stat(r.path) @@ -166,7 +166,6 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { //TODO : FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. - //Get the oci layout rooted in the file system at path, layout index struct l, err := layout.FromPath(r.path) if err != nil { return nil, err @@ -174,27 +173,18 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { ImageIndex, err := l.ImageIndex() - digest, err := ImageIndex.Digest() - - //fmt.Println("ImageIndex's digest :", digest) - - myImageIndex := imagedesc.ImageIndexIntermediate{ + ImageIndexIntermediate := imagedesc.ImageIndexIntermediate{ Index: ImageIndex, } - fmt.Println("ImageIndex's tag :", myImageIndex.Tag()) - fmt.Println("ImageIndex's ref :", myImageIndex.Ref()) - - //convert digest into a string + // Update ref + digest, err := ImageIndex.Digest() digestStr := digest.String() + ref := reponame + "@" + digestStr + ImageIndexIntermediate.SetRef(ref) - ref := "index.docker.io/" + "ashpect/testrepo22@" + digestStr - - //fmt.Println("Ref to be updated :", ref) - - myImageIndex.SetRef(ref) - - var i imagedesc.ImageIndexWithRef = myImageIndex + // Create and populate imageOrIndex + var i imagedesc.ImageIndexWithRef = ImageIndexIntermediate imageOrIndex := imagedesc.ImageOrIndex{ Image: nil, @@ -206,23 +196,9 @@ func (r TarReader) ReadOci() ([]imagedesc.ImageOrIndex, error) { OrigRef: "original-reference", } - //add imageOrIndex to the slice of imageOrIndex + //Add imageOrIndex to the slice of imageOrIndex var imageOrIndexSlice []imagedesc.ImageOrIndex imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) - //imgOrIndexes = append(imgOrIndexes, imagedesc.ImageOrIndex{Index: ImageIndex}) - //Handle multiples cases when manifests in index.json are >1 - - //IMP - //ok is the bool that tells us if the image is an image or an index - //t is the v1.Image or v1.ImageIndex - // t, _ := ImageIndex.(v1.Image) - // if err != nil { - // return nil, err - // } - // ----> file := tarFile{r.path} - - //crane.SaveOCI(t, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/hotstuff") - return imageOrIndexSlice, nil } From d6f454a15cc3b5523db5f1cf84959e1c0ced3178 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 29 Nov 2023 04:58:56 +0530 Subject: [PATCH 10/20] update folders to tar Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 15 +++- pkg/imgpkg/image/tar_image.go | 129 +++++++++++++++++++++++++++ pkg/imgpkg/imageset/tar_image_set.go | 5 -- pkg/imgpkg/plainimage/contents.go | 4 + 4 files changed, 147 insertions(+), 6 deletions(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index 29bc16985..740b4b59f 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -5,8 +5,10 @@ package cmd import ( "fmt" + "os" ctlbundle "carvel.dev/imgpkg/pkg/imgpkg/bundle" + "carvel.dev/imgpkg/pkg/imgpkg/image" "carvel.dev/imgpkg/pkg/imgpkg/imageset" ctlimgset "carvel.dev/imgpkg/pkg/imgpkg/imageset" "carvel.dev/imgpkg/pkg/imgpkg/imagetar" @@ -61,6 +63,7 @@ func (c CopyRepoSrc) CopyToTar(dstPath string, resume bool) error { func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) { c.logger.Tracef("CopyToRepo(%s)\n", repo) + var tempDir string var processedImages *ctlimgset.ProcessedImages importRepo, err := regname.NewRepository(repo) if err != nil { @@ -73,7 +76,12 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) } if c.OciFlags.IsOci() { - processedImages, err = c.tarImageSet.Import(c.OciFlags.OcitoReg, importRepo, c.registry, true) + tempDir, err = image.ExtractOciTarGz(c.OciFlags.OcitoReg) + if err != nil { + return nil, err + } + processedImages, err = c.tarImageSet.Import(tempDir, importRepo, c.registry, true) + } else { processedImages, err = c.tarImageSet.Import(c.TarFlags.TarSrc, importRepo, c.registry, false) } @@ -139,6 +147,11 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) return nil, fmt.Errorf("Tagging images: %s", err) } + err = os.RemoveAll(tempDir) + if err != nil { + fmt.Println("Error cleaning up temporary directory:", err) + } + return processedImages, nil } diff --git a/pkg/imgpkg/image/tar_image.go b/pkg/imgpkg/image/tar_image.go index bcdd4f7ea..960dc0f3a 100644 --- a/pkg/imgpkg/image/tar_image.go +++ b/pkg/imgpkg/image/tar_image.go @@ -5,8 +5,10 @@ package image import ( "archive/tar" + "compress/gzip" "fmt" "io" + "io/ioutil" "os" "path/filepath" "runtime" @@ -180,3 +182,130 @@ func (i *TarImage) isExcluded(relPath string) bool { } return false } + +func CreateOciTarFileAndDeleteFolder(source, target string) error { + + tarFile, err := os.Create(target) + if err != nil { + return err + } + defer tarFile.Close() + + gzipWriter := gzip.NewWriter(tarFile) + defer gzipWriter.Close() + + tarWriter := tar.NewWriter(gzipWriter) + defer tarWriter.Close() + + err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + + header.Name, err = filepath.Rel(source, path) + if err != nil { + return err + } + + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + + if !info.IsDir() { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(tarWriter, file) + if err != nil { + return err + } + } + + return nil + }) + if err != nil { + return err + } + + err = os.RemoveAll(source) + if err != nil { + return err + } + + return nil +} + +func ExtractOciTarGz(tarGzFilePath string) (string, error) { + // Create a temporary directory + tempDir, err := ioutil.TempDir("", "imgpkg-oci-extract-") + if err != nil { + return "", err + } + //defer os.RemoveAll(tempDir) // Clean up the temporary directory when done + + // Open the tar.gz file + tarGzFile, err := os.Open(tarGzFilePath) + if err != nil { + return "", err + } + defer tarGzFile.Close() + + // Create a gzip reader + gzipReader, err := gzip.NewReader(tarGzFile) + if err != nil { + return "", err + } + defer gzipReader.Close() + + // Create a tar reader + tarReader := tar.NewReader(gzipReader) + + // Extract files to the temporary directory + for { + header, err := tarReader.Next() + + if err == io.EOF { + break // End of archive + } + + if err != nil { + return "", err + } + + // Construct the full path to the file in the temporary directory + targetPath := filepath.Join(tempDir, header.Name) + + // Check if the file is a directory + if header.FileInfo().IsDir() { + // Create the directory + err := os.MkdirAll(targetPath, os.ModePerm) + if err != nil { + return "", err + } + continue + } + + // Create the file + file, err := os.Create(targetPath) + if err != nil { + return "", err + } + defer file.Close() + + // Copy the file contents + _, err = io.Copy(file, tarReader) + if err != nil { + return "", err + } + } + + return tempDir, nil +} diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index 893abc1ed..1f0594274 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -130,13 +130,8 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr if tarisoci { imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci(importRepo.Name()) - // ---- FOR TESTING PORPOSES ---- - //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") } else { imgOrIndexes, err = imagetar.NewTarReader(path).Read() - // fmt.Println(imgOrIndexes[0].Ref()) - // ---- FOR TESTING PORPOSES ---- - //crane.SaveOCI(*imgOrIndexes[0].Image, "/Users/ashishkumarsingh/Desktop/stuff/ashpect/imgpkg/cmd/imgpkg/a0") } processedImages, err := i.imageSet.Import(imgOrIndexes, importRepo, registry) diff --git a/pkg/imgpkg/plainimage/contents.go b/pkg/imgpkg/plainimage/contents.go index c65197fc3..882c4bf69 100644 --- a/pkg/imgpkg/plainimage/contents.go +++ b/pkg/imgpkg/plainimage/contents.go @@ -62,6 +62,10 @@ func (i Contents) Push(uploadRef regname.Tag, labels map[string]string, writer I if err = p.AppendImage(img); err != nil { return "", err } + err = ctlimg.CreateOciTarFileAndDeleteFolder(i.ociTarPath, i.ociTarPath+".tar.gz") + if err != nil { + return "", err + } } defer img.Remove() From 64bf701bd66699b3f39a653dc22efaec25ee75f2 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 29 Nov 2023 07:46:54 +0530 Subject: [PATCH 11/20] updates Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 7 +++++- pkg/imgpkg/image/tar_image.go | 40 ++++++++++--------------------- pkg/imgpkg/imagetar/tar_reader.go | 3 +-- 3 files changed, 20 insertions(+), 30 deletions(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index 740b4b59f..f322f3127 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -5,6 +5,7 @@ package cmd import ( "fmt" + "io/ioutil" "os" ctlbundle "carvel.dev/imgpkg/pkg/imgpkg/bundle" @@ -76,7 +77,11 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) } if c.OciFlags.IsOci() { - tempDir, err = image.ExtractOciTarGz(c.OciFlags.OcitoReg) + tempDir, err := ioutil.TempDir("", "imgpkg-oci-extract-") + if err != nil { + return nil, err + } + err = image.ExtractOciTarGz(c.OciFlags.OcitoReg, tempDir) if err != nil { return nil, err } diff --git a/pkg/imgpkg/image/tar_image.go b/pkg/imgpkg/image/tar_image.go index 960dc0f3a..17a332d27 100644 --- a/pkg/imgpkg/image/tar_image.go +++ b/pkg/imgpkg/image/tar_image.go @@ -8,7 +8,6 @@ import ( "compress/gzip" "fmt" "io" - "io/ioutil" "os" "path/filepath" "runtime" @@ -243,69 +242,56 @@ func CreateOciTarFileAndDeleteFolder(source, target string) error { return nil } -func ExtractOciTarGz(tarGzFilePath string) (string, error) { - // Create a temporary directory - tempDir, err := ioutil.TempDir("", "imgpkg-oci-extract-") - if err != nil { - return "", err +func ExtractOciTarGz(inputDir, extractDir string) error { + + if !strings.HasSuffix(inputDir, ".tar.gz") { + return fmt.Errorf("inputDir '%s' is not a tar.gz file", inputDir) } - //defer os.RemoveAll(tempDir) // Clean up the temporary directory when done - // Open the tar.gz file - tarGzFile, err := os.Open(tarGzFilePath) + tarGzFile, err := os.Open(inputDir) if err != nil { - return "", err + return err } defer tarGzFile.Close() - // Create a gzip reader gzipReader, err := gzip.NewReader(tarGzFile) if err != nil { - return "", err + return err } defer gzipReader.Close() // Create a tar reader tarReader := tar.NewReader(gzipReader) - - // Extract files to the temporary directory for { header, err := tarReader.Next() if err == io.EOF { break // End of archive } - if err != nil { - return "", err + return err } + targetPath := filepath.Join(extractDir, header.Name) - // Construct the full path to the file in the temporary directory - targetPath := filepath.Join(tempDir, header.Name) - - // Check if the file is a directory if header.FileInfo().IsDir() { - // Create the directory err := os.MkdirAll(targetPath, os.ModePerm) if err != nil { - return "", err + return err } continue } - // Create the file file, err := os.Create(targetPath) if err != nil { - return "", err + return err } defer file.Close() - // Copy the file contents _, err = io.Copy(file, tarReader) if err != nil { - return "", err + return err } } - return tempDir, nil + return nil } diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index 9bd376ddd..89867559f 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -160,8 +160,7 @@ func (r TarReader) ReadOci(reponame string) ([]imagedesc.ImageOrIndex, error) { } if !stat.IsDir() { - //give error "not a directory" - return nil, err + return nil, fmt.Errorf("path %s is not a directory", r.path) } //TODO : FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. From a8e23c04022f931bdcee3f8a1bf4a6f4b3a0cc00 Mon Sep 17 00:00:00 2001 From: ashpect Date: Fri, 8 Dec 2023 09:12:03 +0530 Subject: [PATCH 12/20] Added support for imageindex and minor changes Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 47 ++++++++------- pkg/imgpkg/imagedesc/image_intermediate.go | 67 +++++++++++++++++++++ pkg/imgpkg/imageset/tar_image_set.go | 2 +- pkg/imgpkg/imagetar/tar_reader.go | 69 +++++++++++++++------- 4 files changed, 142 insertions(+), 43 deletions(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index f322f3127..f20a501ab 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -95,33 +95,36 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) return nil, err } - var parentBundle *ctlbundle.Bundle - foundRootBundle := false - for _, processedImage := range processedImages.All() { - if processedImage.ImageIndex != nil { - continue - } + // This is added to not read the lockfile and change the ref for oci-flag. Will be removed once we add an inflate option to copy the refs. + if !c.OciFlags.IsOci() { + var parentBundle *ctlbundle.Bundle + foundRootBundle := false + for _, processedImage := range processedImages.All() { + if processedImage.ImageIndex != nil { + continue + } - if _, ok := processedImage.Labels[rootBundleLabelKey]; ok { - if foundRootBundle { - panic("Internal inconsistency: expected only 1 root bundle") + if _, ok := processedImage.Labels[rootBundleLabelKey]; ok { + if foundRootBundle { + panic("Internal inconsistency: expected only 1 root bundle") + } + foundRootBundle = true + pImage := plainimage.NewFetchedPlainImageWithTag(processedImage.DigestRef, processedImage.Tag, processedImage.Image) + lockReader := ctlbundle.NewImagesLockReader() + parentBundle = ctlbundle.NewBundle(pImage, c.registry, lockReader, ctlbundle.NewFetcherFromProcessedImages(processedImages.All(), c.registry, lockReader)) } - foundRootBundle = true - pImage := plainimage.NewFetchedPlainImageWithTag(processedImage.DigestRef, processedImage.Tag, processedImage.Image) - lockReader := ctlbundle.NewImagesLockReader() - parentBundle = ctlbundle.NewBundle(pImage, c.registry, lockReader, ctlbundle.NewFetcherFromProcessedImages(processedImages.All(), c.registry, lockReader)) } - } - if foundRootBundle { - bundles, _, err := parentBundle.AllImagesLockRefs(c.Concurrency, c.logger) - if err != nil { - return nil, err - } + if foundRootBundle { + bundles, _, err := parentBundle.AllImagesLockRefs(c.Concurrency, c.logger) + if err != nil { + return nil, err + } - for _, bundle := range bundles { - if err := bundle.NoteCopy(processedImages, c.registry, c.logger); err != nil { - return nil, fmt.Errorf("Creating copy information for bundle %s: %s", bundle.DigestRef(), err) + for _, bundle := range bundles { + if err := bundle.NoteCopy(processedImages, c.registry, c.logger); err != nil { + return nil, fmt.Errorf("Creating copy information for bundle %s: %s", bundle.DigestRef(), err) + } } } } diff --git a/pkg/imgpkg/imagedesc/image_intermediate.go b/pkg/imgpkg/imagedesc/image_intermediate.go index be3732aaf..caf657879 100644 --- a/pkg/imgpkg/imagedesc/image_intermediate.go +++ b/pkg/imgpkg/imagedesc/image_intermediate.go @@ -14,6 +14,72 @@ type ImageIndexIntermediate struct { tag string } +type ImageIntermediate struct { + Image regv1.Image + ref string + tag string +} + +func (mi ImageIntermediate) Ref() string { + return mi.ref +} + +func (mi *ImageIntermediate) SetRef(ref string) { + mi.ref = ref +} + +func (mi ImageIntermediate) Tag() string { + return mi.tag +} + +func (mi *ImageIntermediate) SetTag(tag string) { + mi.tag = tag +} + +func (mi ImageIntermediate) Layers() ([]regv1.Layer, error) { + return mi.Image.Layers() +} + +func (mi ImageIntermediate) MediaType() (types.MediaType, error) { + return mi.Image.MediaType() +} + +func (mi ImageIntermediate) Size() (int64, error) { + return mi.Image.Size() +} + +func (mi ImageIntermediate) ConfigName() (regv1.Hash, error) { + return mi.Image.ConfigName() +} + +func (mi ImageIntermediate) ConfigFile() (*regv1.ConfigFile, error) { + return mi.Image.ConfigFile() +} + +func (mi ImageIntermediate) RawConfigFile() ([]byte, error) { + return mi.Image.RawConfigFile() +} + +func (mi ImageIntermediate) Digest() (regv1.Hash, error) { + return mi.Image.Digest() +} + +func (mi ImageIntermediate) Manifest() (*regv1.Manifest, error) { + return mi.Image.Manifest() +} + +func (mi ImageIntermediate) RawManifest() ([]byte, error) { + return mi.Image.RawManifest() +} + +func (mi ImageIntermediate) LayerByDigest(h regv1.Hash) (regv1.Layer, error) { + return mi.Image.LayerByDigest(h) +} + +func (mi ImageIntermediate) LayerByDiffID(h regv1.Hash) (regv1.Layer, error) { + return mi.Image.LayerByDiffID(h) +} + func (mi ImageIndexIntermediate) Ref() string { return mi.ref } @@ -59,3 +125,4 @@ func (mi ImageIndexIntermediate) ImageIndex(h regv1.Hash) (regv1.ImageIndex, err } var _ regv1.ImageIndex = ImageIndexIntermediate{} +var _ regv1.Image = ImageIntermediate{} diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index 1f0594274..d814993ae 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -129,7 +129,7 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr var err error if tarisoci { - imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci(importRepo.Name()) + imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci(importRepo) } else { imgOrIndexes, err = imagetar.NewTarReader(path).Read() } diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index 89867559f..5fac5426d 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "path/filepath" "carvel.dev/imgpkg/pkg/imgpkg/imagedesc" "carvel.dev/imgpkg/pkg/imgpkg/imageutils/verify" @@ -151,9 +152,8 @@ func (r TarReader) getIdsFromManifest(file tarFile) (*imagedesc.ImageRefDescript return ids, nil } -func (r TarReader) ReadOci(reponame string) ([]imagedesc.ImageOrIndex, error) { +func (r TarReader) ReadOci(importRepo name.Repository) ([]imagedesc.ImageOrIndex, error) { - //Check if the path is a OCI layout directory stat, err := os.Stat(r.path) if err != nil { return nil, err @@ -163,39 +163,68 @@ func (r TarReader) ReadOci(reponame string) ([]imagedesc.ImageOrIndex, error) { return nil, fmt.Errorf("path %s is not a directory", r.path) } - //TODO : FromPath checks for index.json but does not check for oci-layout, so add a check for oci-layout here. + _, err = os.Stat(filepath.Join(r.path, "oci-layout")) + if err != nil { + return nil, err + } l, err := layout.FromPath(r.path) if err != nil { return nil, err } - ImageIndex, err := l.ImageIndex() + ii, err := l.ImageIndex() + m, err := ii.IndexManifest() + desc := m.Manifests[0] - ImageIndexIntermediate := imagedesc.ImageIndexIntermediate{ - Index: ImageIndex, - } + var ImageIntermediate imagedesc.ImageIntermediate + var ImageIndexIntermediate imagedesc.ImageIndexIntermediate + var ref string + + if desc.MediaType.IsImage() { + img, err := ii.Image(desc.Digest) + if err != nil { + return nil, err + } + + ImageIntermediate = imagedesc.ImageIntermediate{ + Image: img, + } - // Update ref - digest, err := ImageIndex.Digest() - digestStr := digest.String() - ref := reponame + "@" + digestStr - ImageIndexIntermediate.SetRef(ref) + digest, err := img.Digest() + digestStr := digest.String() + ref = importRepo.Name() + "@" + digestStr - // Create and populate imageOrIndex - var i imagedesc.ImageIndexWithRef = ImageIndexIntermediate + ImageIntermediate.SetRef(ref) + + } else if desc.MediaType.IsIndex() { + idx, err := ii.ImageIndex(desc.Digest) + if err != nil { + return nil, err + } + ImageIndexIntermediate = imagedesc.ImageIndexIntermediate{ + Index: idx, + } + + digest, err := idx.Digest() + digestStr := digest.String() + ref = importRepo.Name() + "@" + digestStr + ImageIndexIntermediate.SetRef(ref) + + } else { + return nil, fmt.Errorf("Unexpected media type: %s", desc.MediaType) + } + var b imagedesc.ImageWithRef = ImageIntermediate imageOrIndex := imagedesc.ImageOrIndex{ - Image: nil, - Index: &i, + Image: &b, + Index: nil, Labels: map[string]string{ - "label1": "value1", - "label2": "value2", + "dev.carvel.imgpkg.copy.root-bundle": "", }, - OrigRef: "original-reference", + OrigRef: "", } - //Add imageOrIndex to the slice of imageOrIndex var imageOrIndexSlice []imagedesc.ImageOrIndex imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) From c4ba9734c615c1ea4aabee11862832c4f381aa41 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 22 May 2024 12:38:12 +0530 Subject: [PATCH 13/20] changes for failing tests Signed-off-by: ashpect --- pkg/imgpkg/imagedesc/image_intermediate.go | 1 + pkg/imgpkg/imagedesc/types.go | 1 + pkg/imgpkg/registry/registry.go | 2 -- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/imgpkg/imagedesc/image_intermediate.go b/pkg/imgpkg/imagedesc/image_intermediate.go index caf657879..59bc97dd7 100644 --- a/pkg/imgpkg/imagedesc/image_intermediate.go +++ b/pkg/imgpkg/imagedesc/image_intermediate.go @@ -8,6 +8,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) +// intermediate struct for Image and ImageIndex to convert to imagwithref type for oci image type ImageIndexIntermediate struct { Index regv1.ImageIndex ref string diff --git a/pkg/imgpkg/imagedesc/types.go b/pkg/imgpkg/imagedesc/types.go index 5474ccd03..ddff09f41 100644 --- a/pkg/imgpkg/imagedesc/types.go +++ b/pkg/imgpkg/imagedesc/types.go @@ -18,6 +18,7 @@ type ImageOrIndex struct { OrigRef string } + type ImageWithRef interface { regv1.Image Ref() string diff --git a/pkg/imgpkg/registry/registry.go b/pkg/imgpkg/registry/registry.go index 2a1e320cd..9a10033b4 100644 --- a/pkg/imgpkg/registry/registry.go +++ b/pkg/imgpkg/registry/registry.go @@ -410,8 +410,6 @@ func (r *SimpleRegistry) MultiWrite(imageOrIndexesToUpload map[regname.Reference if updatesCh != nil { rOpts = append(rOpts, regremote.WithProgress(updatesCh)) } - - //THIS HERE FINAL PUSHES return regremote.MultiWrite(overriddenImageOrIndexesToUploadRef, rOpts...) } From 3d721b0caf187fd6177eb5f953ff24443193c4f0 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 22 May 2024 13:19:56 +0530 Subject: [PATCH 14/20] restructuring and adding comments to exported functions Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 8 +- pkg/imgpkg/cmd/oci_flags.go | 5 +- pkg/imgpkg/image/tar_image.go | 2 + pkg/imgpkg/imagedesc/image_intermediate.go | 32 +++- pkg/imgpkg/imageset/image_set.go | 1 - pkg/imgpkg/imageset/tar_image_set.go | 6 + pkg/imgpkg/imagetar/tar_reader.go | 171 +++++++++++---------- 7 files changed, 140 insertions(+), 85 deletions(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index f20a501ab..507cbfb78 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -83,12 +83,18 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) } err = image.ExtractOciTarGz(c.OciFlags.OcitoReg, tempDir) if err != nil { - return nil, err + return nil, fmt.Errorf("Extracting OCI tar: %s", err) } processedImages, err = c.tarImageSet.Import(tempDir, importRepo, c.registry, true) + if err != nil { + return nil, fmt.Errorf("Importing OCI tar: %s", err) + } } else { processedImages, err = c.tarImageSet.Import(c.TarFlags.TarSrc, importRepo, c.registry, false) + if err != nil { + return nil, fmt.Errorf("Importing tar: %s", err) + } } if err != nil { diff --git a/pkg/imgpkg/cmd/oci_flags.go b/pkg/imgpkg/cmd/oci_flags.go index 3c8baa0d5..d6dbc09ad 100644 --- a/pkg/imgpkg/cmd/oci_flags.go +++ b/pkg/imgpkg/cmd/oci_flags.go @@ -1,4 +1,4 @@ -// Copyright 2020 VMware, Inc. +// Copyright 2023 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 package cmd @@ -7,14 +7,17 @@ import ( "github.com/spf13/cobra" ) +// OciFlags is a struct that holds the flags for the OCI tar file. type OciFlags struct { OcitoReg string OciTar string } +// Set sets the flags for the OCI tar file. func (o *OciFlags) Set(cmd *cobra.Command) { cmd.Flags().StringVar(&o.OciTar, "to-oci-tar", "", "Set OciTarPath to be saved to disk (example: /path/file.tar)") cmd.Flags().StringVar(&o.OcitoReg, "oci-tar", "", "Give path to OCI tar file (example: /path/file.tar)") } +// IsOci returns true if the OCI tar file is set. func (o OciFlags) IsOci() bool { return o.OcitoReg != "" } diff --git a/pkg/imgpkg/image/tar_image.go b/pkg/imgpkg/image/tar_image.go index 17a332d27..10e9cd549 100644 --- a/pkg/imgpkg/image/tar_image.go +++ b/pkg/imgpkg/image/tar_image.go @@ -182,6 +182,7 @@ func (i *TarImage) isExcluded(relPath string) bool { return false } +// CreateOciTarFileAndDeleteFolder creates a oci tar file from the source folder and deletes the source folder. Used while pushing the oci-tar func CreateOciTarFileAndDeleteFolder(source, target string) error { tarFile, err := os.Create(target) @@ -242,6 +243,7 @@ func CreateOciTarFileAndDeleteFolder(source, target string) error { return nil } +// ExtractOciTarGz extracts the oci tar file to the extractDir func ExtractOciTarGz(inputDir, extractDir string) error { if !strings.HasSuffix(inputDir, ".tar.gz") { diff --git a/pkg/imgpkg/imagedesc/image_intermediate.go b/pkg/imgpkg/imagedesc/image_intermediate.go index 59bc97dd7..99445ddde 100644 --- a/pkg/imgpkg/imagedesc/image_intermediate.go +++ b/pkg/imgpkg/imagedesc/image_intermediate.go @@ -1,4 +1,4 @@ -// Copyright 2020 VMware, Inc. +// Copyright 2023 VMware, Inc. // SPDX-License-Identifier: Apache-2.0 package imagedesc @@ -8,119 +8,145 @@ import ( "github.com/google/go-containerregistry/pkg/v1/types" ) -// intermediate struct for Image and ImageIndex to convert to imagwithref type for oci image +// ImageIndexIntermediate struct for ImageIndex to help in convertion to ImagIndexWithRef struct for oci image workflow.. type ImageIndexIntermediate struct { Index regv1.ImageIndex ref string tag string } +// ImageIntermediate struct for Image to help in convertion to ImageWithRef struct for oci image workflow. type ImageIntermediate struct { Image regv1.Image ref string tag string } +// Ref returns the reference in the imageintermediate struct. func (mi ImageIntermediate) Ref() string { return mi.ref } +// SetRef sets the reference in the imageintermediate struct. func (mi *ImageIntermediate) SetRef(ref string) { mi.ref = ref } +// Tag returns the tag value set in the imageintermediate struct. func (mi ImageIntermediate) Tag() string { return mi.tag } +// SetTag sets the tag value in the imageintermediate struct. func (mi *ImageIntermediate) SetTag(tag string) { mi.tag = tag } +// Layers returns the ordered collection of filesystem layers that comprise this image. func (mi ImageIntermediate) Layers() ([]regv1.Layer, error) { return mi.Image.Layers() } - +// MediaType of the image's manifest. func (mi ImageIntermediate) MediaType() (types.MediaType, error) { return mi.Image.MediaType() } +// Size returns the size of the image. func (mi ImageIntermediate) Size() (int64, error) { return mi.Image.Size() } +// ConfigName returns the name of the image's configuration. func (mi ImageIntermediate) ConfigName() (regv1.Hash, error) { return mi.Image.ConfigName() } +// ConfigFile returns the image's config file. func (mi ImageIntermediate) ConfigFile() (*regv1.ConfigFile, error) { return mi.Image.ConfigFile() } +// RawConfigFile returns the serialized bytes of ConfigFile(). func (mi ImageIntermediate) RawConfigFile() ([]byte, error) { return mi.Image.RawConfigFile() } +// Digest returns the sha256 of this image's manifest. func (mi ImageIntermediate) Digest() (regv1.Hash, error) { return mi.Image.Digest() } +// Manifest returns this image's Manifest object. func (mi ImageIntermediate) Manifest() (*regv1.Manifest, error) { return mi.Image.Manifest() } +// RawManifest returns the serialized bytes of Manifest() func (mi ImageIntermediate) RawManifest() ([]byte, error) { return mi.Image.RawManifest() } +// LayerByDigest returns a Layer for interacting with a particular layer of the image, looking it up by "digest" (the compressed hash). func (mi ImageIntermediate) LayerByDigest(h regv1.Hash) (regv1.Layer, error) { return mi.Image.LayerByDigest(h) } +// LayerByDiffID is an analog to LayerByDigest, looking up by "diff id" (the uncompressed hash). func (mi ImageIntermediate) LayerByDiffID(h regv1.Hash) (regv1.Layer, error) { return mi.Image.LayerByDiffID(h) } +// Ref returns the reference in the imageindexintermediate struct. func (mi ImageIndexIntermediate) Ref() string { return mi.ref } +// SetRef sets the reference in the imageindexintermediate struct. func (mi *ImageIndexIntermediate) SetRef(ref string) { mi.ref = ref } +// Tag returns the tag value set in the imageindexintermediate struct. func (mi ImageIndexIntermediate) Tag() string { return mi.tag } +// SetTag sets the tag value in the imageindexintermediate struct. func (mi *ImageIndexIntermediate) SetTag(tag string) { mi.tag = tag } +// MediaType of the imageindex's manifest. func (mi ImageIndexIntermediate) MediaType() (types.MediaType, error) { return mi.Index.MediaType() } +// Digest returns the sha256 of this imageindex's manifest. func (mi ImageIndexIntermediate) Digest() (regv1.Hash, error) { return mi.Index.Digest() } +// Size returns the size of the imageindex. func (mi ImageIndexIntermediate) Size() (int64, error) { return mi.Index.Size() } +// IndexManifest returns this image index's manifest object. func (mi ImageIndexIntermediate) IndexManifest() (*regv1.IndexManifest, error) { return mi.Index.IndexManifest() } +// RawManifest returns the serialized bytes of IndexManifest(). func (mi ImageIndexIntermediate) RawManifest() ([]byte, error) { return mi.Index.RawManifest() } +// Image returns a v1.Image that this ImageIndex references. func (mi ImageIndexIntermediate) Image(h regv1.Hash) (regv1.Image, error) { return mi.Index.Image(h) } +// ImageIndex returns a v1.ImageIndex that this ImageIndex references. func (mi ImageIndexIntermediate) ImageIndex(h regv1.Hash) (regv1.ImageIndex, error) { return mi.Index.ImageIndex(h) } diff --git a/pkg/imgpkg/imageset/image_set.go b/pkg/imgpkg/imageset/image_set.go index 09ac5c5ae..c3faa3d75 100644 --- a/pkg/imgpkg/imageset/image_set.go +++ b/pkg/imgpkg/imageset/image_set.go @@ -71,7 +71,6 @@ func (i ImageSet) Export(foundImages *UnprocessedImageRefs, return ids, nil } -// this pushes probably to a registry func (i *ImageSet) Import(imgOrIndexes []imagedesc.ImageOrIndex, importRepo regname.Repository, registry registry.ImagesReaderWriter) (*ProcessedImages, error) { diff --git a/pkg/imgpkg/imageset/tar_image_set.go b/pkg/imgpkg/imageset/tar_image_set.go index d814993ae..8bf6d813c 100644 --- a/pkg/imgpkg/imageset/tar_image_set.go +++ b/pkg/imgpkg/imageset/tar_image_set.go @@ -130,8 +130,14 @@ func (i *TarImageSet) Import(path string, importRepo regname.Repository, registr if tarisoci { imgOrIndexes, err = imagetar.NewTarReader(path).ReadOci(importRepo) + if err != nil { + return nil, err + } } else { imgOrIndexes, err = imagetar.NewTarReader(path).Read() + if err != nil { + return nil, err + } } processedImages, err := i.imageSet.Import(imgOrIndexes, importRepo, registry) diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index 5fac5426d..d34d1d55f 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -35,6 +35,98 @@ func (r TarReader) Read() ([]imagedesc.ImageOrIndex, error) { return imagedesc.NewDescribedReader(ids, file).Read(), nil } +// ReadOci reads the OCI layout from the tar file and returns the image or index. Equivalent to Read() but for OCI layout +func (r TarReader) ReadOci(importRepo name.Repository) ([]imagedesc.ImageOrIndex, error) { + + stat, err := os.Stat(r.path) + if err != nil { + return nil, err + } + + if !stat.IsDir() { + return nil, fmt.Errorf("path %s is not a directory", r.path) + } + + _, err = os.Stat(filepath.Join(r.path, "oci-layout")) + if err != nil { + return nil, err + } + + l, err := layout.FromPath(r.path) + if err != nil { + return nil, err + } + + ii, err := l.ImageIndex() + if err != nil { + return nil, fmt.Errorf("Unable to read image index: %s", err) + } + m, err := ii.IndexManifest() + if err != nil { + return nil, fmt.Errorf("Unable to read index manifest: %s", err) + } + desc := m.Manifests[0] + + var ImageIntermediate imagedesc.ImageIntermediate + var ImageIndexIntermediate imagedesc.ImageIndexIntermediate + var ref string + + if desc.MediaType.IsImage() { + img, err := ii.Image(desc.Digest) + if err != nil { + return nil, err + } + + ImageIntermediate = imagedesc.ImageIntermediate{ + Image: img, + } + + digest, err := img.Digest() + if err != nil { + return nil, fmt.Errorf("Unable to get digest from image: %s", err) + } + digestStr := digest.String() + ref = importRepo.Name() + "@" + digestStr + + ImageIntermediate.SetRef(ref) + + } else if desc.MediaType.IsIndex() { + idx, err := ii.ImageIndex(desc.Digest) + if err != nil { + return nil, err + } + ImageIndexIntermediate = imagedesc.ImageIndexIntermediate{ + Index: idx, + } + + digest, err := idx.Digest() + if err != nil { + return nil, fmt.Errorf("Unable to get digest from index: %s", err) + } + digestStr := digest.String() + ref = importRepo.Name() + "@" + digestStr + ImageIndexIntermediate.SetRef(ref) + + } else { + return nil, fmt.Errorf("Unexpected media type: %s", desc.MediaType) + } + + var b imagedesc.ImageWithRef = ImageIntermediate + imageOrIndex := imagedesc.ImageOrIndex{ + Image: &b, + Index: nil, + Labels: map[string]string{ + "dev.carvel.imgpkg.copy.root-bundle": "", + }, + OrigRef: "", + } + + var imageOrIndexSlice []imagedesc.ImageOrIndex + imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) + + return imageOrIndexSlice, nil +} + // PresentLayers retrieves all the layers that are present in a tar file func (r TarReader) PresentLayers() ([]v1.Layer, error) { var result []v1.Layer @@ -151,82 +243,3 @@ func (r TarReader) getIdsFromManifest(file tarFile) (*imagedesc.ImageRefDescript } return ids, nil } - -func (r TarReader) ReadOci(importRepo name.Repository) ([]imagedesc.ImageOrIndex, error) { - - stat, err := os.Stat(r.path) - if err != nil { - return nil, err - } - - if !stat.IsDir() { - return nil, fmt.Errorf("path %s is not a directory", r.path) - } - - _, err = os.Stat(filepath.Join(r.path, "oci-layout")) - if err != nil { - return nil, err - } - - l, err := layout.FromPath(r.path) - if err != nil { - return nil, err - } - - ii, err := l.ImageIndex() - m, err := ii.IndexManifest() - desc := m.Manifests[0] - - var ImageIntermediate imagedesc.ImageIntermediate - var ImageIndexIntermediate imagedesc.ImageIndexIntermediate - var ref string - - if desc.MediaType.IsImage() { - img, err := ii.Image(desc.Digest) - if err != nil { - return nil, err - } - - ImageIntermediate = imagedesc.ImageIntermediate{ - Image: img, - } - - digest, err := img.Digest() - digestStr := digest.String() - ref = importRepo.Name() + "@" + digestStr - - ImageIntermediate.SetRef(ref) - - } else if desc.MediaType.IsIndex() { - idx, err := ii.ImageIndex(desc.Digest) - if err != nil { - return nil, err - } - ImageIndexIntermediate = imagedesc.ImageIndexIntermediate{ - Index: idx, - } - - digest, err := idx.Digest() - digestStr := digest.String() - ref = importRepo.Name() + "@" + digestStr - ImageIndexIntermediate.SetRef(ref) - - } else { - return nil, fmt.Errorf("Unexpected media type: %s", desc.MediaType) - } - - var b imagedesc.ImageWithRef = ImageIntermediate - imageOrIndex := imagedesc.ImageOrIndex{ - Image: &b, - Index: nil, - Labels: map[string]string{ - "dev.carvel.imgpkg.copy.root-bundle": "", - }, - OrigRef: "", - } - - var imageOrIndexSlice []imagedesc.ImageOrIndex - imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) - - return imageOrIndexSlice, nil -} From 9e0d43059eb22a0a826029f327b869a52427fe79 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 22 May 2024 13:24:16 +0530 Subject: [PATCH 15/20] relace ioutil with os Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index 507cbfb78..67ed15cb4 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -5,7 +5,6 @@ package cmd import ( "fmt" - "io/ioutil" "os" ctlbundle "carvel.dev/imgpkg/pkg/imgpkg/bundle" @@ -77,7 +76,7 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) } if c.OciFlags.IsOci() { - tempDir, err := ioutil.TempDir("", "imgpkg-oci-extract-") + tempDir, err := os.MkdirTemp("", "imgpkg-oci-extract-") if err != nil { return nil, err } From 77d89300420969ee225d60a63ce232500e4d1952 Mon Sep 17 00:00:00 2001 From: ashpect Date: Wed, 22 May 2024 15:15:13 +0530 Subject: [PATCH 16/20] update headers Signed-off-by: ashpect --- pkg/imgpkg/cmd/oci_flags.go | 2 +- pkg/imgpkg/imagedesc/image_intermediate.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/imgpkg/cmd/oci_flags.go b/pkg/imgpkg/cmd/oci_flags.go index d6dbc09ad..599014a84 100644 --- a/pkg/imgpkg/cmd/oci_flags.go +++ b/pkg/imgpkg/cmd/oci_flags.go @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023 The Carvel Authors. // SPDX-License-Identifier: Apache-2.0 package cmd diff --git a/pkg/imgpkg/imagedesc/image_intermediate.go b/pkg/imgpkg/imagedesc/image_intermediate.go index 99445ddde..407606737 100644 --- a/pkg/imgpkg/imagedesc/image_intermediate.go +++ b/pkg/imgpkg/imagedesc/image_intermediate.go @@ -1,4 +1,4 @@ -// Copyright 2023 VMware, Inc. +// Copyright 2023 The Carvel Authors. // SPDX-License-Identifier: Apache-2.0 package imagedesc @@ -46,6 +46,7 @@ func (mi *ImageIntermediate) SetTag(tag string) { func (mi ImageIntermediate) Layers() ([]regv1.Layer, error) { return mi.Image.Layers() } + // MediaType of the image's manifest. func (mi ImageIntermediate) MediaType() (types.MediaType, error) { return mi.Image.MediaType() From 187106d9ac897ad97ebfa95273ff7e3a5ddb3f30 Mon Sep 17 00:00:00 2001 From: ashpect Date: Thu, 23 May 2024 13:16:28 +0530 Subject: [PATCH 17/20] fixing error bug importing tar Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index 67ed15cb4..661fe9f57 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -92,7 +92,7 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) } else { processedImages, err = c.tarImageSet.Import(c.TarFlags.TarSrc, importRepo, c.registry, false) if err != nil { - return nil, fmt.Errorf("Importing tar: %s", err) + return nil, err } } From 387e4a6e0544cfd63f7d66fedfda434416ad3eb5 Mon Sep 17 00:00:00 2001 From: ashpect Date: Thu, 23 May 2024 22:35:55 +0530 Subject: [PATCH 18/20] add support for index Signed-off-by: ashpect --- pkg/imgpkg/imagetar/tar_reader.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/pkg/imgpkg/imagetar/tar_reader.go b/pkg/imgpkg/imagetar/tar_reader.go index d34d1d55f..b27ed4971 100644 --- a/pkg/imgpkg/imagetar/tar_reader.go +++ b/pkg/imgpkg/imagetar/tar_reader.go @@ -70,6 +70,14 @@ func (r TarReader) ReadOci(importRepo name.Repository) ([]imagedesc.ImageOrIndex var ImageIntermediate imagedesc.ImageIntermediate var ImageIndexIntermediate imagedesc.ImageIndexIntermediate var ref string + imageOrIndex := imagedesc.ImageOrIndex{ + Image: nil, + Index: nil, + Labels: map[string]string{ + "dev.carvel.imgpkg.copy.root-bundle": "", + }, + OrigRef: "", + } if desc.MediaType.IsImage() { img, err := ii.Image(desc.Digest) @@ -90,6 +98,9 @@ func (r TarReader) ReadOci(importRepo name.Repository) ([]imagedesc.ImageOrIndex ImageIntermediate.SetRef(ref) + var b imagedesc.ImageWithRef = ImageIntermediate + imageOrIndex.Image = &b + } else if desc.MediaType.IsIndex() { idx, err := ii.ImageIndex(desc.Digest) if err != nil { @@ -107,20 +118,13 @@ func (r TarReader) ReadOci(importRepo name.Repository) ([]imagedesc.ImageOrIndex ref = importRepo.Name() + "@" + digestStr ImageIndexIntermediate.SetRef(ref) + var b imagedesc.ImageIndexWithRef = ImageIndexIntermediate + imageOrIndex.Index = &b + } else { return nil, fmt.Errorf("Unexpected media type: %s", desc.MediaType) } - var b imagedesc.ImageWithRef = ImageIntermediate - imageOrIndex := imagedesc.ImageOrIndex{ - Image: &b, - Index: nil, - Labels: map[string]string{ - "dev.carvel.imgpkg.copy.root-bundle": "", - }, - OrigRef: "", - } - var imageOrIndexSlice []imagedesc.ImageOrIndex imageOrIndexSlice = append(imageOrIndexSlice, imageOrIndex) From 733a2a0af92d6e1e1cf266b76fe89c474bb87d7d Mon Sep 17 00:00:00 2001 From: ashpect Date: Thu, 23 May 2024 22:36:16 +0530 Subject: [PATCH 19/20] update vendor Signed-off-by: ashpect --- .../go-containerregistry/pkg/v1/layout/gc.go | 137 ++++++++++++++++++ vendor/modules.txt | 1 + 2 files changed, 138 insertions(+) create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/gc.go diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/gc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/gc.go new file mode 100644 index 000000000..5fdb2c05d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/gc.go @@ -0,0 +1,137 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is an EXPERIMENTAL package, and may change in arbitrary ways without notice. +package layout + +import ( + "fmt" + "io/fs" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// GarbageCollect removes unreferenced blobs from the oci-layout +// +// This is an experimental api, and not subject to any stability guarantees +// We may abandon it at any time, without prior notice. +// Deprecated: Use it at your own risk! +func (l Path) GarbageCollect() ([]v1.Hash, error) { + idx, err := l.ImageIndex() + if err != nil { + return nil, err + } + blobsToKeep := map[string]bool{} + if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil { + return nil, err + } + blobsDir := l.path("blobs") + removedBlobs := []v1.Hash{} + + err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + return nil + } + + rel, err := filepath.Rel(blobsDir, path) + if err != nil { + return err + } + hashString := strings.Replace(rel, "/", ":", 1) + if present := blobsToKeep[hashString]; !present { + h, err := v1.NewHash(hashString) + if err != nil { + return err + } + removedBlobs = append(removedBlobs, h) + } + return nil + }) + + if err != nil { + return nil, err + } + + return removedBlobs, nil +} + +func (l Path) garbageCollectImageIndex(index v1.ImageIndex, blobsToKeep map[string]bool) error { + idxm, err := index.IndexManifest() + if err != nil { + return err + } + + h, err := index.Digest() + if err != nil { + return err + } + + blobsToKeep[h.String()] = true + + for _, descriptor := range idxm.Manifests { + if descriptor.MediaType.IsImage() { + img, err := index.Image(descriptor.Digest) + if err != nil { + return err + } + if err := l.garbageCollectImage(img, blobsToKeep); err != nil { + return err + } + } else if descriptor.MediaType.IsIndex() { + idx, err := index.ImageIndex(descriptor.Digest) + if err != nil { + return err + } + if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil { + return err + } + } else { + return fmt.Errorf("gc: unknown media type: %s", descriptor.MediaType) + } + } + return nil +} + +func (l Path) garbageCollectImage(image v1.Image, blobsToKeep map[string]bool) error { + h, err := image.Digest() + if err != nil { + return err + } + blobsToKeep[h.String()] = true + + h, err = image.ConfigName() + if err != nil { + return err + } + blobsToKeep[h.String()] = true + + ls, err := image.Layers() + if err != nil { + return err + } + for _, l := range ls { + h, err := l.Digest() + if err != nil { + return err + } + blobsToKeep[h.String()] = true + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index cfb803490..fe440594b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -196,6 +196,7 @@ github.com/google/go-containerregistry/pkg/v1 github.com/google/go-containerregistry/pkg/v1/empty github.com/google/go-containerregistry/pkg/v1/fake github.com/google/go-containerregistry/pkg/v1/google +github.com/google/go-containerregistry/pkg/v1/layout github.com/google/go-containerregistry/pkg/v1/match github.com/google/go-containerregistry/pkg/v1/mutate github.com/google/go-containerregistry/pkg/v1/partial From b1defd27d31012d0e4e4fc59f5e3b4681bcbf33d Mon Sep 17 00:00:00 2001 From: ashpect Date: Sat, 25 May 2024 16:37:04 +0530 Subject: [PATCH 20/20] resolving reviews Signed-off-by: ashpect --- pkg/imgpkg/cmd/copy_repo_src.go | 11 +++-------- pkg/imgpkg/cmd/oci_flags.go | 2 +- pkg/imgpkg/image/tar_image.go | 6 ++---- pkg/imgpkg/imagedesc/image_intermediate.go | 2 +- pkg/imgpkg/plainimage/contents.go | 2 +- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/pkg/imgpkg/cmd/copy_repo_src.go b/pkg/imgpkg/cmd/copy_repo_src.go index 661fe9f57..4ef95ae3e 100644 --- a/pkg/imgpkg/cmd/copy_repo_src.go +++ b/pkg/imgpkg/cmd/copy_repo_src.go @@ -63,7 +63,6 @@ func (c CopyRepoSrc) CopyToTar(dstPath string, resume bool) error { func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) { c.logger.Tracef("CopyToRepo(%s)\n", repo) - var tempDir string var processedImages *ctlimgset.ProcessedImages importRepo, err := regname.NewRepository(repo) if err != nil { @@ -74,12 +73,12 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) if c.TarFlags.IsDst() { return nil, fmt.Errorf("Cannot use tar source (--tar) with tar destination (--to-tar)") } - if c.OciFlags.IsOci() { tempDir, err := os.MkdirTemp("", "imgpkg-oci-extract-") if err != nil { return nil, err } + defer os.RemoveAll(tempDir) err = image.ExtractOciTarGz(c.OciFlags.OcitoReg, tempDir) if err != nil { return nil, fmt.Errorf("Extracting OCI tar: %s", err) @@ -100,7 +99,8 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) return nil, err } - // This is added to not read the lockfile and change the ref for oci-flag. Will be removed once we add an inflate option to copy the refs. + // Cuurently when copying images from an oci-tar to a repository, imgpkg will not try to access the origin repositories and will only copy the OCI Image to the registry, + // similar to the behavior we currently have on the `imgpkg push` command. Adding `inflate` flag, to access the origin repos will be an future improvement. if !c.OciFlags.IsOci() { var parentBundle *ctlbundle.Bundle foundRootBundle := false @@ -160,11 +160,6 @@ func (c CopyRepoSrc) CopyToRepo(repo string) (*ctlimgset.ProcessedImages, error) return nil, fmt.Errorf("Tagging images: %s", err) } - err = os.RemoveAll(tempDir) - if err != nil { - fmt.Println("Error cleaning up temporary directory:", err) - } - return processedImages, nil } diff --git a/pkg/imgpkg/cmd/oci_flags.go b/pkg/imgpkg/cmd/oci_flags.go index 599014a84..417e76c0a 100644 --- a/pkg/imgpkg/cmd/oci_flags.go +++ b/pkg/imgpkg/cmd/oci_flags.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Carvel Authors. +// Copyright 2024 The Carvel Authors. // SPDX-License-Identifier: Apache-2.0 package cmd diff --git a/pkg/imgpkg/image/tar_image.go b/pkg/imgpkg/image/tar_image.go index 10e9cd549..e850a9478 100644 --- a/pkg/imgpkg/image/tar_image.go +++ b/pkg/imgpkg/image/tar_image.go @@ -182,9 +182,8 @@ func (i *TarImage) isExcluded(relPath string) bool { return false } -// CreateOciTarFileAndDeleteFolder creates a oci tar file from the source folder and deletes the source folder. Used while pushing the oci-tar -func CreateOciTarFileAndDeleteFolder(source, target string) error { - +// CreateOciTarFromFiles creates a oci tar from the obtained files in the open folder. +func CreateOciTarFromFiles(source, target string) error { tarFile, err := os.Create(target) if err != nil { return err @@ -245,7 +244,6 @@ func CreateOciTarFileAndDeleteFolder(source, target string) error { // ExtractOciTarGz extracts the oci tar file to the extractDir func ExtractOciTarGz(inputDir, extractDir string) error { - if !strings.HasSuffix(inputDir, ".tar.gz") { return fmt.Errorf("inputDir '%s' is not a tar.gz file", inputDir) } diff --git a/pkg/imgpkg/imagedesc/image_intermediate.go b/pkg/imgpkg/imagedesc/image_intermediate.go index 407606737..c57d6d874 100644 --- a/pkg/imgpkg/imagedesc/image_intermediate.go +++ b/pkg/imgpkg/imagedesc/image_intermediate.go @@ -1,4 +1,4 @@ -// Copyright 2023 The Carvel Authors. +// Copyright 2024 The Carvel Authors. // SPDX-License-Identifier: Apache-2.0 package imagedesc diff --git a/pkg/imgpkg/plainimage/contents.go b/pkg/imgpkg/plainimage/contents.go index 882c4bf69..e6a64153f 100644 --- a/pkg/imgpkg/plainimage/contents.go +++ b/pkg/imgpkg/plainimage/contents.go @@ -62,7 +62,7 @@ func (i Contents) Push(uploadRef regname.Tag, labels map[string]string, writer I if err = p.AppendImage(img); err != nil { return "", err } - err = ctlimg.CreateOciTarFileAndDeleteFolder(i.ociTarPath, i.ociTarPath+".tar.gz") + err = ctlimg.CreateOciTarFromFiles(i.ociTarPath, i.ociTarPath+".tar.gz") if err != nil { return "", err }