Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP: dont merge] #2109

Draft
wants to merge 82 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
82 commits
Select commit Hold shift + click to select a range
835902d
commands: manifest: command to handle manifests
husni-faiz Mar 20, 2023
ffe1f93
cmd: manifest create: first PoC
husni-faiz Mar 20, 2023
08ec406
cmd: manifest: refactor code
husni-faiz Mar 27, 2023
9b2a978
cmd: manifest: use ggcr library
husni-faiz Apr 1, 2023
3fc79cc
cmd: manifest: use imgutil
husni-faiz Apr 5, 2023
6b6f2b0
manifest: use pack client interface
husni-faiz Apr 18, 2023
e3908c2
manifest_create: separate input validation and manifest creation
husni-faiz Apr 18, 2023
d767b1e
client: manifest sub command logic function templates
husni-faiz Apr 18, 2023
3c0b9cc
commands: manifest sub command templates
husni-faiz Apr 18, 2023
e9ca7cb
bugfix: manifest: create: use single char for shorthand flag
husni-faiz Apr 25, 2023
3a7c262
manifest: create: move flag validations into command logic
husni-faiz Apr 25, 2023
40eade3
manifest: create: remove path option from remote index:
husni-faiz Apr 25, 2023
67d5644
create_manifest: raise error if saving the index fails
husni-faiz Apr 25, 2023
b6b0f0d
manifest_create: add default layout path to save an index
husni-faiz Apr 25, 2023
43f97b5
manifest: flag command as experimental
husni-faiz Apr 30, 2023
c3ac0ce
client: create an index factory
husni-faiz May 1, 2023
1530c23
create_manifest: use the index factory
husni-faiz May 1, 2023
8460b97
manifest: register subcommands
husni-faiz May 2, 2023
715e18a
manifest: add annotate subcommand
husni-faiz May 2, 2023
4a478b9
manifest: add 'add' subcommand
husni-faiz May 2, 2023
b840b19
manifest: use pack home directory to store local manifests
husni-faiz May 9, 2023
6dee05d
manifest: return error instead of calling panic
husni-faiz May 21, 2023
ba95ad8
manifest add: AppendManifest wrapper function will be removed from im…
husni-faiz May 21, 2023
0631b34
manfiest: add remove command
husni-faiz May 23, 2023
605a994
manifest create: remove layout option
husni-faiz May 23, 2023
bb7902e
manifest add: remove redundent code
husni-faiz May 23, 2023
7be9189
manifest annotate: update example to use cnbs/sample-package
husni-faiz May 23, 2023
12b1c0f
manifest create: remove redundant comment
husni-faiz May 23, 2023
1824233
manifest create: simplify the logic for getting the mediatype
husni-faiz May 23, 2023
0b99d2b
manifest command: wrap erros with useful information
husni-faiz May 26, 2023
16b3496
manifest rm: rename file
husni-faiz May 26, 2023
090a0ef
manifest rm: change command name remove to rm
husni-faiz May 26, 2023
802b794
manifest rm: rename file
husni-faiz Jun 1, 2023
6ca7fcc
manifest remove: new command to delete a local index
husni-faiz Jun 1, 2023
bf1503e
manifest inspect: new command to inspect local index
husni-faiz Jun 1, 2023
2b8fdf1
manifest: code formatting
husni-faiz Jun 5, 2023
729c390
go mod tidy: imgutil v1.3.0
husni-faiz Jun 5, 2023
3cf9b10
Merge branch 'main' into test-manifest
jjbustamante Jun 30, 2023
4fe3a96
Merge branch 'main' into test-manifest
husni-faiz Jul 12, 2023
d4f809d
Use imgutil v1.4.0
husni-faiz Jul 12, 2023
ac9855f
fix lint
husni-faiz Jul 12, 2023
2cae965
manifest: make generate
husni-faiz Jul 12, 2023
a504b1a
fix format
husni-faiz Jul 12, 2023
4be0f38
Merge branch 'main' into test-manifest
jjbustamante Sep 20, 2023
1a7a930
WIP added pack manifest cli
WYGIN Dec 15, 2023
8fcb673
WIP updated code as per imgutil's ImageIndex PR
WYGIN Jan 9, 2024
9c52f2d
WIP upgraded imgutil to latest version
WYGIN Jan 22, 2024
13ee7f0
WIP fix some bugs of pack manifest cli
WYGIN Jan 22, 2024
5d5d8ec
WIP fix manifest cli
Feb 6, 2024
c74fe0f
WIP fix bugs related to OSFeatures and Push index
WYGIN Feb 9, 2024
6597bc8
WIP fix bugs related to imgutils
WYGIN Feb 12, 2024
3be294f
Merge branch 'main' into test-manifest
jjbustamante Feb 13, 2024
ddaf6bf
Merge branch 'test-manifest' into image-index
WYGIN Feb 14, 2024
c9e5850
WIP: added couple of tests
WYGIN Feb 19, 2024
5afa60a
added tests for client directive
WYGIN Feb 27, 2024
6d65501
added tests in commands package
WYGIN Feb 28, 2024
71316c3
chore: updated imgutil to latest version
WYGIN Mar 1, 2024
76a3678
refactor: change features, os-features, annotations to relevant flags
WYGIN Mar 4, 2024
0b477b2
fix: bug causing ignore --features, --os-features, --urls, --annotations
WYGIN Mar 4, 2024
2610c47
WIP: feat: add multi-arch support
WYGIN Mar 5, 2024
a540add
WIP: created new file to handle multi-arch
WYGIN Mar 5, 2024
ab1df4e
WIP: refactor multi_arch_config
WYGIN Mar 6, 2024
4e3aa93
WIP some bp chnages
WYGIN Mar 6, 2024
1a2bfee
WIP: feat: create image index for #SaveAsImage
WYGIN Mar 7, 2024
c8b35e7
WIP: added Setters for WorkableImage
WYGIN Mar 8, 2024
0bc2ae8
chore: generate tests for new changes
WYGIN Mar 8, 2024
6c3d14a
feat: add image to index when creating multiArch buildpacks
WYGIN Mar 15, 2024
a30445b
chore: add tests for multiArchBP
WYGIN Mar 18, 2024
3574670
fix: few bug fixes
WYGIN Mar 19, 2024
776f99d
fix: bugs in package_buildpack
WYGIN Mar 21, 2024
8063906
WIP: fix bugs; still pack buildpack package has nil pointer bugs
WYGIN Mar 22, 2024
be83b01
remove debug
WYGIN Mar 22, 2024
1aea687
WIP: feat added multi-arch for and
WYGIN Mar 25, 2024
ac2fd30
WIP: add multi-arch support for Usage:
WYGIN Mar 25, 2024
536eade
WIP: chore upgrade imgutil dependency
WYGIN Mar 25, 2024
a129b74
WIP: minor bug fixes
WYGIN Mar 26, 2024
150d28c
WIP: bug fixes for pack builder create
WYGIN Mar 26, 2024
83f7539
refactor: improve code readability
WYGIN Mar 30, 2024
94f78dc
Merge branch 'image-index' of https://github.com/WYGIN/buildpacks-pac…
SaikiranIndia Mar 30, 2024
8cd5055
upgrade dependency
WYGIN Mar 31, 2024
5d6a423
WIP: added tests for config toml files
WYGIN Apr 1, 2024
a78dd26
feat: add multi-arch file format to save
WYGIN Apr 9, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 0 additions & 18 deletions .gitpod.yml

This file was deleted.

186 changes: 180 additions & 6 deletions builder/config_reader.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/pkg/errors"

"github.com/buildpacks/pack/buildpackage"
"github.com/buildpacks/pack/internal/config"
"github.com/buildpacks/pack/internal/style"
"github.com/buildpacks/pack/pkg/dist"
Expand All @@ -25,6 +26,13 @@ type Config struct {
Lifecycle LifecycleConfig `toml:"lifecycle"`
Run RunConfig `toml:"run"`
Build BuildConfig `toml:"build"`
WithTargets []dist.Target `toml:"targets,omitempty"`
}

type MultiArchConfig struct {
Config
flagTargets []dist.Target
relativeBaseDir string
}

// ModuleCollection is a list of ModuleConfigs
Expand Down Expand Up @@ -115,6 +123,151 @@ func ReadConfig(path string) (config Config, warnings []string, err error) {
return config, warnings, nil
}

func (c *MultiArchConfig) Targets() []dist.Target {
if len(c.flagTargets) != 0 {
return c.flagTargets
}

return c.WithTargets
}

func ReadMultiArchConfig(path string, flagTargets []dist.Target) (config MultiArchConfig, warnings []string, err error) {
file, err := os.Open(filepath.Clean(path))
if err != nil {
return MultiArchConfig{}, nil, errors.Wrap(err, "opening config file")
}
defer file.Close()

config, err = parseMultiArchConfig(file)
if err != nil {
return MultiArchConfig{}, nil, errors.Wrapf(err, "parse contents of '%s'", path)
}

if len(config.Order) == 0 {
warnings = append(warnings, fmt.Sprintf("empty %s definition", style.Symbol("order")))
}

config.mergeStackWithImages()
if len(flagTargets) != 0 {
config.flagTargets = flagTargets
}
return config, warnings, nil
}

func (c *MultiArchConfig) BuilderConfigs(getIndexManifest buildpackage.GetIndexManifestFn) (configs []Config, err error) {
for _, target := range c.Targets() {
if err := target.Range(func(target dist.Target, distroName, distroVersion string) error {
cfg, err := c.processTarget(target, "", "", getIndexManifest)
configs = append(configs, cfg)
return err
}); err != nil {
return configs, err
}
}
return configs, nil
}

func copyConfig(config Config) Config {
return Config{
Description: config.Description,
Buildpacks: make(ModuleCollection, len(config.Buildpacks)),
Extensions: make(ModuleCollection, len(config.Extensions)),
Order: make(dist.Order, len(config.Order)),
OrderExtensions: make(dist.Order, len(config.OrderExtensions)),
Stack: StackConfig{
RunImageMirrors: make([]string, len(config.Stack.RunImageMirrors)),
},
Run: RunConfig{
Images: make([]RunImageConfig, len(config.Run.Images)),
},
}
}

func (c *MultiArchConfig) processTarget(target dist.Target, distroName, distroVersion string, getIndexManifest buildpackage.GetIndexManifestFn) (config Config, err error) {
config = copyConfig(c.Config)
target = buildpackage.ProcessTarget(target, distroName, distroVersion)
for i, bp := range c.Config.Buildpacks {
if bp.URI != "" {
if config.Buildpacks[i].URI, err = buildpackage.GetRelativeURI(bp.URI, c.relativeBaseDir, &target, getIndexManifest); err != nil {
return config, err
}
}
}

for i, ext := range c.Config.Extensions {
if ext.URI != "" {
if config.Extensions[i].URI, err = buildpackage.GetRelativeURI(ext.URI, c.relativeBaseDir, &target, getIndexManifest); err != nil {
return config, err
}
}
}

if img := c.Config.Build.Image; img != "" {
if config.Build.Image, err = buildpackage.ParseURItoString(img, target, getIndexManifest); err != nil {
return config, err
}
}

for i, runImg := range c.Config.Run.Images {
config.Run.Images[i].Image, err = buildpackage.ParseURItoString(runImg.Image, target, getIndexManifest)
if len(config.Run.Images[i].Mirrors) == 0 {
config.Run.Images[i].Mirrors = make([]string, len(runImg.Mirrors))
}

if err != nil {
for j, mirror := range runImg.Mirrors {
if config.Run.Images[i].Mirrors[j], err = buildpackage.ParseURItoString(mirror, target, getIndexManifest); err == nil {
break
}
}

if err != nil {
return config, err
}
}
}

if img := c.Config.Stack.BuildImage; img != "" {
if config.Stack.BuildImage, err = buildpackage.ParseURItoString(img, target, getIndexManifest); err != nil {
return config, err
}
}

if img := c.Config.Stack.RunImage; img != "" {
config.Stack.RunImage, err = buildpackage.ParseURItoString(img, target, getIndexManifest)
}

if err != nil {
for i, mirror := range c.Config.Stack.RunImageMirrors {
if config.Stack.RunImageMirrors[i], err = buildpackage.ParseURItoString(mirror, target, getIndexManifest); err == nil {
break
}
}
}

config.Order = c.Order
config.OrderExtensions = c.OrderExtensions
config.WithTargets = []dist.Target{target}
return config, err
}

func (c *MultiArchConfig) MultiArch() bool {
targets := c.Targets()
if len(targets) > 1 {
return true
}

targetsLen := 0
for _, target := range targets {
target.Range(func(_ dist.Target, _, _ string) error {
targetsLen++
return nil
})
}

return targetsLen > 1
}

// ValidateConfig validates the config
func ValidateConfig(c Config) error {
if c.Build.Image == "" && c.Stack.BuildImage == "" {
Expand Down Expand Up @@ -182,27 +335,48 @@ func parseConfig(file *os.File) (Config, error) {
return builderConfig, nil
}

// parseMultiArchConfig reads a builder configuration from file
func parseMultiArchConfig(file *os.File) (MultiArchConfig, error) {
multiArchBuilderConfig := MultiArchConfig{}
tomlMetadata, err := toml.NewDecoder(file).Decode(&multiArchBuilderConfig)
if err != nil {
return MultiArchConfig{}, errors.Wrap(err, "decoding MultiArchBuilder Toml")
}

undecodedKeys := tomlMetadata.Undecoded()
if len(undecodedKeys) > 0 {
unknownElementsMsg := config.FormatUndecodedKeys(undecodedKeys)

return MultiArchConfig{}, errors.Errorf("%s in %s",
unknownElementsMsg,
style.Symbol(file.Name()),
)
}

return multiArchBuilderConfig, nil
}

func ParseBuildConfigEnv(env []BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) {
envMap = map[string]string{}
var appendOrPrependWithoutDelim = 0
for _, v := range env {
if name := v.Name; name == "" || len(name) == 0 {
if name := v.Name; name == "" {
return nil, nil, errors.Wrapf(errors.Errorf("env name should not be empty"), "parse contents of '%s'", path)
}
if val := v.Value; val == "" || len(val) == 0 {
if val := v.Value; val == "" {
warnings = append(warnings, fmt.Sprintf("empty value for key/name %s", style.Symbol(v.Name)))
}
suffixName, delimName, err := getBuildConfigEnvFileName(v)
if err != nil {
return envMap, warnings, err
}
if val, e := envMap[suffixName]; e {
if val, ok := envMap[suffixName]; ok {
warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and suffix: %s from %s to %s", style.Symbol(v.Name), style.Symbol(string(v.Suffix)), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path))
}
if val, e := envMap[delimName]; e {
if val, ok := envMap[delimName]; ok {
warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and delim: %s from %s to %s", style.Symbol(v.Name), style.Symbol(v.Delim), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path))
}
if delim := v.Delim; (delim != "" || len(delim) != 0) && (delimName != "" || len(delimName) != 0) {
if delim := v.Delim; delim != "" && delimName != "" {
envMap[delimName] = delim
}
envMap[suffixName] = v.Value
Expand Down Expand Up @@ -234,7 +408,7 @@ func getBuildConfigEnvFileName(env BuildConfigEnv) (suffixName, delimName string
} else {
suffixName = env.Name + suffix
}
if delim := env.Delim; delim != "" || len(delim) != 0 {
if delim := env.Delim; delim != "" {
delimName = env.Name + ".delim"
}
return suffixName, delimName, err
Expand Down
Loading
Loading