Merge pull request #18 from marcial-lopezferrada-hs/config-dir

add support for a config.d directory

config/config.go

@@ -122,7 +122,6 @@ func ReadConfig(log zerolog.Logger, config []byte, inputPrefix, outputPrefix str
 	}
 
 	current.log = log
-
 	errs := current.prepareConfig(inputPrefix, outputPrefix)
 
 	if len(errs) > 0 {
@@ -151,7 +150,7 @@ func ReadConfig(log zerolog.Logger, config []byte, inputPrefix, outputPrefix str
 		Composites:  composites,
 	}, nil
 }
-func ReadConfigFile(configFile string, inputPrefix, outputPrefix string) (*ControlToolConfig, error) {
+func ReadConfigFile(configFile string, configDir string, inputPrefix, outputPrefix string) (*ControlToolConfig, error) {
 	if configFile == "" {
 		return nil, fmt.Errorf("a --config file is required to be specified")
 	}
@@ -168,7 +167,52 @@ func ReadConfigFile(configFile string, inputPrefix, outputPrefix string) (*Contr
 		return nil, fmt.Errorf("trouble reading config file %q: %w", absConfigFile, err)
 	}
 
-	return ReadConfig(log, yamlFile, inputPrefix, outputPrefix)
+	config, err := ReadConfig(log, yamlFile, inputPrefix, outputPrefix)
+
+	if err != nil {
+		log.Error().Err(err).Msg("could not parse config file")
+		return nil, fmt.Errorf("could not parse config file %q: %w", absConfigFile, err)
+	}
+
+	_, err = os.Stat(util.AbsolutePath(inputPrefix, configDir))
+	if os.IsNotExist(err) {
+		zlog.Error().Err(err).Msg("config directory doesn't exist")
+
+	} else {
+		items, err := ioutil.ReadDir(util.AbsolutePath(inputPrefix, configDir))
+		if err != nil {
+			log.Error().Err(err).Msg("Error reading directory")
+			return nil, fmt.Errorf("could not read config directory %q: %w", configDir, err)
+		}
+		for _, item := range items {
+			fileExtension := filepath.Ext(item.Name())
+			if (fileExtension == ".yaml" || fileExtension == ".yml") && !item.IsDir() {
+
+				currentConfigFile := util.AbsolutePath(inputPrefix, configDir) + "/" + item.Name()
+				log.Debug().Msg("reading config file " + currentConfigFile)
+				yamlFile, err := ioutil.ReadFile(currentConfigFile)
+				if err != nil {
+					log.Error().Err(err).Msg("could not read config file")
+					return nil, fmt.Errorf("trouble reading config file %q: %w", currentConfigFile, err)
+				}
+
+				currentConfig, err := ReadConfig(log, yamlFile, inputPrefix, outputPrefix)
+				config.VaultConfig.Secrets = append(config.VaultConfig.Secrets, currentConfig.VaultConfig.Secrets...)
+				config.VaultConfig.Templates = append(config.VaultConfig.Templates, currentConfig.VaultConfig.Templates...)
+				config.VaultConfig.SSHCertificates = append(config.VaultConfig.SSHCertificates, currentConfig.VaultConfig.SSHCertificates...)
+				config.VaultConfig.AWS = append(config.VaultConfig.AWS, currentConfig.VaultConfig.AWS...)
+				for k, v := range currentConfig.Templates {
+					config.Templates[k] = v
+				}
+				for k, v := range currentConfig.Composites {
+					config.Composites[k] = v
+				}
+
+			}
+		}
+	}
+
+	return config, err
 
 }
 

config/config_test.go

@@ -3,6 +3,8 @@ package config
 import (
 	"io/ioutil"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 var validConfigs = map[string]string{
@@ -47,11 +49,27 @@ secrets:
 `,
 }
 
+var validSubConfig = map[string]string{
+	"sub config1": `---
+version: 3
+secrets:
+ - key: test
+   path: /secret/test
+   lifetime: static`,
+	"sub config2": `---
+version: 3
+secrets:
+ - key: test2
+   path: /secret/test2
+   lifetime: static
+`,
+}
+
 func TestValidConfigs(t *testing.T) {
 	for k, v := range validConfigs {
 		t.Run(k, func(t *testing.T) {
-			filename := mkConfig(t, v)
-			_, err := ReadConfigFile(filename, "", "")
+			filename := mkConfig(t, t.TempDir(), v)
+			_, err := ReadConfigFile(filename, "", "", "")
 			if err != nil {
 				t.Fatalf("this config must be okay, got error: %v", err)
 			}
@@ -62,17 +80,38 @@ func TestValidConfigs(t *testing.T) {
 func TestInvalidConfigs(t *testing.T) {
 	for k, v := range invalidConfigs {
 		t.Run(k, func(t *testing.T) {
-			filename := mkConfig(t, v)
-			_, err := ReadConfigFile(filename, "", "")
+			filename := mkConfig(t, t.TempDir(), v)
+			_, err := ReadConfigFile(filename, "", "", "")
 			if err == nil {
 				t.Fatal("this config must generate an error")
 			}
 		})
 	}
 }
 
-func mkConfig(t *testing.T, body string) string {
-	f, err := ioutil.TempFile(t.TempDir(), "config_test_*")
+func TestConfigDir(t *testing.T) {
+
+	dir := t.TempDir()
+	for configName, configData := range validSubConfig {
+		f := mkConfig(t, dir, configData)
+		t.Logf("created config for: %s : %s", configName, f)
+	}
+	emptyConfig := mkConfig(t, dir, "")
+	config, err := ReadConfigFile(emptyConfig, dir, "", "")
+	if err != nil {
+		t.Log("this config must be okay, got error")
+		t.Fail()
+	}
+
+	key := SecretType{Key: "test", Path: "/secret/test", Lifetime: "static"}
+	assert.Contains(t, config.VaultConfig.Secrets, key)
+
+	key2 := SecretType{Key: "test2", Path: "/secret/test2", Lifetime: "static"}
+	assert.Contains(t, config.VaultConfig.Secrets, key2)
+}
+
+func mkConfig(t *testing.T, directory string, body string) string {
+	f, err := ioutil.TempFile(directory, "config_test_*.yml")
 
 	if err != nil {
 		t.Fatalf("could not make temp file: %v", err)

perform.go

@@ -199,7 +199,7 @@ func PerformCleanup(flags util.CliFlags) error {
 		}
 	}
 
-	cfg, err := config.ReadConfigFile(flags.ConfigFile, flags.InputPrefix, flags.OutputPrefix)
+	cfg, err := config.ReadConfigFile(flags.ConfigFile, flags.ConfigDir, flags.InputPrefix, flags.OutputPrefix)
 	if err != nil {
 		log.Warn().Msg("could not read config file - unsure what to cleanup")
 		return fmt.Errorf("could not read config file %q: %w", flags.ConfigFile, err)

syncer/sync.go

@@ -58,7 +58,7 @@ func configureSyncerDependencies(flags util.CliFlags) (zerolog.Logger, *config.C
 
 	log := log.With().Str("cfg", flags.ConfigFile).Logger()
 
-	cfg, err := config.ReadConfigFile(flags.ConfigFile, flags.InputPrefix, flags.OutputPrefix)
+	cfg, err := config.ReadConfigFile(flags.ConfigFile, flags.ConfigDir, flags.InputPrefix, flags.OutputPrefix)
 	if err != nil {
 		return log, nil, nil, err
 	}

util/flags.go

@@ -27,6 +27,7 @@ type CliFlags struct {
 	IAMAuthRole             string        // Role to use when performing IAM authentication of EC2 instances
 	IAMVaultAuthBackend     string        // Override IAM auth path in Vault
 	ConfigFile              string        // location of vault-config, either relative to input prefix, or absolute
+	ConfigDir               string        // location of vault-config directory, either relative to input prefix, or absolute
 	OutputPrefix            string        // prefix to use when writing output files
 	InputPrefix             string        // prefix to use when looking for input files
 	ServiceSecretPrefix     string        // override prefix for relative KV secrets
@@ -110,8 +111,9 @@ func ProcessFlags(args []string) (*CliFlags, error) {
 
 	app.Flag("init", "Run in init mode, process templates and exit.").Default("false").BoolVar(&flags.PerformInit)
 	app.Flag("config", "Full path of the config file to read.").Default("vault-config.yml").StringVar(&flags.ConfigFile)
+	app.Flag("config-dir", "Full path of the config directory to read config files. Be aware that the config version is read only once so all sub config should have the same version.").Default("config.d").StringVar(&flags.ConfigDir)
 	app.Flag("output-prefix", "Path to prefix to all output files (such as /etc/secrets)").StringVar(&flags.OutputPrefix)
-	app.Flag("input-prefix", "Path to prefix on all files being read; including the config file.").StringVar(&flags.InputPrefix)
+	app.Flag("input-prefix", "Path to prefix on all files being read; including the config file. Only the main config file (--config-file) will have his root values read. Those in the config directory will be ignored.").StringVar(&flags.InputPrefix)
 	app.Flag("secret-prefix", "Vault path to prepend to secrets with relative paths").StringVar(&flags.ServiceSecretPrefix)
 
 	app.Flag("renew-interval", "Interval to renew credentials").Default("9m").DurationVar(&flags.RenewInterval)