Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: introducing "copy_terraform_lock_file" to fine tune Lock File Handling #2889

Merged
merged 12 commits into from
Sep 24, 2024
Merged
8 changes: 6 additions & 2 deletions cli/commands/terraform/action.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ func runTerragruntWithConfig(ctx context.Context, originalTerragruntOptions *opt
runTerraformError := runTerraformWithRetry(ctx, terragruntOptions)

var lockFileError error
if shouldCopyLockFile(terragruntOptions.TerraformCliArgs) {
if shouldCopyLockFile(terragruntOptions.TerraformCliArgs, terragruntConfig.Terraform) {
// Copy the lock file from the Terragrunt working dir (e.g., .terragrunt-cache/xxx/<some-module>) to the
// user's working dir (e.g., /live/stage/vpc). That way, the lock file will end up right next to the user's
// terragrunt.hcl and can be checked into version control. Note that in the past, Terragrunt allowed the
Expand Down Expand Up @@ -347,7 +347,11 @@ func confirmActionWithDependentModules(ctx context.Context, terragruntOptions *o
// There are lots of details at [hashicorp/terraform#27264](https://github.com/hashicorp/terraform/issues/27264#issuecomment-743389837)
// The `providers lock` sub command enables you to ensure that the lock file is
// fully populated.
func shouldCopyLockFile(args []string) bool {
func shouldCopyLockFile(args []string, terraformConfig *config.TerraformConfig) bool {
if terraformConfig != nil && terraformConfig.CopyTerraformLockFile != nil && !*terraformConfig.CopyTerraformLockFile {
return false
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be simplified as return *terraformConfig.CopyTerraformLockFile, with the last part of the if statement removed above.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks for the suggestion @yhakbar, but I am not sure if we should perform an early return like that, if we do that and the user explicitly defines copyTerraformLockFile = true on their terragrunt.hcl, terragrunt will copy the file on all commands, which will change the existing behaviour about copying it only for the init and providers lock command. Thoughts?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, good point. It might help to leave a comment above the check to make sure that someone doesn't make the same logical mistake I'm making.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done @yhakbar, added the comment, do you mind having another look? thanks in advance

}

if util.FirstArg(args) == terraform.CommandNameInit {
return true
}
Expand Down
74 changes: 74 additions & 0 deletions cli/commands/terraform/action_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -480,3 +480,77 @@ func createTempFile(t *testing.T) string {

return filepath.ToSlash(tmpFile.Name())
}

func TestShouldCopyLockFile(t *testing.T) {
type args struct {
args []string
terraformConfig *config.TerraformConfig
}
tests := []struct {
name string
args args
want bool
}{
{
name: "init without terraform config",
args: args{
args: []string{"init"},
},
want: true,
},
{
name: "providers lock without terraform config",
args: args{
args: []string{"providers", "lock"},
},
want: true,
},
{
name: "providers schema without terraform config",
args: args{
args: []string{"providers", "schema"},
},
want: false,
},
{
name: "plan without terraform config",
args: args{
args: []string{"plan"},
},
want: false,
},
{
name: "init with empty terraform config",
args: args{
args: []string{"init"},
terraformConfig: &config.TerraformConfig{},
},
want: true,
},
{
name: "init with CopyTerraformLockFile enabled",
args: args{
args: []string{"init"},
terraformConfig: &config.TerraformConfig{
CopyTerraformLockFile: &[]bool{true}[0],
},
},
want: true,
},
{
name: "init with CopyTerraformLockFile disabled",
args: args{
args: []string{"init"},
terraformConfig: &config.TerraformConfig{
CopyTerraformLockFile: &[]bool{false}[0],
},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, shouldCopyLockFile(tt.args.args, tt.args.terraformConfig), "shouldCopyLockFile(%v, %v)", tt.args.args, tt.args.terraformConfig)
})
}
}
2 changes: 2 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,8 @@ type TerraformConfig struct {
// Ideally we can avoid the pointer to list slice, but if it is not a pointer, Terraform requires the attribute to
// be defined and we want to make this optional.
IncludeInCopy *[]string `hcl:"include_in_copy,attr"`

CopyTerraformLockFile *bool `hcl:"copy_terraform_lock_file,attr"`
}

func (conf *TerraformConfig) String() string {
Expand Down
26 changes: 14 additions & 12 deletions config/config_as_cty.go
Original file line number Diff line number Diff line change
Expand Up @@ -398,12 +398,13 @@ func catalogConfigAsCty(config *CatalogConfig) (cty.Value, error) {
// ctyTerraformConfig is an alternate representation of TerraformConfig that converts internal blocks into a map that
// maps the name to the underlying struct, as opposed to a list representation.
type ctyTerraformConfig struct {
ExtraArgs map[string]TerraformExtraArguments `cty:"extra_arguments"`
Source *string `cty:"source"`
IncludeInCopy *[]string `cty:"include_in_copy"`
BeforeHooks map[string]Hook `cty:"before_hook"`
AfterHooks map[string]Hook `cty:"after_hook"`
ErrorHooks map[string]ErrorHook `cty:"error_hook"`
ExtraArgs map[string]TerraformExtraArguments `cty:"extra_arguments"`
Source *string `cty:"source"`
IncludeInCopy *[]string `cty:"include_in_copy"`
CopyTerraformLockFile *bool `cty:"copy_terraform_lock_file"`
BeforeHooks map[string]Hook `cty:"before_hook"`
AfterHooks map[string]Hook `cty:"after_hook"`
ErrorHooks map[string]ErrorHook `cty:"error_hook"`
}

// Serialize TerraformConfig to a cty Value, but with maps instead of lists for the blocks.
Expand All @@ -413,12 +414,13 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) {
}

configCty := ctyTerraformConfig{
Source: config.Source,
IncludeInCopy: config.IncludeInCopy,
ExtraArgs: map[string]TerraformExtraArguments{},
BeforeHooks: map[string]Hook{},
AfterHooks: map[string]Hook{},
ErrorHooks: map[string]ErrorHook{},
Source: config.Source,
IncludeInCopy: config.IncludeInCopy,
CopyTerraformLockFile: config.CopyTerraformLockFile,
ExtraArgs: map[string]TerraformExtraArguments{},
BeforeHooks: map[string]Hook{},
AfterHooks: map[string]Hook{},
ErrorHooks: map[string]ErrorHook{},
}

for _, arg := range config.ExtraArgs {
Expand Down
6 changes: 6 additions & 0 deletions config/include.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,9 @@ func (targetConfig *TerragruntConfig) Merge(sourceConfig *TerragruntConfig, terr
if sourceConfig.Terraform.Source != nil {
targetConfig.Terraform.Source = sourceConfig.Terraform.Source
}
if sourceConfig.Terraform.CopyTerraformLockFile != nil {
targetConfig.Terraform.CopyTerraformLockFile = sourceConfig.Terraform.CopyTerraformLockFile
}
mergeExtraArgs(terragruntOptions, sourceConfig.Terraform.ExtraArgs, &targetConfig.Terraform.ExtraArgs)

mergeHooks(terragruntOptions, sourceConfig.Terraform.BeforeHooks, &targetConfig.Terraform.BeforeHooks)
Expand Down Expand Up @@ -408,6 +411,9 @@ func (targetConfig *TerragruntConfig) DeepMerge(sourceConfig *TerragruntConfig,
if sourceConfig.Terraform.Source != nil {
targetConfig.Terraform.Source = sourceConfig.Terraform.Source
}
if sourceConfig.Terraform.CopyTerraformLockFile != nil {
targetConfig.Terraform.CopyTerraformLockFile = sourceConfig.Terraform.CopyTerraformLockFile
}

if sourceConfig.Terraform.IncludeInCopy != nil {
srcList := *sourceConfig.Terraform.IncludeInCopy
Expand Down
11 changes: 11 additions & 0 deletions config/include_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,11 @@ func TestMergeConfigIntoIncludedConfig(t *testing.T) {
&TerragruntConfig{IamWebIdentityToken: "token"},
&TerragruntConfig{IamWebIdentityToken: "token"},
},
{
&TerragruntConfig{Terraform: &TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}},
&TerragruntConfig{Terraform: &TerraformConfig{IncludeInCopy: &[]string{"abc"}}},
&TerragruntConfig{Terraform: &TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}},
},
}

for _, testCase := range testCases {
Expand Down Expand Up @@ -286,6 +291,12 @@ func TestDeepMergeConfigIntoIncludedConfig(t *testing.T) {
&TerragruntConfig{Inputs: originalMap},
&TerragruntConfig{Inputs: mergedMap},
},
{
"terraform copy_terraform_lock_file",
&TerragruntConfig{Terraform: &TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0]}},
&TerragruntConfig{Terraform: &TerraformConfig{IncludeInCopy: &[]string{"abc"}}},
&TerragruntConfig{Terraform: &TerraformConfig{CopyTerraformLockFile: &[]bool{false}[0], IncludeInCopy: &[]string{"abc"}}},
},
}

for _, testCase := range testCases {
Expand Down
13 changes: 13 additions & 0 deletions docs/_docs/02_features/lock-file-handling.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,16 @@ should end up looking something like this:

Also, any time you change the providers you're using, and re-run `init`, the lock file will be updated, so make sure
to check the updates into version control too.

### Disabling the copy of the generated lock file

In certain use cases, like when using a remote source module, containing the lock file within it, you probability
rodrigorfk marked this conversation as resolved.
Show resolved Hide resolved
don't want Terragrunt to also copy the lock file into you working directory, in such scenarios you can opt out from the copy
rodrigorfk marked this conversation as resolved.
Show resolved Hide resolved
feature by using `copy_terraform_lock_file = false` in the `terragrunt.hcl` file as following:
rodrigorfk marked this conversation as resolved.
Show resolved Hide resolved

```hcl
terraform {
...
copy_terraform_lock_file = false
}
```
5 changes: 5 additions & 0 deletions docs/_docs/04_reference/config-blocks-and-attributes.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ The `terraform` block supports the following arguments:
can specify that in this list to ensure it gets copied over to the scratch copy
(e.g., `include_in_copy = [".python-version"]`).

- `copy_terraform_lock_file` (attribute): In certain use cases, you don't want to check the terraform provider lock
file into your source repository from your working directory as described in
[Lock File Handling]({{site.baseurl}}/docs/features/lock-file-handling/). This attribute allows you disabling the copy
rodrigorfk marked this conversation as resolved.
Show resolved Hide resolved
of the generated or existing `.terraform.lock.hcl` from the temp folder into the working directory. Default is `true`.

- `extra_arguments` (block): Nested blocks used to specify extra CLI arguments to pass to the `terraform` CLI. Learn more
about its usage in the [Keep your CLI flags DRY]({{site.baseurl}}/docs/features/keep-your-cli-flags-dry/) use case overview. Supports
the following arguments:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
inputs = {
name = "World"
}

terraform {
source = "../hello-world"
copy_terraform_lock_file = false
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
inputs = {
name = "Module A"
}

terraform {
source = "../../hello-world"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
inputs = {
name = "Module B"
}

terraform {
source = "../../hello-world"
copy_terraform_lock_file = false
}

prevent_destroy = true

include {
path = find_in_parent_folders("terragrunt.hcl")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
terraform {
include_in_copy = ["**/.terraform-version"]
}

dependencies {
paths = ["../module-a"]
}
5 changes: 3 additions & 2 deletions test/fixture-read-config/full/source.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ remote_state {
}

terraform {
source = "./delorean"
include_in_copy = ["time_machine.*"]
source = "./delorean"
include_in_copy = ["time_machine.*"]
copy_terraform_lock_file = true

extra_arguments "var-files" {
commands = ["apply", "plan"]
Expand Down
60 changes: 45 additions & 15 deletions test/integration_download_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,23 @@ import (
)

const (
testFixtureLocalDownloadPath = "fixture-download/local"
testFixtureCustomLockFile = "fixture-download/custom-lock-file"
testFixtureRemoteDownloadPath = "fixture-download/remote"
testFixtureInvalidRemoteDownloadPath = "fixture-download/remote-invalid"
testFixtureOverrideDonwloadPath = "fixture-download/override"
testFixtureLocalRelativeDownloadPath = "fixture-download/local-relative"
testFixtureRemoteRelativeDownloadPath = "fixture-download/remote-relative"
testFixtureLocalWithBackend = "fixture-download/local-with-backend"
testFixtureLocalWithExcludeDir = "fixture-download/local-with-exclude-dir"
testFixtureLocalWithIncludeDir = "fixture-download/local-with-include-dir"
testFixtureRemoteWithBackend = "fixture-download/remote-with-backend"
testFixtureRemoteModuleInRoot = "fixture-download/remote-module-in-root"
testFixtureLocalMissingBackend = "fixture-download/local-with-missing-backend"
testFixtureLocalWithHiddenFolder = "fixture-download/local-with-hidden-folder"
testFixtureLocalWithAllowedHidden = "fixture-download/local-with-allowed-hidden"
testFixtureLocalDownloadPath = "fixture-download/local"
testFixtureCustomLockFile = "fixture-download/custom-lock-file"
testFixtureRemoteDownloadPath = "fixture-download/remote"
testFixtureInvalidRemoteDownloadPath = "fixture-download/remote-invalid"
testFixtureOverrideDonwloadPath = "fixture-download/override"
testFixtureLocalRelativeDownloadPath = "fixture-download/local-relative"
testFixtureRemoteRelativeDownloadPath = "fixture-download/remote-relative"
testFixtureLocalWithBackend = "fixture-download/local-with-backend"
testFixtureLocalWithExcludeDir = "fixture-download/local-with-exclude-dir"
testFixtureLocalWithIncludeDir = "fixture-download/local-with-include-dir"
testFixtureRemoteWithBackend = "fixture-download/remote-with-backend"
testFixtureRemoteModuleInRoot = "fixture-download/remote-module-in-root"
testFixtureLocalMissingBackend = "fixture-download/local-with-missing-backend"
testFixtureLocalWithHiddenFolder = "fixture-download/local-with-hidden-folder"
testFixtureLocalWithAllowedHidden = "fixture-download/local-with-allowed-hidden"
testFixtureDisableCopyLockFilePath = "fixture-download/local-disable-copy-terraform-lock-file"
testFixtureIncludeDisableCopyLockFilePath = "fixture-download/local-include-disable-copy-lock-file/module-b"
)

func TestLocalDownload(t *testing.T) {
Expand All @@ -49,6 +51,34 @@ func TestLocalDownload(t *testing.T) {
runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", testFixtureLocalDownloadPath))
}

func TestLocalDownloadDisableCopyTerraformLockFile(t *testing.T) {
t.Parallel()

cleanupTerraformFolder(t, testFixtureDisableCopyLockFilePath)

runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", testFixtureDisableCopyLockFilePath))

// The terraform lock file should not be copied if `copy_terraform_lock_file = false`
assert.NoFileExists(t, util.JoinPath(testFixtureDisableCopyLockFilePath, util.TerraformLockFile))

// Run a second time to make sure the temporary folder can be reused without errors
runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", testFixtureDisableCopyLockFilePath))
}

func TestLocalIncludeDisableCopyTerraformLockFile(t *testing.T) {
t.Parallel()

cleanupTerraformFolder(t, testFixtureIncludeDisableCopyLockFilePath)

runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", testFixtureIncludeDisableCopyLockFilePath))

// The terraform lock file should not be copied if `copy_terraform_lock_file = false`
assert.NoFileExists(t, util.JoinPath(testFixtureIncludeDisableCopyLockFilePath, util.TerraformLockFile))

// Run a second time to make sure the temporary folder can be reused without errors
runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-non-interactive --terragrunt-working-dir %s", testFixtureIncludeDisableCopyLockFilePath))
}

func TestLocalDownloadWithHiddenFolder(t *testing.T) {
t.Parallel()

Expand Down
18 changes: 10 additions & 8 deletions test/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3640,8 +3640,9 @@ func TestReadTerragruntConfigFull(t *testing.T) {
t,
terraformOut,
map[string]interface{}{
"source": "./delorean",
"include_in_copy": []interface{}{"time_machine.*"},
"source": "./delorean",
"include_in_copy": []interface{}{"time_machine.*"},
"copy_terraform_lock_file": true,
"extra_arguments": map[string]interface{}{
"var-files": map[string]interface{}{
"name": "var-files",
Expand Down Expand Up @@ -5679,12 +5680,13 @@ func TestRenderJsonMetadataTerraform(t *testing.T) {
var expectedTerraform = map[string]interface{}{
"metadata": terragruntMetadata,
"value": map[string]interface{}{
"after_hook": map[string]interface{}{},
"before_hook": map[string]interface{}{},
"error_hook": map[string]interface{}{},
"extra_arguments": map[string]interface{}{},
"include_in_copy": nil,
"source": "../terraform",
"after_hook": map[string]interface{}{},
"before_hook": map[string]interface{}{},
"copy_terraform_lock_file": nil,
"error_hook": map[string]interface{}{},
"extra_arguments": map[string]interface{}{},
"include_in_copy": nil,
"source": "../terraform",
},
}

Expand Down