Repository: gruntwork-io/terragrunt (language: go)

Issue:
TFLint execution fail on reading generated files
Found that in case of multiple dependent modules, execution of tflint hook may fail on reading random files from `.terragrunt-cache` directory

```
Failed to load configurations; :0,0-0: Failed to read file; The file "../../../../../../../../../_regional/ecr-repos/.terragrunt-cache/7I4-il5IuWxHyoYlk8UCZ4i7ALo/fCw8UgbtyEKZwmAHcT_oAWkqDzs/modules/data-stores/ecr-repos/outputs.tf" does not exist., and 3 other diagnostic(s):

  on  line 0:
  (source code not available)

The file "../../../../../../../../../_regional/ecr-repos/.terragrunt-cache/7I4-il5IuWxHyoYlk8UCZ4i7ALo/fCw8UgbtyEKZwmAHcT_oAWkqDzs/modules/data-stores/ecr-repos/outputs.tf" does not exist.

Error: Failed to read file

  on  line 0:
  (source code not available)

The file "../../../../../../../../../_regional/ecr-repos/.terragrunt-cache/7I4-il5IuWxHyoYlk8UCZ4i7ALo/fCw8UgbtyEKZwmAHcT_oAWkqDzs/modules/data-stores/ecr-repos/provider.tf" does not exist.

```


Discussion / hints:

Fix patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9cc63bcc4e..f509075392 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -85,6 +85,14 @@ jobs:
       - run:
           command: terratest_log_parser --testlog logs/integration.log --outputdir logs
           when: always
+      # Run TFLint tests separately as tflint during execution change working directory.
+      - run:
+          command: |
+            run-go-tests --packages "-tags tflint -run TestTflint ./test" | tee logs/integration.log
+          no_output_timeout: 30m
+      - run:
+          command: terratest_log_parser --testlog logs/integration.log --outputdir logs
+          when: always
       - store_artifacts:
           path: logs
       - store_test_results:
diff --git a/cli/cli_app.go b/cli/cli_app.go
index be33c39777..88f059a7d0 100644
--- a/cli/cli_app.go
+++ b/cli/cli_app.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gruntwork-io/terragrunt/tflint"
@@ -296,6 +297,9 @@ var terragruntHelp = map[string]string{
 	CMD_TERRAGRUNT_VALIDATE_INPUTS: validateInputsHelp,
 }
 
+// map to keep track of locks for source changes
+var sourceChangeLocks = sync.Map{}
+
 // Create the Terragrunt CLI App
 func CreateTerragruntCli(version string, writer io.Writer, errwriter io.Writer) *cli.App {
 	cli.OsExiter = func(exitCode int) {
@@ -494,21 +498,8 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 
 	// Handle code generation configs, both generate blocks and generate attribute of remote_state.
 	// Note that relative paths are relative to the terragrunt working dir (where terraform is called).
-	for _, config := range terragruntConfig.GenerateConfigs {
-		if err := codegen.WriteToFile(updatedTerragruntOptions, updatedTerragruntOptions.WorkingDir, config); err != nil {
-			return err
-		}
-	}
-	if terragruntConfig.RemoteState != nil && terragruntConfig.RemoteState.Generate != nil {
-		if err := terragruntConfig.RemoteState.GenerateTerraformCode(updatedTerragruntOptions); err != nil {
-			return err
-		}
-	} else if terragruntConfig.RemoteState != nil {
-		// We use else if here because we don't need to check the backend configuration is defined when the remote state
-		// block has a `generate` attribute configured.
-		if err := checkTerraformCodeDefinesBackend(updatedTerragruntOptions, terragruntConfig.RemoteState.Backend); err != nil {
-			return err
-		}
+	if err = generateConfig(terragruntConfig, updatedTerragruntOptions); err != nil {
+		return err
 	}
 
 	// We do the terragrunt input validation here, after all the terragrunt generated terraform files are created so
@@ -539,6 +530,31 @@ func RunTerragrunt(terragruntOptions *options.TerragruntOptions) error {
 	return runTerragruntWithConfig(terragruntOptions, updatedTerragruntOptions, terragruntConfig, false)
 }
 
+func generateConfig(terragruntConfig *config.TerragruntConfig, updatedTerragruntOptions *options.TerragruntOptions) error {
+	rawActualLock, _ := sourceChangeLocks.LoadOrStore(updatedTerragruntOptions.DownloadDir, &sync.Mutex{})
+	actualLock := rawActualLock.(*sync.Mutex)
+	defer actualLock.Unlock()
+	actualLock.Lock()
+
+	for _, config := range terragruntConfig.GenerateConfigs {
+		if err := codegen.WriteToFile(updatedTerragruntOptions, updatedTerragruntOptions.WorkingDir, config); err != nil {
+			return err
+		}
+	}
+	if terragruntConfig.RemoteState != nil && terragruntConfig.RemoteState.Generate != nil {
+		if err := terragruntConfig.RemoteState.GenerateTerraformCode(updatedTerragruntOptions); err != nil {
+			return err
+		}
+	} else if terragruntConfig.RemoteState != nil {
+		// We use else if here because we don't need to check the backend configuration is defined when the remote state
+		// block has a `generate` attribute configured.
+		if err := checkTerraformCodeDefinesBackend(updatedTerragruntOptions, terragruntConfig.RemoteState.Backend); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // Check the version constraints of both terragrunt and terraform. Note that as a side effect this will set the
 // following settings on terragruntOptions:
 // - TerraformPath
@@ -715,6 +731,10 @@ func processHooks(hooks []config.Hook, terragruntOptions *options.TerragruntOpti
 				workingDir = *curHook.WorkingDir
 			}
 
+			rawActualLock, _ := sourceChangeLocks.LoadOrStore(workingDir, &sync.Mutex{})
+			actualLock := rawActualLock.(*sync.Mutex)
+			actualLock.Lock()
+
 			actionToExecute := curHook.Execute[0]
 			actionParams := curHook.Execute[1:]
 
@@ -737,7 +757,7 @@ func processHooks(hooks []config.Hook, terragruntOptions *options.TerragruntOpti
 					errorsOccured = multierror.Append(errorsOccured, possibleError)
 				}
 			}
-
+			actualLock.Unlock()
 		}
 	}
 


Test patch:
diff --git a/test/fixture-parallel-run/.tflint.hcl b/test/fixture-parallel-run/.tflint.hcl
new file mode 100644
index 0000000000..1e7464a5b6
--- /dev/null
+++ b/test/fixture-parallel-run/.tflint.hcl
@@ -0,0 +1,13 @@
+plugin "aws-cis" {
+  enabled = true
+  version = "0.0.2"
+  source  = "github.com/gruntwork-io/tflint-ruleset-aws-cis"
+}
+
+config {
+  module = true
+}
+
+plugin "terraform" {
+  enabled = false
+}
diff --git a/test/fixture-parallel-run/common/terragrunt.hcl b/test/fixture-parallel-run/common/terragrunt.hcl
new file mode 100644
index 0000000000..1832eca1e9
--- /dev/null
+++ b/test/fixture-parallel-run/common/terragrunt.hcl
@@ -0,0 +1,63 @@
+terraform {
+  source = "github.com/gruntwork-io/terragrunt.git//test/fixture-dirs?ref=v0.35.1"
+}
+
+generate "providers" {
+  path      = "providers.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+provider "aws" {
+  region              = "us-east-1"
+}
+EOF
+}
+
+generate "outputs_1" {
+  path      = "outputs_1.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+output "outputs_1" {
+    value = "outputs_1"
+}
+EOF
+}
+
+generate "outputs_2" {
+  path      = "outputs_2.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+output "outputs_2" {
+    value = "outputs_2"
+}
+EOF
+}
+
+generate "outputs_3" {
+  path      = "outputs_3.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+output "outputs_3" {
+    value = "outputs_3"
+}
+EOF
+}
+
+generate "outputs_4" {
+  path      = "outputs_4.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+output "outputs_4" {
+    value = "outputs_4"
+}
+EOF
+}
+
+generate "outputs_5" {
+  path      = "outputs_5.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+output "outputs_5" {
+    value = "outputs_5"
+}
+EOF
+}
diff --git a/test/fixture-parallel-run/dev/app/terragrunt.hcl b/test/fixture-parallel-run/dev/app/terragrunt.hcl
new file mode 100644
index 0000000000..b7c0457740
--- /dev/null
+++ b/test/fixture-parallel-run/dev/app/terragrunt.hcl
@@ -0,0 +1,49 @@
+include "root" {
+  path = find_in_parent_folders()
+}
+
+terraform {
+  source = "github.com/gruntwork-io/terragrunt.git//test/fixture-dirs?ref=v0.35.1"
+}
+
+dependency "common" {
+  config_path = "../../common"
+  mock_outputs_allowed_terraform_commands = ["validate", "plan"]
+  mock_outputs = {
+    vpc_id = "fake-vpc-id"
+  }
+
+  skip_outputs = "true"
+}
+
+
+generate "provider" {
+  path      = "providers.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+provider "aws" {
+  region              = "us-east-1"
+}
+EOF
+}
+
+
+generate "outputs_1" {
+  path      = "outputs_1.tf"
+  if_exists = "overwrite"
+  contents = <<EOF
+output "outputs_1" {
+    value = "outputs_1"
+}
+EOF
+}
+
+generate "outputs_2" {
+  path      = "outputs_2.tf"
+  if_exists = "overwrite"
+  contents  = <<EOF
+output "outputs_2" {
+    value = "outputs_2"
+}
+EOF
+}
diff --git a/test/fixture-parallel-run/terragrunt.hcl b/test/fixture-parallel-run/terragrunt.hcl
new file mode 100644
index 0000000000..642066a6a3
--- /dev/null
+++ b/test/fixture-parallel-run/terragrunt.hcl
@@ -0,0 +1,7 @@
+terraform {
+  include_in_copy = [".terraform-version"]
+  before_hook "tflint" {
+    commands = ["apply", "plan", "init"]
+    execute  = ["tflint"]
+  }
+}
diff --git a/test/integration_test.go b/test/integration_test.go
index 60f26406bc..137739e5f9 100644
--- a/test/integration_test.go
+++ b/test/integration_test.go
@@ -138,6 +138,7 @@ const (
 	TEST_FIXTURE_TFLINT_NO_CONFIG_FILE                      = "fixture-tflint/no-config-file"
 	TEST_FIXTURE_TFLINT_MODULE_FOUND                        = "fixture-tflint/module-found"
 	TEST_FIXTURE_TFLINT_NO_TF_SOURCE_PATH                   = "fixture-tflint/no-tf-source"
+	TEST_FIXTURE_PARALLEL_RUN                               = "fixture-parallel-run"
 	TERRAFORM_BINARY                                        = "terraform"
 	TERRAFORM_FOLDER                                        = ".terraform"
 	TERRAFORM_STATE                                         = "terraform.tfstate"
@@ -5224,7 +5225,6 @@ func TestMockOutputsMergeWithState(t *testing.T) {
 	err = runTerragruntCommand(t, fmt.Sprintf("terragrunt apply --terragrunt-log-level debug --terragrunt-non-interactive -auto-approve --terragrunt-working-dir %s", shallowPath), &stdout, &stderr)
 	logBufferContentsLineByLine(t, stdout, "shallow-map-executed")
 	require.NoError(t, err)
-
 }
 
 func validateBoolOutput(t *testing.T, outputs map[string]TerraformOutput, key string, value bool) {
@@ -5233,57 +5233,3 @@ func validateBoolOutput(t *testing.T, outputs map[string]TerraformOutput, key st
 	require.Truef(t, hasPlatform, "Expected output %s to be defined", key)
 	require.Equalf(t, output.Value, value, "Expected output %s to be %t", key, value)
 }
-
-func TestTflintFindsNoIssuesWithValidCode(t *testing.T) {
-	out := new(bytes.Buffer)
-	errOut := new(bytes.Buffer)
-	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_NO_ISSUES_FOUND)
-	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_NO_ISSUES_FOUND)
-	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-log-level debug --terragrunt-working-dir %s", modulePath), out, errOut)
-	assert.NoError(t, err)
-
-	assert.NotContains(t, errOut.String(), "Error while running tflint with args:")
-	assert.NotContains(t, errOut.String(), "Tflint found issues in the project. Check for the tflint logs above.")
-
-	assert.Contains(t, errOut.String(), fmt.Sprintf("--config %s/.tflint.hcl", modulePath))
-}
-
-func TestTflintFindsModule(t *testing.T) {
-	out := new(bytes.Buffer)
-	errOut := new(bytes.Buffer)
-	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_MODULE_FOUND)
-	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_MODULE_FOUND)
-	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-working-dir %s", modulePath), out, errOut)
-	assert.NoError(t, err)
-
-	assert.NotContains(t, errOut.String(), "Error while running tflint with args:")
-	assert.NotContains(t, errOut.String(), "Tflint found issues in the project. Check for the tflint logs above.")
-}
-
-func TestTflintFindsIssuesWithInvalidInput(t *testing.T) {
-	errOut := new(bytes.Buffer)
-	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_ISSUES_FOUND)
-	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_ISSUES_FOUND)
-	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-working-dir %s", modulePath), os.Stdout, errOut)
-	assert.Error(t, err, "Tflint found issues in the project. Check for the tflint logs")
-}
-
-func TestTflintWithoutConfigFile(t *testing.T) {
-	errOut := new(bytes.Buffer)
-	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_NO_CONFIG_FILE)
-	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_NO_CONFIG_FILE)
-	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-working-dir %s", modulePath), io.Discard, errOut)
-	assert.Error(t, err, "Could not find .tflint.hcl config file in the parent folders:")
-}
-
-func TestTflintFindsConfigInCurrentPath(t *testing.T) {
-	out := new(bytes.Buffer)
-	errOut := new(bytes.Buffer)
-	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_NO_TF_SOURCE_PATH)
-	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_NO_TF_SOURCE_PATH)
-	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-log-level debug --terragrunt-working-dir %s", modulePath), out, errOut)
-	assert.NoError(t, err)
-
-	assert.Contains(t, errOut.String(), "Tflint has run successfully. No issues found")
-	assert.Contains(t, errOut.String(), fmt.Sprintf("--config %s/.tflint.hcl", modulePath))
-}
diff --git a/test/integration_tflint_test.go b/test/integration_tflint_test.go
new file mode 100644
index 0000000000..f4a1aa8557
--- /dev/null
+++ b/test/integration_tflint_test.go
@@ -0,0 +1,86 @@
+//go:build tflint
+// +build tflint
+
+package test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/gruntwork-io/terragrunt/util"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTflintFindsNoIssuesWithValidCode(t *testing.T) {
+	out := new(bytes.Buffer)
+	errOut := new(bytes.Buffer)
+	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_NO_ISSUES_FOUND)
+	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_NO_ISSUES_FOUND)
+	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-log-level debug --terragrunt-working-dir %s", modulePath), out, errOut)
+	assert.NoError(t, err)
+
+	assert.NotContains(t, errOut.String(), "Error while running tflint with args:")
+	assert.NotContains(t, errOut.String(), "Tflint found issues in the project. Check for the tflint logs above.")
+
+	assert.Contains(t, errOut.String(), fmt.Sprintf("--config %s/.tflint.hcl", modulePath))
+}
+
+func TestTflintFindsModule(t *testing.T) {
+	out := new(bytes.Buffer)
+	errOut := new(bytes.Buffer)
+	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_MODULE_FOUND)
+	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_MODULE_FOUND)
+	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-working-dir %s", modulePath), out, errOut)
+	assert.NoError(t, err)
+
+	assert.NotContains(t, errOut.String(), "Error while running tflint with args:")
+	assert.NotContains(t, errOut.String(), "Tflint found issues in the project. Check for the tflint logs above.")
+}
+
+func TestTflintFindsIssuesWithInvalidInput(t *testing.T) {
+	errOut := new(bytes.Buffer)
+	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_ISSUES_FOUND)
+	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_ISSUES_FOUND)
+	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-working-dir %s", modulePath), os.Stdout, errOut)
+	assert.Error(t, err, "Tflint found issues in the project. Check for the tflint logs")
+}
+
+func TestTflintWithoutConfigFile(t *testing.T) {
+	errOut := new(bytes.Buffer)
+	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_NO_CONFIG_FILE)
+	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_NO_CONFIG_FILE)
+	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-working-dir %s", modulePath), io.Discard, errOut)
+	assert.Error(t, err, "Could not find .tflint.hcl config file in the parent folders:")
+}
+
+func TestTflintFindsConfigInCurrentPath(t *testing.T) {
+	out := new(bytes.Buffer)
+	errOut := new(bytes.Buffer)
+	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_TFLINT_NO_TF_SOURCE_PATH)
+	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_TFLINT_NO_TF_SOURCE_PATH)
+	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt plan --terragrunt-log-level debug --terragrunt-working-dir %s", modulePath), out, errOut)
+	assert.NoError(t, err)
+
+	assert.Contains(t, errOut.String(), "Tflint has run successfully. No issues found")
+	assert.Contains(t, errOut.String(), fmt.Sprintf("--config %s/.tflint.hcl", modulePath))
+}
+
+func TestTflintInitSameModule(t *testing.T) {
+	rootPath := copyEnvironmentWithTflint(t, TEST_FIXTURE_PARALLEL_RUN)
+	t.Cleanup(func() {
+		removeFolder(t, rootPath)
+	})
+	modulePath := util.JoinPath(rootPath, TEST_FIXTURE_PARALLEL_RUN)
+	runPath := util.JoinPath(rootPath, TEST_FIXTURE_PARALLEL_RUN, "dev")
+	appTemplate := util.JoinPath(rootPath, TEST_FIXTURE_PARALLEL_RUN, "dev", "app")
+	// generate multiple "app" modules that will be initialized in parallel
+	for i := 0; i < 50; i++ {
+		appPath := util.JoinPath(modulePath, "dev", fmt.Sprintf("app-%d", i))
+		err := util.CopyFolderContents(appTemplate, appPath, ".terragrunt-test", []string{})
+		assert.NoError(t, err)
+	}
+	runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-log-level debug --terragrunt-non-interactive --terragrunt-working-dir %s", runPath))
+}