Repository: gruntwork-io/terragrunt (language: go)

Issue:
Terragrunt Hangs Waiting for User Input in CI if the Remote State Backend S3 Bucket is Not Accessible
**Describe the bug**

When using an AWS S3 remote state backend and running Terragrunt within a non-interactive CI process, if the S3 bucket is not accessible for some reason (wrong credentials/permissions, doesn't exist yet etc) the Terragrunt process will hang waiting for user input that can never be given, with the message:

```
Remote state S3 bucket <BucketName> does not exist or you don't have permissions to access it. Would you like Terragrunt to create it? (y/n) 
```

**Expected behavior**

Terragrunt should either detect that it is running within a CI environment (normally env var `CI` is set) or have another CLI option to not prompt to create the bucket, but just fail.

**Versions**

- Terragrunt version: v0.46.1
- Terraform version: v1.4.6


Discussion / hints:

Fix patch:
diff --git a/cli/args.go b/cli/args.go
index 2e39e33050..81eb0cb02e 100644
--- a/cli/args.go
+++ b/cli/args.go
@@ -237,6 +237,8 @@ func parseTerragruntOptionsFromArgs(terragruntVersion string, args []string, wri
 	opts.AwsProviderPatchOverrides = awsProviderPatchOverrides
 	opts.FetchDependencyOutputFromState = parseBooleanArg(args, optTerragruntFetchDependencyOutputFromState, os.Getenv("TERRAGRUNT_FETCH_DEPENDENCY_OUTPUT_FROM_STATE") == "true")
 	opts.UsePartialParseConfigCache = parseBooleanArg(args, optTerragruntUsePartialParseConfigCache, os.Getenv("TERRAGRUNT_USE_PARTIAL_PARSE_CONFIG_CACHE") == "true")
+	opts.DisableBucketUpdate = parseBooleanArg(args, optTerragruntDisableBucketUpdate, os.Getenv("TERRAGRUNT_DISABLE_BUCKET_UPDATE") == "true")
+	opts.FailIfBucketCreationRequired = parseBooleanArg(args, optTerragruntFailOnStateBucketCreation, os.Getenv("TERRAGRUNT_FAIL_ON_STATE_BUCKET_CREATION") == "true")
 	opts.RenderJsonWithMetadata = parseBooleanArg(args, optTerragruntOutputWithMetadata, false)
 
 	opts.JSONOut, err = parseStringArg(args, optTerragruntJSONOut, "")
diff --git a/cli/cli_app.go b/cli/cli_app.go
index 6eb5a946d1..43ba54be63 100644
--- a/cli/cli_app.go
+++ b/cli/cli_app.go
@@ -64,6 +64,8 @@ const (
 	optTerragruntFetchDependencyOutputFromState = "terragrunt-fetch-dependency-output-from-state"
 	optTerragruntUsePartialParseConfigCache     = "terragrunt-use-partial-parse-config-cache"
 	optTerragruntIncludeModulePrefix            = "terragrunt-include-module-prefix"
+	optTerragruntFailOnStateBucketCreation      = "terragrunt-fail-on-state-bucket-creation"
+	optTerragruntDisableBucketUpdate            = "terragrunt-disable-bucket-update"
 	optTerragruntOutputWithMetadata             = "with-metadata"
 )
 
@@ -85,6 +87,8 @@ var allTerragruntBooleanOpts = []string{
 	optTerragruntUsePartialParseConfigCache,
 	optTerragruntOutputWithMetadata,
 	optTerragruntIncludeModulePrefix,
+	optTerragruntFailOnStateBucketCreation,
+	optTerragruntDisableBucketUpdate,
 }
 var allTerragruntStringOpts = []string{
 	optTerragruntConfig,
@@ -272,6 +276,8 @@ GLOBAL OPTIONS:
    terragrunt-json-out                          The file path that terragrunt should use when rendering the terragrunt.hcl config as json. Only used in the render-json command. Defaults to terragrunt_rendered.json.
    terragrunt-use-partial-parse-config-cache    Enables caching of includes during partial parsing operations. Will also be used for the --terragrunt-iam-role option if provided.
    terragrunt-include-module-prefix             When this flag is set output from Terraform sub-commands is prefixed with module path.
+   terragrunt-fail-on-state-bucket-creation     When this flag is set Terragrunt will fail if the remote state bucket needs to be created.
+   terragrunt-disable-bucket-update             When this flag is set Terragrunt will not update the remote state bucket.
 
 VERSION:
    {{.Version}}{{if len .Authors}}
diff --git a/docs/_docs/04_reference/cli-options.md b/docs/_docs/04_reference/cli-options.md
index 003cd2fad3..c59d128b42 100644
--- a/docs/_docs/04_reference/cli-options.md
+++ b/docs/_docs/04_reference/cli-options.md
@@ -502,6 +502,8 @@ prefix `--terragrunt-` (e.g., `--terragrunt-config`). The currently available op
 - [terragrunt-fetch-dependency-output-from-state](#terragrunt-fetch-dependency-output-from-state)
 - [terragrunt-use-partial-parse-config-cache](#terragrunt-use-partial-parse-config-cache)
 - [terragrunt-include-module-prefix](#terragrunt-include-module-prefix)
+- [terragrunt-fail-on-state-bucket-creation](#terragrunt-fail-on-state-bucket-creation)
+- [terragrunt-disable-bucket-update](#terragrunt-disable-bucket-update)
 
 ### terragrunt-config
 
@@ -930,3 +932,17 @@ NOTE: This is an experimental feature, use with caution.
 **Environment Variable**: `TERRAGRUNT_INCLUDE_MODULE_PREFIX` (set to `true`)
 
 When this flag is set output from Terraform sub-commands is prefixed with module path.
+
+### terragrunt-fail-on-state-bucket-creation
+
+**CLI Arg**: `--terragrunt-fail-on-state-bucket-creation`
+**Environment Variable**: `TERRAGRUNT_FAIL_ON_STATE_BUCKET_CREATION` (set to `true`)
+
+When this flag is set, Terragrunt will wait for execution if it is required to create the remote state bucket.
+
+### terragrunt-disable-bucket-update
+
+**CLI Arg**: `--terragrunt-disable-bucket-update`
+**Environment Variable**: `TERRAGRUNT_DISABLE_BUCKET_UPDATE` (set to `true`)
+
+When this flag is set, Terragrunt does not update the remote state bucket, which is useful to set if the state bucket is managed by a third party.
diff --git a/options/options.go b/options/options.go
index 6386e91ea3..fe5a57a815 100644
--- a/options/options.go
+++ b/options/options.go
@@ -205,6 +205,12 @@ type TerragruntOptions struct {
 
 	// Controls if a module prefix will be prepended to TF outputs
 	IncludeModulePrefix bool
+
+	// Fail execution if is required to create S3 bucket
+	FailIfBucketCreationRequired bool
+
+	// Controls if s3 bucket should be updated or skipped
+	DisableBucketUpdate bool
 }
 
 // IAMOptions represents options that are used by Terragrunt to assume an IAM role.
@@ -385,6 +391,8 @@ func (terragruntOptions *TerragruntOptions) Clone(terragruntConfigPath string) *
 		UsePartialParseConfigCache:     terragruntOptions.UsePartialParseConfigCache,
 		OutputPrefix:                   terragruntOptions.OutputPrefix,
 		IncludeModulePrefix:            terragruntOptions.IncludeModulePrefix,
+		FailIfBucketCreationRequired:   terragruntOptions.FailIfBucketCreationRequired,
+		DisableBucketUpdate:            terragruntOptions.DisableBucketUpdate,
 	}
 }
 
diff --git a/remote/remote_state.go b/remote/remote_state.go
index 81add2c7aa..88375ad192 100644
--- a/remote/remote_state.go
+++ b/remote/remote_state.go
@@ -208,3 +208,9 @@ var (
 	ErrRemoteBackendMissing             = fmt.Errorf("the remote_state.backend field cannot be empty")
 	ErrGenerateCalledWithNoGenerateAttr = fmt.Errorf("generate code routine called when no generate attribute is configured")
 )
+
+type BucketCreationNotAllowed string
+
+func (bucketName BucketCreationNotAllowed) Error() string {
+	return fmt.Sprintf("Creation of remote state bucket %s is not allowed", string(bucketName))
+}
diff --git a/remote/remote_state_gcs.go b/remote/remote_state_gcs.go
index fea740940b..0bb07f1900 100644
--- a/remote/remote_state_gcs.go
+++ b/remote/remote_state_gcs.go
@@ -258,6 +258,10 @@ func createGCSBucketIfNecessary(gcsClient *storage.Client, config *ExtendedRemot
 			return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("location"))
 		}
 
+		if terragruntOptions.FailIfBucketCreationRequired {
+			return BucketCreationNotAllowed(config.remoteStateConfigGCS.Bucket)
+		}
+
 		prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.remoteStateConfigGCS.Bucket)
 		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
 		if err != nil {
diff --git a/remote/remote_state_s3.go b/remote/remote_state_s3.go
index e0a9dd92f9..90974723a2 100644
--- a/remote/remote_state_s3.go
+++ b/remote/remote_state_s3.go
@@ -270,7 +270,7 @@ func (s3Initializer S3Initializer) Initialize(remoteState *RemoteState, terragru
 		return err
 	}
 
-	if !s3ConfigExtended.DisableBucketUpdate {
+	if !terragruntOptions.DisableBucketUpdate && !s3ConfigExtended.DisableBucketUpdate {
 		if err := updateS3BucketIfNecessary(s3Client, s3ConfigExtended, terragruntOptions); err != nil {
 			return err
 		}
@@ -373,6 +373,10 @@ func validateS3Config(extendedConfig *ExtendedRemoteStateConfigS3, terragruntOpt
 // confirms, create the bucket and enable versioning for it.
 func createS3BucketIfNecessary(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
 	if !DoesS3BucketExist(s3Client, &config.remoteStateConfigS3.Bucket) {
+		if terragruntOptions.FailIfBucketCreationRequired {
+			return BucketCreationNotAllowed(config.remoteStateConfigS3.Bucket)
+		}
+
 		prompt := fmt.Sprintf("Remote state S3 bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.remoteStateConfigS3.Bucket)
 		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
 		if err != nil {
@@ -401,6 +405,9 @@ func createS3BucketIfNecessary(s3Client *s3.S3, config *ExtendedRemoteStateConfi
 
 func updateS3BucketIfNecessary(s3Client *s3.S3, config *ExtendedRemoteStateConfigS3, terragruntOptions *options.TerragruntOptions) error {
 	if !DoesS3BucketExist(s3Client, &config.remoteStateConfigS3.Bucket) {
+		if terragruntOptions.FailIfBucketCreationRequired {
+			return BucketCreationNotAllowed(config.remoteStateConfigS3.Bucket)
+		}
 		return errors.WithStackTrace(fmt.Errorf("remote state S3 bucket %s does not exist or you don't have permissions to access it", config.remoteStateConfigS3.Bucket))
 	}
 
@@ -710,6 +717,9 @@ func CreateS3BucketWithVersioningSSEncryptionAndAccessLogging(s3Client *s3.S3, c
 
 func CreateLogsS3BucketIfNecessary(s3Client *s3.S3, logsBucketName *string, terragruntOptions *options.TerragruntOptions) error {
 	if !DoesS3BucketExist(s3Client, logsBucketName) {
+		if terragruntOptions.FailIfBucketCreationRequired {
+			return BucketCreationNotAllowed(*logsBucketName)
+		}
 		prompt := fmt.Sprintf("Logs S3 bucket %s for the remote state does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", *logsBucketName)
 		shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
 		if err != nil {


Test patch:
diff --git a/test/integration_test.go b/test/integration_test.go
index b27d1832d2..e47dc56df9 100644
--- a/test/integration_test.go
+++ b/test/integration_test.go
@@ -4196,6 +4196,32 @@ func assertS3PublicAccessBlocks(t *testing.T, client *s3.S3, bucketName string)
 	assert.True(t, aws.BoolValue(publicAccessBlockConfig.RestrictPublicBuckets))
 }
 
+// createS3BucketE create test S3 bucket.
+func createS3BucketE(t *testing.T, awsRegion string, bucketName string) error {
+	mockOptions, err := options.NewTerragruntOptionsForTest("integration_test")
+	if err != nil {
+		t.Logf("Error creating mockOptions: %v", err)
+		return err
+	}
+
+	sessionConfig := &aws_helper.AwsSessionConfig{
+		Region: awsRegion,
+	}
+
+	s3Client, err := remote.CreateS3Client(sessionConfig, mockOptions)
+	if err != nil {
+		t.Logf("Error creating S3 client: %v", err)
+		return err
+	}
+
+	t.Logf("Creating test s3 bucket %s", bucketName)
+	if _, err := s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+		t.Logf("Failed to create S3 bucket %s: %v", bucketName, err)
+		return err
+	}
+	return nil
+}
+
 // deleteS3BucketWithRetry will attempt to delete the specified S3 bucket, retrying up to 3 times if there are errors to
 // handle eventual consistency issues.
 func deleteS3BucketWithRetry(t *testing.T, awsRegion string, bucketName string) {
@@ -5678,6 +5704,49 @@ func TestRenderJsonWithInputsNotExistingOutput(t *testing.T) {
 	assert.True(t, reflect.DeepEqual(expectedInputs, inputs))
 }
 
+func TestTerragruntFailIfBucketCreationIsRequired(t *testing.T) {
+	t.Parallel()
+
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PATH)
+	rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PATH)
+	cleanupTerraformFolder(t, rootPath)
+
+	s3BucketName := fmt.Sprintf("terragrunt-test-bucket-%s", strings.ToLower(uniqueId()))
+	lockTableName := fmt.Sprintf("terragrunt-test-locks-%s", strings.ToLower(uniqueId()))
+
+	tmpTerragruntConfigPath := createTmpTerragruntConfig(t, rootPath, s3BucketName, lockTableName, config.DefaultTerragruntConfigPath)
+
+	stdout := bytes.Buffer{}
+	stderr := bytes.Buffer{}
+	err := runTerragruntCommand(t, fmt.Sprintf("terragrunt apply --terragrunt-fail-on-state-bucket-creation --terragrunt-non-interactive --terragrunt-config %s --terragrunt-working-dir %s", tmpTerragruntConfigPath, rootPath), &stdout, &stderr)
+	assert.Error(t, err)
+}
+
+func TestTerragruntDisableBucketUpdate(t *testing.T) {
+	t.Parallel()
+
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PATH)
+	rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PATH)
+	cleanupTerraformFolder(t, rootPath)
+
+	s3BucketName := fmt.Sprintf("terragrunt-test-bucket-%s", strings.ToLower(uniqueId()))
+	lockTableName := fmt.Sprintf("terragrunt-test-locks-%s", strings.ToLower(uniqueId()))
+
+	err := createS3BucketE(t, TERRAFORM_REMOTE_STATE_S3_REGION, s3BucketName)
+	assert.NoError(t, err)
+
+	defer deleteS3Bucket(t, TERRAFORM_REMOTE_STATE_S3_REGION, s3BucketName)
+	defer cleanupTableForTest(t, lockTableName, TERRAFORM_REMOTE_STATE_S3_REGION)
+
+	tmpTerragruntConfigPath := createTmpTerragruntConfig(t, rootPath, s3BucketName, lockTableName, config.DefaultTerragruntConfigPath)
+
+	runTerragrunt(t, fmt.Sprintf("terragrunt apply -auto-approve --terragrunt-disable-bucket-update --terragrunt-non-interactive --terragrunt-config %s --terragrunt-working-dir %s", tmpTerragruntConfigPath, rootPath))
+
+	_, err = bucketPolicy(t, TERRAFORM_REMOTE_STATE_S3_REGION, s3BucketName)
+	// validate that bucket policy is not updated, because of --terragrunt-disable-bucket-update
+	assert.Error(t, err)
+}
+
 func validateOutput(t *testing.T, outputs map[string]TerraformOutput, key string, value interface{}) {
 	t.Helper()
 	output, hasPlatform := outputs[key]