Repository: gruntwork-io/terragrunt (language: go)

Issue:
Proposal: Error hooks
The idea behind error hooks is to execute arbitrary shell commands when a terraform command returns an error. 

Example syntax would be similar to `before_hook` and `after_hook` with required properties `commands` and `execute` and instead of `run_on_error` a different property `on_errors` would be required that would describe a list of errors on which should the hook run, described in the same manner as `retryable_errors`.

An example configuration that imports would run a shell script with `terraform import`:
```hcl
terraform {
  error_hook "import_resource" {
    commands  = ["apply"]
    execute   = ["./import_known_resource.sh"]
    on_errors = [
      "(?s).*A resource with the ID .* already exists.*",
    ]
  }
}
```

The idea behind this proposal is that there are some resources on terraform-providers that create an 'umbrella' resource which can create some child-resources on the cloud provider which are not being represented as a terraform resource. In some cases, the default child-resource configuration is ok, but when that resource needs to be modified, it becomes problematic.

One example of this would be an `azurerm` resource `azurerm_api_management` creates 2 `azurerm_api_management_product` instances which have one `azurerm_api_management_product_policy` each. The `product` can be accessed as `data` but the `product_policy` needs to be imported to state in order to be accessible from terraform.


Discussion / hints:
This seems reasonable! We're buried with other initiatives so most likely won't implement this any time soon, but PRs from the community are welcome!
@yorinasub17 @Ben10k anyone working on this? If not I can take this over
Instead of adding new type of hook wouldn't it be better to add `run_only_on_error` to existing hooks and we also add error message and exit code as env vars and let the script that gets executed by the hook decide what to do. Thoughts? 
> @yorinasub17 @Ben10k anyone working on this? If not I can take this over

@smitthakkar96 I did not have a chance to work on this, you can take over.

Although I am not sure if adding `run_only_on_error` instead of adding a new hook type is the way to go, as I believe it would increase both the complexity of terragrunt and the readability of the `terragrunt.hcl` configurations.
@Ben10k thanks for your response, your comment regarding complexity and readability makes sense. I have one more question, what will the ordering be like? Will after_hook with `run_on_error` execute first or `error_hook` execute first?

Fix patch:
diff --git a/cli/cli_app.go b/cli/cli_app.go
index 0a2fe19c41..b3b2e5f7d2 100644
--- a/cli/cli_app.go
+++ b/cli/cli_app.go
@@ -602,6 +602,41 @@ func shouldApplyAwsProviderPatch(terragruntOptions *options.TerragruntOptions) b
 	return util.ListContainsElement(terragruntOptions.TerraformCliArgs, CMD_AWS_PROVIDER_PATCH)
 }
 
+func processErrorHooks(hooks []config.ErrorHook, terragruntOptions *options.TerragruntOptions, previousExecErrors *multierror.Error) error {
+	if len(hooks) == 0 || previousExecErrors.ErrorOrNil() == nil {
+		return nil
+	}
+
+	var errorsOccured *multierror.Error
+
+	terragruntOptions.Logger.Debugf("Detected %d error Hooks", len(hooks))
+
+	for _, curHook := range hooks {
+		if util.MatchesAny(curHook.OnErrors, previousExecErrors.GoString()) && util.ListContainsElement(curHook.Commands, terragruntOptions.TerraformCommand) {
+			terragruntOptions.Logger.Infof("Executing hook: %s", curHook.Name)
+			workingDir := ""
+			if curHook.WorkingDir != nil {
+				workingDir = *curHook.WorkingDir
+			}
+
+			actionToExecute := curHook.Execute[0]
+			actionParams := curHook.Execute[1:]
+			_, possibleError := shell.RunShellCommandWithOutput(
+				terragruntOptions,
+				workingDir,
+				false,
+				false,
+				actionToExecute, actionParams...,
+			)
+			if possibleError != nil {
+				terragruntOptions.Logger.Errorf("Error running hook %s with message: %s", curHook.Name, possibleError.Error())
+				errorsOccured = multierror.Append(errorsOccured, possibleError)
+			}
+		}
+	}
+	return errorsOccured.ErrorOrNil()
+}
+
 func processHooks(hooks []config.Hook, terragruntOptions *options.TerragruntOptions, previousExecErrors *multierror.Error) error {
 	if len(hooks) == 0 {
 		return nil
@@ -790,9 +825,9 @@ func runActionWithHooks(description string, terragruntOptions *options.Terragrun
 	} else {
 		terragruntOptions.Logger.Errorf("Errors encountered running before_hooks. Not running '%s'.", description)
 	}
-
 	postHookErrors := processHooks(terragruntConfig.Terraform.GetAfterHooks(), terragruntOptions, allErrors)
-	allErrors = multierror.Append(allErrors, postHookErrors)
+	errorHookErrors := processErrorHooks(terragruntConfig.Terraform.GetErrorHooks(), terragruntOptions, allErrors)
+	allErrors = multierror.Append(allErrors, postHookErrors, errorHookErrors)
 
 	return allErrors.ErrorOrNil()
 }
diff --git a/config/config.go b/config/config.go
index 51ba386a86..4f21d7f62d 100644
--- a/config/config.go
+++ b/config/config.go
@@ -291,9 +291,20 @@ type Hook struct {
 	WorkingDir *string  `hcl:"working_dir,attr" cty:"working_dir"`
 }
 
+type ErrorHook struct {
+	Name       string   `hcl:"name,label" cty:"name"`
+	Commands   []string `hcl:"commands,attr" cty:"commands"`
+	Execute    []string `hcl:"execute,attr" cty:"execute"`
+	OnErrors   []string `hcl:"on_errors,attr" cty:"on_errors"`
+	WorkingDir *string  `hcl:"working_dir,attr" cty:"working_dir"`
+}
+
 func (conf *Hook) String() string {
 	return fmt.Sprintf("Hook{Name = %s, Commands = %v}", conf.Name, len(conf.Commands))
 }
+func (conf *ErrorHook) String() string {
+	return fmt.Sprintf("Hook{Name = %s, Commands = %v}", conf.Name, len(conf.Commands))
+}
 
 // TerraformConfig specifies where to find the Terraform configuration files
 // NOTE: If any attributes or blocks are added here, be sure to add it to ctyTerraformConfig in config_as_cty.go as
@@ -303,6 +314,7 @@ type TerraformConfig struct {
 	Source      *string                   `hcl:"source,attr"`
 	BeforeHooks []Hook                    `hcl:"before_hook,block"`
 	AfterHooks  []Hook                    `hcl:"after_hook,block"`
+	ErrorHooks  []ErrorHook               `hcl:"error_hook,block"`
 
 	// Ideally we can avoid the pointer to list slice, but if it is not a pointer, Terraform requires the attribute to
 	// be defined and we want to make this optional.
@@ -329,10 +341,24 @@ func (conf *TerraformConfig) GetAfterHooks() []Hook {
 	return conf.AfterHooks
 }
 
+func (conf *TerraformConfig) GetErrorHooks() []ErrorHook {
+	if conf == nil {
+		return nil
+	}
+
+	return conf.ErrorHooks
+}
+
 func (conf *TerraformConfig) ValidateHooks() error {
-	allHooks := append(conf.GetBeforeHooks(), conf.GetAfterHooks()...)
+	beforeAndAfterHooks := append(conf.GetBeforeHooks(), conf.GetAfterHooks()...)
+
+	for _, curHook := range beforeAndAfterHooks {
+		if len(curHook.Execute) < 1 || curHook.Execute[0] == "" {
+			return InvalidArgError(fmt.Sprintf("Error with hook %s. Need at least one non-empty argument in 'execute'.", curHook.Name))
+		}
+	}
 
-	for _, curHook := range allHooks {
+	for _, curHook := range conf.GetErrorHooks() {
 		if len(curHook.Execute) < 1 || curHook.Execute[0] == "" {
 			return InvalidArgError(fmt.Sprintf("Error with hook %s. Need at least one non-empty argument in 'execute'.", curHook.Name))
 		}
diff --git a/config/config_as_cty.go b/config/config_as_cty.go
index 0dfb9e610c..25cdf248d0 100644
--- a/config/config_as_cty.go
+++ b/config/config_as_cty.go
@@ -132,6 +132,7 @@ type ctyTerraformConfig struct {
 	IncludeInCopy *[]string                          `cty:"include_in_copy"`
 	BeforeHooks   map[string]Hook                    `cty:"before_hook"`
 	AfterHooks    map[string]Hook                    `cty:"after_hook"`
+	ErrorHooks    map[string]ErrorHook               `cty:"error_hook"`
 }
 
 // Serialize TerraformConfig to a cty Value, but with maps instead of lists for the blocks.
@@ -146,6 +147,7 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) {
 		ExtraArgs:     map[string]TerraformExtraArguments{},
 		BeforeHooks:   map[string]Hook{},
 		AfterHooks:    map[string]Hook{},
+		ErrorHooks:    map[string]ErrorHook{},
 	}
 
 	for _, arg := range config.ExtraArgs {
@@ -157,6 +159,9 @@ func terraformConfigAsCty(config *TerraformConfig) (cty.Value, error) {
 	for _, hook := range config.AfterHooks {
 		configCty.AfterHooks[hook.Name] = hook
 	}
+	for _, errorHook := range config.ErrorHooks {
+		configCty.ErrorHooks[errorHook.Name] = errorHook
+	}
 
 	return goTypeToCty(configCty)
 }
diff --git a/docs/_docs/02_features/before-and-after-hooks.md b/docs/_docs/02_features/hooks.md
similarity index 76%
rename from docs/_docs/02_features/before-and-after-hooks.md
rename to docs/_docs/02_features/hooks.md
index 4a2dc0a082..544cec243b 100644
--- a/docs/_docs/02_features/before-and-after-hooks.md
+++ b/docs/_docs/02_features/hooks.md
@@ -1,9 +1,9 @@
 ---
 layout: collection-browser-doc
-title: Before and After Hooks
+title: Before, After, and Error Hooks
 category: features
 categories_url: features
-excerpt: Learn how to execute custom code before or after running Terraform.
+excerpt: Learn how to execute custom code before or after running Terraform, or when errors occur.
 tags: ["hooks"]
 order: 240
 nav_title: Documentation
@@ -58,3 +58,19 @@ to Terraform.
 
 You can learn more about all the various configuration options supported in [the reference docs for the terraform
 block](/docs/reference/config-blocks-and-attributes/#terraform).
+
+## Error Hooks
+*Error hooks* are a special type of after hook that act as exception handlers. They allow you to specify a list of expressions that can be used to catch errors and run custom commands when those errors occur. Error hooks are executed after the before/after hooks.
+
+Here is an example:
+``` hcl
+terraform {
+  error_hook "import_resource" {
+    commands  = ["apply"]
+    execute   = ["echo", "Error Hook executed"]
+    on_errors = [
+      ".*",
+    ]
+  }
+}
+```
diff --git a/docs/_docs/04_reference/config-blocks-and-attributes.md b/docs/_docs/04_reference/config-blocks-and-attributes.md
index 848442bf56..c91e5944da 100644
--- a/docs/_docs/04_reference/config-blocks-and-attributes.md
+++ b/docs/_docs/04_reference/config-blocks-and-attributes.md
@@ -104,6 +104,8 @@ The `terraform` block supports the following arguments:
 - `after_hook` (block): Nested blocks used to specify command hooks that should be run after `terraform` is called.
   Hooks run from the terragrunt configuration directory (the directory where `terragrunt.hcl` lives). Supports the same
   arguments as `before_hook`.
+- `error_hook` (block): Nested blocks used to specify command hooks that run when an error is thrown. The
+error must match one of the expressions listed in the `on_errors` attribute. Error hooks are executed after the before/after hooks.
 
 In addition to supporting before and after hooks for all terraform commands, the following specialized hooks are also
 supported:
@@ -190,6 +192,16 @@ terraform {
     run_on_error = true
   }
 
+  # After an error occurs during apply or plan, run "echo Error Hook executed". This hook is configured so that it will run
+  # after any error, with the ".*" expression.
+  error_hook "error_hook_1" {
+    commands  = ["apply", "plan"]
+    execute   = ["echo", "Error Hook executed"]
+    on_errors = [
+      ".*",
+    ]
+  }
+  
   # A special after hook to always run after the init-from-module step of the Terragrunt pipeline. In this case, we will
   # copy the "foo.tf" file located by the parent terragrunt.hcl file to the current working directory.
   after_hook "init_from_module" {


Test patch:
diff --git a/config/config_as_cty_test.go b/config/config_as_cty_test.go
index ff48403336..e712fb6a05 100644
--- a/config/config_as_cty_test.go
+++ b/config/config_as_cty_test.go
@@ -44,6 +44,14 @@ func TestTerragruntConfigAsCtyDrift(t *testing.T) {
 					Execute:  []string{"true"},
 				},
 			},
+			ErrorHooks: []ErrorHook{
+				ErrorHook{
+					Name:     "init",
+					Commands: []string{"init"},
+					Execute:  []string{"true"},
+					OnErrors: []string{".*"},
+				},
+			},
 		},
 		TerraformBinary:             "terraform",
 		TerraformVersionConstraint:  "= 0.12.20",
diff --git a/config/config_helpers_test.go b/config/config_helpers_test.go
index 3ee112f310..022b88783e 100644
--- a/config/config_helpers_test.go
+++ b/config/config_helpers_test.go
@@ -883,7 +883,7 @@ func TestReadTerragruntConfigHooks(t *testing.T) {
 	t.Parallel()
 
 	options := terragruntOptionsForTest(t, DefaultTerragruntConfigPath)
-	tgConfigCty, err := readTerragruntConfig("../test/fixture-hooks/before-and-after/terragrunt.hcl", nil, options)
+	tgConfigCty, err := readTerragruntConfig("../test/fixture-hooks/before-after-and-on-error/terragrunt.hcl", nil, options)
 	require.NoError(t, err)
 
 	tgConfigMap, err := parseCtyValueToMap(tgConfigCty)
@@ -913,6 +913,12 @@ func TestReadTerragruntConfigHooks(t *testing.T) {
 		afterHooksMap["after_hook_2"].(map[string]interface{})["execute"].([]interface{}),
 		[]interface{}{"echo", "AFTER_TERRAGRUNT_READ_CONFIG"},
 	)
+	errorHooksMap := terraformMap["error_hook"].(map[string]interface{})
+	assert.Equal(
+		t,
+		errorHooksMap["error_hook_1"].(map[string]interface{})["execute"].([]interface{}),
+		[]interface{}{"echo", "ON_APPLY_ERROR"},
+	)
 }
 
 func TestReadTerragruntConfigLocals(t *testing.T) {
diff --git a/test/fixture-hooks/before-after-and-on-error/main.tf b/test/fixture-hooks/before-after-and-on-error/main.tf
new file mode 100644
index 0000000000..15d67cca89
--- /dev/null
+++ b/test/fixture-hooks/before-after-and-on-error/main.tf
@@ -0,0 +1,7 @@
+data "template_file" "example" {
+  template = "hello, world"
+}
+
+output "example" {
+  value = data.template_file.example.rendered
+}
diff --git a/test/fixture-hooks/before-after-and-on-error/terragrunt.hcl b/test/fixture-hooks/before-after-and-on-error/terragrunt.hcl
new file mode 100644
index 0000000000..49a8f5510b
--- /dev/null
+++ b/test/fixture-hooks/before-after-and-on-error/terragrunt.hcl
@@ -0,0 +1,35 @@
+terraform {
+  # This hook configures Terragrunt to create an empty file called before.out
+  # before execution of terragrunt
+  before_hook "before_hook_1" {
+    commands = ["apply", "plan"]
+    execute = ["touch","before.out"]
+    run_on_error = true
+  }
+
+  # This hook configures Terragrunt to create an empty file called after.out
+  # after execution of terragrunt
+  after_hook "after_hook_1" {
+    commands = ["apply", "plan"]
+    execute = ["touch","after.out"]
+    run_on_error = true
+  }
+
+  before_hook "before_hook_2" {
+    commands = ["terragrunt-read-config"]
+    execute = ["echo", "BEFORE_TERRAGRUNT_READ_CONFIG"]
+    run_on_error = true
+  }
+
+  after_hook "after_hook_2" {
+    commands = ["terragrunt-read-config"]
+    execute = ["echo", "AFTER_TERRAGRUNT_READ_CONFIG"]
+    run_on_error = true
+  }
+
+  error_hook "error_hook_1" {
+    commands  = ["apply"]
+    execute   = ["echo", "ON_APPLY_ERROR"]
+    on_errors = [".*"]
+  }
+}
diff --git a/test/fixture-hooks/skip-on-error/terragrunt.hcl b/test/fixture-hooks/skip-on-error/terragrunt.hcl
index fd973b0729..ba20cf79c2 100644
--- a/test/fixture-hooks/skip-on-error/terragrunt.hcl
+++ b/test/fixture-hooks/skip-on-error/terragrunt.hcl
@@ -29,5 +29,9 @@ terraform {
     execute = ["echo","AFTER_SHOULD_DISPLAY"]
     run_on_error = true
   }
-
+  error_hook "error_hook" {
+    commands  = ["apply", "plan"]
+    execute   = ["echo", "ERROR_HOOK_EXECUTED"]
+    on_errors = [".*"]
+  }
 }
diff --git a/test/integration_test.go b/test/integration_test.go
index f74a0d001b..d9966acc64 100644
--- a/test/integration_test.go
+++ b/test/integration_test.go
@@ -395,6 +395,9 @@ func TestTerragruntSkipOnError(t *testing.T) {
 
 		assert.Contains(t, output, "AFTER_SHOULD_DISPLAY")
 		assert.NotContains(t, output, "AFTER_NODISPLAY")
+
+		assert.Contains(t, output, "ERROR_HOOK_EXECUTED")
+
 	} else {
 		t.Error("Expected NO terragrunt execution due to previous errors but it did run.")
 	}