Repository: gruntwork-io/terragrunt (go)

Issue:
Experiment `runner-pool`: `run --all` evaluates `terragrunt.stack.hcl` as unit config
## Describe the bug

Starting with release 0.81.6, running `run --all -- apply` seems to be evaluating (nested) stack files as regular terragrunt.hcl files. Reverting to 0.81.5 solves this problem.

## Output

```
terragrunt --non-interactive --queue-include-external run --all -- apply
```

```
11:22:23.651 INFO   Using RunnerPoolStackBuilder to build stack for .
11:22:23.655 ERROR  Error: Unsupported block type
11:22:23.655 ERROR    on ./terragrunt.stack.hcl line 5, in unit "id":
11:22:23.655 ERROR     5: unit "id" {
11:22:23.655 ERROR  Blocks of type "unit" are not expected here.
11:22:23.655 ERROR  Error: Unsupported block type
11:22:23.657 ERROR    on ./terragrunt.stack.hcl line 14, in unit "ecr-cache":
11:22:23.657 ERROR    14: unit "ecr-cache" {
11:22:23.657 ERROR  Blocks of type "unit" are not expected here.
11:22:23.657 ERROR  Error: Unsupported block type
11:22:23.657 ERROR    on ./terragrunt.stack.hcl line 45, in stack "network":
11:22:23.657 ERROR    45: stack "network" {
11:22:23.658 ERROR  Blocks of type "stack" are not expected here.
11:22:23.658 ERROR  Error: Unsupported block type
11:22:23.658 ERROR    on ./terragrunt.stack.hcl line 71, in stack "k8s":
11:22:23.658 ERROR    71: stack "k8s" {
11:22:23.658 ERROR  Blocks of type "stack" are not expected here.
11:22:23.658 INFO   The runner-pool stack at . will be processed in the following order for command apply:
```



Pull request description:
Configurable discovery files
<!-- Prepend '[WIP]' to the title if this PR is still a work-in-progress. Remove it when it is ready for review! -->

## Description

* Add configurable files for discovery
* Improved runner pool discovery to detect only Terragrunt config files
* Added test to track that discovery is not reporting errors

Before:

![image](https://github.com/user-attachments/assets/6cb480df-9cc3-497a-9d92-56d94394606c)

With this change:

![image](https://github.com/user-attachments/assets/1943f65f-6744-4814-9cda-d7f12e7a4c87)

Fixes #4450.

<!-- Description of the changes introduced by this PR. -->

## TODOs

Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Gruntwork-Coding-Methodology-02fdcd6e4b004e818553684760bf691e).

- [x] I authored this code entirely myself
- [x] I am submitting code based on open source software (e.g. MIT, MPL-2.0, Apache)]
- [x] I am adding or upgrading a dependency or adapted code and confirm it has a compatible open source license
- [x] Update the docs.
- [x] Run the relevant tests successfully, including pre-commit checks.
- [x] Include release notes. If this PR is backward incompatible, include a migration guide.

## Release Notes (draft)

<!-- One-line description of the PR that can be included in the final release notes. -->
Added / Removed / Updated [X].

### Migration Guide

<!-- Important: If you made any backward incompatible changes, then you must write a migration guide! -->



<!-- This is an auto-generated comment: release notes by coderabbit.ai -->

## Summary by CodeRabbit

* **New Features**
  * Enhanced configuration discovery to support customizable sets of configuration filenames, allowing greater flexibility beyond the default options.

* **Bug Fixes**
  * Improved handling of mixed configuration files to prevent errors related to unsupported block types or unexpected unit blocks.

* **Tests**
  * Added new unit and integration tests to verify correct discovery and handling of custom and mixed configuration files.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Patch:
diff --git a/internal/discovery/discovery.go b/internal/discovery/discovery.go
index 3ffab209..d6c1ed08 100644
--- a/internal/discovery/discovery.go
+++ b/internal/discovery/discovery.go
@@ -75,6 +75,9 @@ type Discovery struct {
 	// hiddenDirMemo is a memoization of hidden directories.
 	hiddenDirMemo []string
 
+	// configFilenames is the list of config filenames to discover. If nil, defaults are used.
+	configFilenames []string
+
 	// maxDependencyDepth is the maximum depth of the dependency tree to discover.
 	maxDependencyDepth int
 
@@ -103,6 +106,9 @@ type Discovery struct {
 // DiscoveryOption is a function that modifies a Discovery.
 type DiscoveryOption func(*Discovery)
 
+// DefaultConfigFilenames are the default Terragrunt config filenames used in discovery.
+var DefaultConfigFilenames = []string{config.DefaultTerragruntConfigPath, config.DefaultStackFile}
+
 // NewDiscovery creates a new Discovery.
 func NewDiscovery(dir string, opts ...DiscoveryOption) *Discovery {
 	discovery := &Discovery{
@@ -190,6 +196,12 @@ func (d *Discovery) WithDiscoveryContext(discoveryContext *DiscoveryContext) *Di
 	return d
 }
 
+// WithConfigFilenames sets the configFilenames field to the given list.
+func (d *Discovery) WithConfigFilenames(filenames []string) *Discovery {
+	d.configFilenames = filenames
+	return d
+}
+
 // String returns a string representation of a DiscoveredConfig.
 func (c *DiscoveredConfig) String() string {
 	return c.Path
@@ -281,6 +293,12 @@ func (d *Discovery) isInHiddenDirectory(path string) bool {
 func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.TerragruntOptions) (DiscoveredConfigs, error) {
 	var cfgs DiscoveredConfigs
 
+	// Set default config filenames if not set
+	filenames := d.configFilenames
+	if len(filenames) == 0 {
+		filenames = DefaultConfigFilenames
+	}
+
 	processFn := func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return errors.New(err)
@@ -294,29 +312,26 @@ func (d *Discovery) Discover(ctx context.Context, l log.Logger, opts *options.Te
 			return nil
 		}
 
-		switch filepath.Base(path) {
-		case config.DefaultTerragruntConfigPath:
-			cfg := &DiscoveredConfig{
-				Type: ConfigTypeUnit,
-				Path: filepath.Dir(path),
-			}
+		base := filepath.Base(path)
+		for _, fname := range filenames {
+			if base == fname {
+				cfgType := ConfigTypeUnit
+				if fname == config.DefaultStackFile {
+					cfgType = ConfigTypeStack
+				}
 
-			if d.discoveryContext != nil {
-				cfg.DiscoveryContext = d.discoveryContext
-			}
+				cfg := &DiscoveredConfig{
+					Type: cfgType,
+					Path: filepath.Dir(path),
+				}
+				if d.discoveryContext != nil {
+					cfg.DiscoveryContext = d.discoveryContext
+				}
 
-			cfgs = append(cfgs, cfg)
-		case config.DefaultStackFile:
-			cfg := &DiscoveredConfig{
-				Type: ConfigTypeStack,
-				Path: filepath.Dir(path),
-			}
+				cfgs = append(cfgs, cfg)
 
-			if d.discoveryContext != nil {
-				cfg.DiscoveryContext = d.discoveryContext
+				break
 			}
-
-			cfgs = append(cfgs, cfg)
 		}
 
 		return nil
diff --git a/internal/runner/runnerpool/builder.go b/internal/runner/runnerpool/builder.go
index 3451b0e5..977b6dc6 100644
--- a/internal/runner/runnerpool/builder.go
+++ b/internal/runner/runnerpool/builder.go
@@ -3,6 +3,7 @@ package runnerpool
 import (
 	"context"
 
+	"github.com/gruntwork-io/terragrunt/config"
 	"github.com/gruntwork-io/terragrunt/internal/discovery"
 	"github.com/gruntwork-io/terragrunt/internal/runner/common"
 	"github.com/gruntwork-io/terragrunt/options"
@@ -19,6 +20,7 @@ func Build(ctx context.Context, l log.Logger, terragruntOptions *options.Terragr
 		WithParseExclude().
 		WithDiscoverDependencies().
 		WithSuppressParseErrors().
+		WithConfigFilenames([]string{config.DefaultTerragruntConfigPath}).
 		WithDiscoveryContext(&discovery.DiscoveryContext{Cmd: terragruntOptions.TerraformCommand})
 
 	discovered, err := d.Discover(ctx, l, terragruntOptions)
diff --git a/tf/getproviders/lock.go b/tf/getproviders/lock.go
index 50ab803b..9302f9fe 100644
--- a/tf/getproviders/lock.go
+++ b/tf/getproviders/lock.go
@@ -16,6 +16,7 @@ import (
 	"github.com/gruntwork-io/terragrunt/internal/errors"
 	"github.com/gruntwork-io/terragrunt/tf"
 	"github.com/gruntwork-io/terragrunt/util"
+	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/hashicorp/hcl/v2/hclwrite"
@@ -90,8 +91,32 @@ func updateProviderBlock(ctx context.Context, providerBlock *hclwrite.Block, pro
 
 	providerBlock.Body().SetAttributeValue("version", cty.StringVal(provider.Version()))
 
-	// Constraints can contain multiple constraint expressions, including comparison operators, but in the Terragrunt Provider Cache use case, we assume that the required_providers are pinned to a specific version to detect the required version without terraform init, so we can simply specify the constraints attribute as the same as the version. This may differ from what terraform generates, but we expect that it doesn't matter in practice.
-	providerBlock.Body().SetAttributeValue("constraints", cty.StringVal(provider.Version()))
+	// If version constraints exist in current lock file and match the new version, we keep them unchanged.
+	// Otherwise, we specify the constraints attribute the same as the version.
+	currentConstraintsAttr := providerBlock.Body().GetAttribute("constraints")
+
+	shouldUpdateConstraints := false
+
+	if currentConstraintsAttr != nil {
+		currentConstraints, err := version.NewConstraint(strings.ReplaceAll(string(currentConstraintsAttr.Expr().BuildTokens(nil).Bytes()), `"`, ""))
+		// If current version constraints are malformed, we should update it.
+		if err != nil {
+			shouldUpdateConstraints = true
+		}
+
+		newVersion, _ := version.NewVersion(provider.Version())
+		// If current version constrains do not match the new provider version, we should update it.
+		if !currentConstraints.Check(newVersion) {
+			shouldUpdateConstraints = true
+		}
+	} else {
+		// If there is no constraints attribute, we should update it.
+		shouldUpdateConstraints = true
+	}
+
+	if shouldUpdateConstraints {
+		providerBlock.Body().SetAttributeValue("constraints", cty.StringVal(provider.Version()))
+	}
 
 	h1Hash, err := PackageHashV1(provider.PackageDir())
 	if err != nil {


Test patch:
diff --git a/internal/discovery/discovery_test.go b/internal/discovery/discovery_test.go
index ee50c6df..f6e1ce10 100644
--- a/internal/discovery/discovery_test.go
+++ b/internal/discovery/discovery_test.go
@@ -483,3 +483,24 @@ exclude {
 		assert.Nil(t, unit3.Parsed.Exclude)
 	}
 }
+
+func TestDiscoveryWithSingleCustomConfigFilename(t *testing.T) {
+	t.Parallel()
+
+	tmpDir := t.TempDir()
+	unit1Dir := filepath.Join(tmpDir, "unit1")
+	err := os.MkdirAll(unit1Dir, 0755)
+	require.NoError(t, err)
+	err = os.WriteFile(filepath.Join(unit1Dir, "custom1.hcl"), []byte(""), 0644)
+	require.NoError(t, err)
+
+	discoveryObj := discovery.NewDiscovery(tmpDir).WithConfigFilenames([]string{"custom1.hcl"})
+	opts, err := options.NewTerragruntOptionsForTest(tmpDir)
+	require.NoError(t, err)
+
+	configs, err := discoveryObj.Discover(t.Context(), logger.CreateLogger(), opts)
+	require.NoError(t, err)
+
+	units := configs.Filter(discovery.ConfigTypeUnit).Paths()
+	assert.ElementsMatch(t, []string{unit1Dir}, units)
+}
diff --git a/test/fixtures/mixed-config/app/main.tf b/test/fixtures/mixed-config/app/main.tf
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/test/fixtures/mixed-config/app/main.tf
@@ -0,0 +1,1 @@
+
diff --git a/test/fixtures/mixed-config/app/terragrunt.hcl b/test/fixtures/mixed-config/app/terragrunt.hcl
new file mode 100644
index 00000000..e69de29b
diff --git a/test/fixtures/mixed-config/stack/terragrunt.stack.hcl b/test/fixtures/mixed-config/stack/terragrunt.stack.hcl
new file mode 100644
index 00000000..a202a40b
--- /dev/null
+++ b/test/fixtures/mixed-config/stack/terragrunt.stack.hcl
@@ -0,0 +1,5 @@
+
+unit "app1" {
+  source = "../unit"
+  path   = "app1"
+}
diff --git a/test/fixtures/mixed-config/unit/main.tf b/test/fixtures/mixed-config/unit/main.tf
new file mode 100644
index 00000000..e69de29b
diff --git a/test/fixtures/mixed-config/unit/terragrunt.hcl b/test/fixtures/mixed-config/unit/terragrunt.hcl
new file mode 100644
index 00000000..e69de29b
diff --git a/test/integration_runner_pool_test.go b/test/integration_runner_pool_test.go
index c8346b85..f63dd402 100644
--- a/test/integration_runner_pool_test.go
+++ b/test/integration_runner_pool_test.go
@@ -13,6 +13,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const (
+	testFixtureMixedConfig = "fixtures/mixed-config"
+)
+
 func TestRunnerPoolDiscovery(t *testing.T) {
 	t.Parallel()
 
@@ -82,3 +86,16 @@ func TestRunnerPoolTerragruntDestroyOrder(t *testing.T) {
 	// module-e must be destroyed before module-d
 	assert.Less(t, index["module-e"], index["module-d"], "module-e should be destroyed before module-d")
 }
+
+func TestRunnerPoolStackConfigIgnored(t *testing.T) {
+	t.Parallel()
+
+	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureMixedConfig)
+	helpers.CleanupTerraformFolder(t, tmpEnvPath)
+	testPath := util.JoinPath(tmpEnvPath, testFixtureMixedConfig)
+
+	_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run --experiment runner-pool --queue-include-external --all --non-interactive --working-dir "+testPath+" -- apply")
+	require.NoError(t, err)
+	require.NotContains(t, stderr, "Error: Unsupported block type")
+	require.NotContains(t, stderr, "Blocks of type \"unit\" are not expected here")
+}
diff --git a/test/integration_test.go b/test/integration_test.go
index 433a3eb0..158fe6cd 100644
--- a/test/integration_test.go
+++ b/test/integration_test.go
@@ -4258,3 +4258,16 @@ func TestVersionIsInvokedInDifferentDirectory(t *testing.T) {
 	assert.Len(t, matches, 2, "Expected exactly one occurrence of '-version' command, found %d", len(matches))
 	assert.Contains(t, stderr, "prefix=dependency-with-custom-version msg=Running command: "+wrappedBinary()+" -version")
 }
+
+func TestMixedStackConfigIgnored(t *testing.T) {
+	t.Parallel()
+
+	tmpEnvPath := helpers.CopyEnvironment(t, testFixtureMixedConfig)
+	helpers.CleanupTerraformFolder(t, tmpEnvPath)
+	testPath := util.JoinPath(tmpEnvPath, testFixtureMixedConfig)
+
+	_, stderr, err := helpers.RunTerragruntCommandWithOutput(t, "terragrunt run --all --non-interactive --working-dir "+testPath+" -- apply")
+	require.NoError(t, err)
+	require.NotContains(t, stderr, "Error: Unsupported block type")
+	require.NotContains(t, stderr, "Blocks of type \"unit\" are not expected here")
+}
diff --git a/tf/getproviders/lock_test.go b/tf/getproviders/lock_test.go
index dee705dc..372f59fc 100644
--- a/tf/getproviders/lock_test.go
+++ b/tf/getproviders/lock_test.go
@@ -117,6 +117,51 @@ provider "registry.terraform.io/hashicorp/azurerm" {
   ]
 }
 
+provider "registry.terraform.io/hashicorp/template" {
+  version     = "2.2.0"
+  constraints = "2.2.0"
+  hashes = [
+    "h1:kvJsWhTmFya0WW8jAfY40fDtYhWQ6mOwPQC2ncDNjZs=",
+    "zh:02d170f0a0f453155686baf35c10b5a7a230ef20ca49f6e26de1c2691ac70a59",
+    "zh:d88ec10849d5a1d9d1db458847bbc62049f0282a2139e5176d645b75a0346992",
+  ]
+}
+`,
+		},
+		{
+			providers: []getproviders.Provider{},
+			initialLockfile: `
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.36.0"
+  constraints = ">= 5.36.0"
+  hashes = [
+    "h1:SHOEBOHEif46z7Bb86YZ5evCtAeK5A4gtHdT8RU5OhA=",
+    "zh:7c810fb11d8b3ded0cb554a27c27a9d002cc644a7a57c29cae01eeea890f0398",
+    "zh:a3366f6b57b0f4b8bf8a5fecf42a834652709a97dd6db1b499c4ab186e33a41f",
+  ]
+}
+			
+provider "registry.terraform.io/hashicorp/template" {
+  version     = "2.1.0"
+  constraints = "<= 2.1.0" 
+  hashes = [
+    "h1:vxE/PD8SWl6Lmh5zRvIW1Y559xfUyuV2T/VeQLXi7f0=",
+    "zh:6fc271665ac28c3fee773b9dc2b8066280ba35b7e9a14a6148194a240c43f42a",
+    "zh:c19f719c9f7ce6d7449fe9c020100faed0705303c7f95beeef81dfd1e4a2004b",
+  ]
+}			
+`,
+			expectedLockfile: `
+provider "registry.terraform.io/hashicorp/aws" {
+  version     = "5.37.0"
+  constraints = ">= 5.36.0"
+  hashes = [
+    "h1:SHOEBOHEif46z7Bb86YZ5evCtAeK5A4gtHdT8RU5OhA=",
+    "zh:7c810fb11d8b3ded0cb554a27c27a9d002cc644a7a57c29cae01eeea890f0398",
+    "zh:a3366f6b57b0f4b8bf8a5fecf42a834652709a97dd6db1b499c4ab186e33a41f",
+  ]
+}
+
 provider "registry.terraform.io/hashicorp/template" {
   version     = "2.2.0"
   constraints = "2.2.0"
@@ -147,6 +192,11 @@ provider "registry.terraform.io/hashicorp/template" {
 					mockProviderUpdateLock(t, ctrl, "registry.terraform.io/hashicorp/aws", "5.36.0"),
 					mockProviderUpdateLock(t, ctrl, "registry.terraform.io/hashicorp/template", "2.2.0"),
 				}
+			case 2:
+				tc.providers = []getproviders.Provider{
+					mockProviderUpdateLock(t, ctrl, "registry.terraform.io/hashicorp/aws", "5.37.0"),
+					mockProviderUpdateLock(t, ctrl, "registry.terraform.io/hashicorp/template", "2.2.0"),
+				}
 			}
 
 			workingDir := t.TempDir()