Repository: gruntwork-io/terragrunt (language: go)

Issue:
Using provider caching with existing `.terraformrc` having `provider_installation` block defined.
@levkohimins

Since the [original issue](https://github.com/gruntwork-io/terragrunt/issues/3278) is now closed I created this one for visibility.

Unfortunately, v0.63.3 didn't fix our issue. Still seems like `network_mirror` is not respected. The error says that it's trying to call default terraform registry to install providers instead of pulling it from jFrog as configured in our .terraform.rc file.

![image](https://github.com/user-attachments/assets/822c69fb-e3e3-4509-b185-3b5a11a83b13)


Discussion / hints:

Fix patch:
diff --git a/cli/provider_cache.go b/cli/provider_cache.go
index 7b2d06f95e..58b876cfd4 100644
--- a/cli/provider_cache.go
+++ b/cli/provider_cache.go
@@ -105,19 +105,17 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
 	for _, registryName := range opts.ProviderCacheRegistryNames {
 		excludeAddrs = append(excludeAddrs, fmt.Sprintf("%s/*/*", registryName))
 	}
+
 	for _, method := range cliCfg.ProviderInstallation.Methods {
 		switch method := method.(type) {
 		case *cliconfig.ProviderInstallationFilesystemMirror:
 			providerHandlers = append(providerHandlers, handlers.NewProviderFilesystemMirrorHandler(providerService, cacheProviderHTTPStatusCode, method))
-			method.AppendExclude(excludeAddrs)
 		case *cliconfig.ProviderInstallationNetworkMirror:
 			providerHandlers = append(providerHandlers, handlers.NewProviderNetworkMirrorHandler(providerService, cacheProviderHTTPStatusCode, method))
-			method.AppendExclude(excludeAddrs)
 		case *cliconfig.ProviderInstallationDirect:
 			providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, method))
-			method.RemoveExclude(excludeAddrs)
-			continue
 		}
+		method.AppendExclude(excludeAddrs)
 	}
 	providerHandlers = append(providerHandlers, handlers.NewProviderDirectHandler(providerService, cacheProviderHTTPStatusCode, new(cliconfig.ProviderInstallationDirect)))
 
@@ -181,7 +179,6 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
 	if err != nil {
 		return nil, err
 	}
-
 	if err := getproviders.UpdateLockfile(ctx, opts.WorkingDir, caches); err != nil {
 		return nil, err
 	}
@@ -250,6 +247,8 @@ func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions
 		cfg.AddProviderInstallationMethods(
 			cliconfig.NewProviderInstallationFilesystemMirror(opts.ProviderCacheDir, providerInstallationIncludes, nil),
 		)
+	} else {
+		cfg.ProviderInstallation = nil
 	}
 
 	cfg.AddProviderInstallationMethods(
diff --git a/terraform/cliconfig/config.go b/terraform/cliconfig/config.go
index 2855c99103..7c7064bd7b 100644
--- a/terraform/cliconfig/config.go
+++ b/terraform/cliconfig/config.go
@@ -80,7 +80,7 @@ func (cfg *Config) AddHost(name string, services map[string]string) {
 	})
 }
 
-// AddProviderInstallationMethods adds installation methods, https://developer.hashicorp.com/terraform/cli/config/config-file#provider-installation
+// AddProviderInstallationMethods merges new installation methods with the current ones, https://developer.hashicorp.com/terraform/cli/config/config-file#provider-installation
 //
 //	provider_installation {
 //		filesystem_mirror {
@@ -91,6 +91,7 @@ func (cfg *Config) AddHost(name string, services map[string]string) {
 //			exclude = ["example.com/*/*"]
 //		}
 //	}
+
 func (cfg *Config) AddProviderInstallationMethods(newMethods ...ProviderInstallationMethod) {
 	if cfg.ProviderInstallation == nil {
 		cfg.ProviderInstallation = &ProviderInstallation{}
diff --git a/terraform/cliconfig/provider_installation.go b/terraform/cliconfig/provider_installation.go
index 2703cb3792..eb40405695 100644
--- a/terraform/cliconfig/provider_installation.go
+++ b/terraform/cliconfig/provider_installation.go
@@ -52,6 +52,7 @@ type ProviderInstallationMethod interface {
 	fmt.Stringer
 	AppendInclude(addrs []string)
 	AppendExclude(addrs []string)
+	RemoveInclude(addrs []string)
 	RemoveExclude(addrs []string)
 	Merge(with ProviderInstallationMethod) bool
 }
@@ -123,6 +124,17 @@ func (method *ProviderInstallationDirect) RemoveExclude(addrs []string) {
 	}
 }
 
+func (method *ProviderInstallationDirect) RemoveInclude(addrs []string) {
+	if len(addrs) == 0 || method.Include == nil {
+		return
+	}
+	*method.Include = util.RemoveSublistFromList(*method.Include, addrs)
+
+	if len(*method.Include) == 0 {
+		method.Include = nil
+	}
+}
+
 func (method *ProviderInstallationDirect) String() string {
 	b, _ := json.Marshal(method) //nolint:errcheck
 	return string(b)
@@ -197,6 +209,17 @@ func (method *ProviderInstallationFilesystemMirror) RemoveExclude(addrs []string
 	}
 }
 
+func (method *ProviderInstallationFilesystemMirror) RemoveInclude(addrs []string) {
+	if len(addrs) == 0 || method.Include == nil {
+		return
+	}
+	*method.Include = util.RemoveSublistFromList(*method.Include, addrs)
+
+	if len(*method.Include) == 0 {
+		method.Include = nil
+	}
+}
+
 func (method *ProviderInstallationFilesystemMirror) String() string {
 	b, _ := json.Marshal(method) //nolint:errcheck
 	return string(b)
@@ -271,6 +294,17 @@ func (method *ProviderInstallationNetworkMirror) RemoveExclude(addrs []string) {
 	}
 }
 
+func (method *ProviderInstallationNetworkMirror) RemoveInclude(addrs []string) {
+	if len(addrs) == 0 || method.Include == nil {
+		return
+	}
+	*method.Include = util.RemoveSublistFromList(*method.Include, addrs)
+
+	if len(*method.Include) == 0 {
+		method.Include = nil
+	}
+}
+
 func (method *ProviderInstallationNetworkMirror) String() string {
 	b, _ := json.Marshal(method) //nolint:errcheck
 	return string(b)


Test patch:
diff --git a/test/fixture-provider-cache/mirror/app/main.tf b/test/fixture-provider-cache/filesystem-mirror/app/main.tf
similarity index 86%
rename from test/fixture-provider-cache/mirror/app/main.tf
rename to test/fixture-provider-cache/filesystem-mirror/app/main.tf
index 3db6ee5fcd..1c4f049b95 100644
--- a/test/fixture-provider-cache/mirror/app/main.tf
+++ b/test/fixture-provider-cache/filesystem-mirror/app/main.tf
@@ -13,7 +13,3 @@ terraform {
     }
   }
 }
-
-provider "null" {
-  # Configuration options
-}
diff --git a/test/fixture-provider-cache/mirror/app/terragrunt.hcl b/test/fixture-provider-cache/filesystem-mirror/app/terragrunt.hcl
similarity index 100%
rename from test/fixture-provider-cache/mirror/app/terragrunt.hcl
rename to test/fixture-provider-cache/filesystem-mirror/app/terragrunt.hcl
diff --git a/test/fixture-provider-cache/network-mirror/app/main.tf b/test/fixture-provider-cache/network-mirror/app/main.tf
new file mode 100644
index 0000000000..46d2418f07
--- /dev/null
+++ b/test/fixture-provider-cache/network-mirror/app/main.tf
@@ -0,0 +1,12 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "example.com/hashicorp/aws"
+      version = "5.59.0"
+    }
+    azurerm = {
+      source  = "example.com/hashicorp/azurerm"
+      version = "3.113.0"
+    }
+  }
+}
diff --git a/test/fixture-provider-cache/network-mirror/app/terragrunt.hcl b/test/fixture-provider-cache/network-mirror/app/terragrunt.hcl
new file mode 100644
index 0000000000..bb7b160deb
--- /dev/null
+++ b/test/fixture-provider-cache/network-mirror/app/terragrunt.hcl
@@ -0,0 +1,1 @@
+# Intentionally empty
diff --git a/test/integration_serial_test.go b/test/integration_serial_test.go
index 22b6db1244..d788e0db3f 100644
--- a/test/integration_serial_test.go
+++ b/test/integration_serial_test.go
@@ -34,9 +34,9 @@ import (
 func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) {
 	// In this test we use os.Setenv to set the Terraform env var TF_CLI_CONFIG_FILE.
 
-	cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR)
-	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR)
-	rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_MIRROR)
+	cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR)
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR)
+	rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR)
 
 	appPath := filepath.Join(rootPath, "app")
 	providersMirrorPath := filepath.Join(rootPath, "providers-mirror")
@@ -86,9 +86,9 @@ func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) {
 	}
 	createCLIConfig(t, cliConfigFilename, cliConfigSettings)
 
-	runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath))
+	runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.opentofu.org --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath))
 
-	expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/*/*"] exclude = ["example.com/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.terraform.io/*/*"] } "direct" { } }`
+	expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/*/*"] exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "direct" { } }`
 	expectedProviderInstallation = fmt.Sprintf(strings.Join(strings.Fields(expectedProviderInstallation), " "), providersMirrorPath, providerCacheDir)
 
 	terraformrcBytes, err := os.ReadFile(filepath.Join(appPath, ".terraformrc"))
@@ -101,9 +101,9 @@ func TestTerragruntProviderCacheWithFilesystemMirror(t *testing.T) {
 func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) {
 	// In this test we use os.Setenv to set the Terraform env var TF_CLI_CONFIG_FILE.
 
-	cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR)
-	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_MIRROR)
-	rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_MIRROR)
+	cleanupTerraformFolder(t, TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR)
+	tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR)
+	rootPath := util.JoinPath(tmpEnvPath, TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR)
 
 	appPath := filepath.Join(rootPath, "app")
 	providersNetkworMirrorPath := filepath.Join(rootPath, "providers-network-mirror")
@@ -170,16 +170,15 @@ func TestTerragruntProviderCacheWithNetworkMirror(t *testing.T) {
 		NetworkMirrorMethods: []CLIConfigProviderInstallationNetworkMirror{
 			{
 				URL:     networkMirrorURL.String(),
-				Include: []string{"example.com/hashicorp/aws"},
+				Exclude: []string{"example.com/hashicorp/azurerm"},
 			},
 		},
 	}
 	createCLIConfig(t, cliConfigFilename, cliConfigSettings)
 
-	runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath))
-
-	expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/hashicorp/azurerm"] exclude = ["example.com/*/*", "registry.terraform.io/*/*"] } "network_mirror" { url = "%s" include = ["example.com/hashicorp/aws"] exclude = ["example.com/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.terraform.io/*/*"] } "direct" { } }`
+	runTerragrunt(t, fmt.Sprintf("terragrunt run-all init --terragrunt-provider-cache --terragrunt-provider-cache-registry-names example.com --terragrunt-provider-cache-registry-names registry.opentofu.org --terragrunt-provider-cache-registry-names registry.terraform.io --terragrunt-provider-cache-dir %s --terragrunt-log-level trace --terragrunt-non-interactive --terragrunt-working-dir %s", providerCacheDir, appPath))
 
+	expectedProviderInstallation := `provider_installation { "filesystem_mirror" { path = "%s" include = ["example.com/hashicorp/azurerm"] exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "network_mirror" { url = "%s" exclude = ["example.com/hashicorp/azurerm", "example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "filesystem_mirror" { path = "%s" include = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } "direct" { exclude = ["example.com/*/*", "registry.opentofu.org/*/*", "registry.terraform.io/*/*"] } }`
 	expectedProviderInstallation = fmt.Sprintf(strings.Join(strings.Fields(expectedProviderInstallation), " "), providersFilesystemMirrorPath, networkMirrorURL.String(), providerCacheDir)
 
 	terraformrcBytes, err := os.ReadFile(filepath.Join(appPath, ".terraformrc"))
diff --git a/test/integration_test.go b/test/integration_test.go
index 0b2dd48aed..95b999d53c 100644
--- a/test/integration_test.go
+++ b/test/integration_test.go
@@ -65,7 +65,8 @@ const (
 	TEST_FIXTURE_INIT_ONCE                                                   = "fixture-init-once"
 	TEST_FIXTURE_PROVIDER_CACHE_MULTIPLE_PLATFORMS                           = "fixture-provider-cache/multiple-platforms"
 	TEST_FIXTURE_PROVIDER_CACHE_DIRECT                                       = "fixture-provider-cache/direct"
-	TEST_FIXTURE_PROVIDER_CACHE_MIRROR                                       = "fixture-provider-cache/mirror"
+	TEST_FIXTURE_PROVIDER_CACHE_NETWORK_MIRROR                               = "fixture-provider-cache/network-mirror"
+	TEST_FIXTURE_PROVIDER_CACHE_FILESYSTEM_MIRROR                            = "fixture-provider-cache/filesystem-mirror"
 	TEST_FIXTURE_DESTROY_ORDER                                               = "fixture-destroy-order"
 	TEST_FIXTURE_CODEGEN_PATH                                                = "fixture-codegen"
 	TEST_FIXTURE_GCS_PATH                                                    = "fixture-gcs/"