Repository: chanzuckerberg/fogg
Pull request #497 (2020-09-25 04:36:21)

[feature ] allow limiting remote state configurations

Add `remote_states` to all components (except global) which will filter
the remote state configurations that are generated. Null means to
default to the current behavior.

PR also includes commit intended for the previous release.

### Test Plan
* ci

### References

codecov-commenter: # [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=h1) Report
> Merging [#497](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=desc) into [main](https://codecov.io/gh/chanzuckerberg/fogg/commit/d659318954367adface480203f818db91d294db9?el=desc) will **increase** coverage by `0.38%`.
> The diff coverage is `64.70%`.

[![Impacted file tree graph](https://codecov.io/gh/chanzuckerberg/fogg/pull/497/graphs/tree.svg?width=650&height=150&src=pr&token=57DhtZ7g2A)](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##             main     #497      +/-   ##
==========================================
+ Coverage   57.94%   58.32%   +0.38%     
==========================================
  Files          19       20       +1     
  Lines        1750     1778      +28     
==========================================
+ Hits         1014     1037      +23     
- Misses        580      586       +6     
+ Partials      156      155       -1     
```


| [Impacted Files](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [config/v2/config.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/497/diff?src=pr&el=tree#diff-Y29uZmlnL3YyL2NvbmZpZy5nbw==) | `8.02% <ø> (ø)` | |
| [config/v2/resolvers.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/497/diff?src=pr&el=tree#diff-Y29uZmlnL3YyL3Jlc29sdmVycy5nbw==) | `49.54% <0.00%> (-0.93%)` | :arrow_down: |
| [util/strings.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/497/diff?src=pr&el=tree#diff-dXRpbC9zdHJpbmdzLmdv) | `0.00% <0.00%> (ø)` | |
| [plan/plan.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/497/diff?src=pr&el=tree#diff-cGxhbi9wbGFuLmdv) | `86.46% <88.00%> (+1.60%)` | :arrow_up: |
| [plan/ci.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/497/diff?src=pr&el=tree#diff-cGxhbi9jaS5nbw==) | `76.64% <0.00%> (+1.19%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=footer). Last update [d659318...f089991](https://codecov.io/gh/chanzuckerberg/fogg/pull/497?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/VERSION b/VERSION
index 9cc9210f..526e900e 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-0.49.1
\ No newline at end of file
+0.49.2
\ No newline at end of file
diff --git a/config/v2/config.go b/config/v2/config.go
index c2625636..7c6f19da 100644
--- a/config/v2/config.go
+++ b/config/v2/config.go
@@ -56,6 +56,7 @@ type Common struct {
 	Owner            *string           `yaml:"owner,omitempty"`
 	Project          *string           `yaml:"project,omitempty"`
 	Providers        *Providers        `yaml:"providers,omitempty"`
+	DependsOn        *DependsOn        `yaml:"depends_on,omitempty"`
 	TerraformVersion *string           `yaml:"terraform_version,omitempty"`
 	Tools            *Tools            `yaml:"tools,omitempty"`
 }
@@ -214,6 +215,11 @@ type CommonCI struct {
 	Providers      map[string]CIProviderConfig `yaml:"providers,omitempty"`
 }
 
+type DependsOn struct {
+	Accounts   []string `yaml:"accounts,omitempty"`
+	Components []string `yaml:"components,omitempty"`
+}
+
 type CIProviderConfig struct {
 	Disabled bool `yaml:"disabled,omitempty"`
 }
diff --git a/config/v2/resolvers.go b/config/v2/resolvers.go
index 5bba0551..1f651de7 100644
--- a/config/v2/resolvers.go
+++ b/config/v2/resolvers.go
@@ -687,3 +687,17 @@ func CircleCISSHFingerprintsGetter(comm Common) []string {
 	}
 	return comm.Tools.CircleCI.SSHKeyFingerprints
 }
+
+func DependsOnAccountsGetter(comm Common) []string {
+	if comm.DependsOn == nil {
+		return nil
+	}
+	return comm.DependsOn.Accounts
+}
+
+func DependsOnComponentsGetter(comm Common) []string {
+	if comm.DependsOn == nil {
+		return nil
+	}
+	return comm.DependsOn.Components
+}
diff --git a/plan/plan.go b/plan/plan.go
index 310aeaf3..93c7e28b 100644
--- a/plan/plan.go
+++ b/plan/plan.go
@@ -234,10 +234,11 @@ type Account struct {
 type Component struct {
 	ComponentCommon `yaml:",inline"`
 
-	Accounts  map[string]Account `yaml:"accounts"` // Reference accounts for remote state
-	Component string             `yaml:"component"`
-	EKS       *v2.EKSConfig      `yaml:"eks,omitempty"`
-	Env       string
+	Accounts        map[string]Account `yaml:"accounts"` // Reference accounts for remote state
+	AccountBackends map[string]Backend `yaml:"account_backends"`
+	Component       string             `yaml:"component"`
+	EKS             *v2.EKSConfig      `yaml:"eks,omitempty"`
+	Env             string
 
 	Kind              *v2.ComponentKind  `yaml:"kind,omitempty"`
 	ModuleSource      *string            `yaml:"module_source"`
@@ -325,10 +326,23 @@ func (p *Plan) buildAccounts(c *v2.Config) map[string]Account {
 		accountBackends[acctName] = acct.Backend
 	}
 
-	for acctName := range accountPlans {
-		a := accountPlans[acctName]
-		a.AccountBackends = accountBackends
-		accountPlans[acctName] = a
+	for name, acct := range c.Accounts {
+
+		accountRemoteStates := v2.ResolveOptionalStringSlice(v2.DependsOnAccountsGetter, defaults.Common, acct.Common)
+		a := accountPlans[name]
+		filtered := map[string]Backend{}
+
+		if accountRemoteStates != nil {
+			for k, v := range accountBackends {
+				if util.SliceContainsString(accountRemoteStates, k) {
+					filtered[k] = v
+				}
+			}
+		} else {
+			filtered = accountBackends
+		}
+		a.AccountBackends = filtered
+		accountPlans[name] = a
 	}
 
 	return accountPlans
@@ -386,6 +400,8 @@ func (p *Plan) buildEnvs(conf *v2.Config) (map[string]Env, error) {
 		envPlan.Env = envName
 
 		for componentName, componentConf := range conf.Envs[envName].Components {
+			accountRemoteStates := v2.ResolveOptionalStringSlice(v2.DependsOnComponentsGetter, defaults.Common, envConf.Common, componentConf.Common)
+
 			componentPlan := Component{
 				Kind: componentConf.Kind,
 			}
@@ -397,6 +413,15 @@ func (p *Plan) buildEnvs(conf *v2.Config) (map[string]Env, error) {
 			if componentConf.Kind.GetOrDefault() == v2.ComponentKindHelmTemplate {
 				componentPlan.EKS = resolveEKSConfig(envPlan.EKS, componentConf.EKS)
 			}
+
+			accountBackends := map[string]Backend{}
+			for k, v := range p.Accounts {
+				if accountRemoteStates == nil || util.SliceContainsString(accountRemoteStates, k) {
+					accountBackends[k] = v.Backend
+				}
+			}
+			componentPlan.AccountBackends = accountBackends
+
 			componentPlan.Accounts = p.Accounts
 
 			componentPlan.ComponentCommon = resolveComponentCommon(defaults.Common, envConf.Common, componentConf.Common)
@@ -431,10 +456,23 @@ func (p *Plan) buildEnvs(conf *v2.Config) (map[string]Env, error) {
 			componentBackends[componentName] = component.Backend
 		}
 
-		for componentName := range envPlan.Components {
-			c := envPlan.Components[componentName]
-			c.ComponentBackends = componentBackends
-			envPlan.Components[componentName] = c
+		for name, componentConf := range conf.Envs[envName].Components {
+			componentRemoteStates := v2.ResolveOptionalStringSlice(v2.DependsOnComponentsGetter, defaults.Common, envConf.Common, componentConf.Common)
+			c := envPlan.Components[name]
+			filtered := map[string]Backend{}
+
+			if componentRemoteStates != nil {
+				for k, v := range componentBackends {
+					if util.SliceContainsString(componentRemoteStates, k) {
+						filtered[k] = v
+					}
+				}
+			} else {
+				filtered = componentBackends
+			}
+
+			c.ComponentBackends = filtered
+			envPlan.Components[name] = c
 		}
 
 		envPlans[envName] = envPlan
diff --git a/templates/component/terraform/fogg.tf.tmpl b/templates/component/terraform/fogg.tf.tmpl
index 5619caf7..17b17f10 100644
--- a/templates/component/terraform/fogg.tf.tmpl
+++ b/templates/component/terraform/fogg.tf.tmpl
@@ -119,12 +119,12 @@ data terraform_remote_state {{ $component }} {
 {{ end }}
 
 # remote state for accounts
-{{ range $accountName, $account := .Accounts }}
-data terraform_remote_state {{ $accountName }} {
-  backend = "{{ $account.Backend.Kind }}"
+{{ range $name, $backend := .AccountBackends }}
+data terraform_remote_state {{ $name }} {
+  backend = "{{ $backend.Kind }}"
 
   config = {
-    {{ template "remote_backend" $account.Backend }}
+    {{ template "remote_backend" $backend }}
   }
 }
 {{ end }}
diff --git a/util/strings.go b/util/strings.go
new file mode 100644
index 00000000..db8383d7
--- /dev/null
+++ b/util/strings.go
@@ -0,0 +1,10 @@
+package util
+
+func SliceContainsString(haystack []string, needle string) bool {
+	for _, s := range haystack {
+		if s == needle {
+			return true
+		}
+	}
+	return false
+}


Test patch:
diff --git a/plan/plan_test.go b/plan/plan_test.go
index 405526c0..dd87460b 100644
--- a/plan/plan_test.go
+++ b/plan/plan_test.go
@@ -125,11 +125,11 @@ func TestPlanBasicV2Yaml(t *testing.T) {
 	r.Equal("terraform/proj/accounts/bar.tfstate", plan.Accounts["bar"].Backend.S3.KeyPath)
 	r.Equal("terraform/proj/accounts/foo.tfstate", plan.Accounts["foo"].Backend.S3.KeyPath)
 
-	r.Len(plan.Accounts["foo"].AccountBackends, 2)
-	r.NotNil(plan.Accounts["foo"].AccountBackends["bar"])
-	r.Equal(BackendKindS3, plan.Accounts["foo"].AccountBackends["bar"].Kind)
-	r.NotNil(plan.Accounts["foo"].AccountBackends["bar"].S3)
-	r.Equal("terraform/proj/accounts/bar.tfstate", plan.Accounts["foo"].AccountBackends["bar"].S3.KeyPath)
+	r.Len(plan.Accounts["bar"].AccountBackends, 2)
+	r.NotNil(plan.Accounts["bar"].AccountBackends["foo"])
+	r.Equal(BackendKindS3, plan.Accounts["bar"].AccountBackends["foo"].Kind)
+	r.NotNil(plan.Accounts["bar"].AccountBackends["foo"].S3)
+	r.Equal("terraform/proj/accounts/foo.tfstate", plan.Accounts["bar"].AccountBackends["foo"].S3.KeyPath)
 }
 
 func TestResolveEKSConfig(t *testing.T) {
diff --git a/testdata/v2_full_yaml/fogg.yml b/testdata/v2_full_yaml/fogg.yml
index ff1723e6..e30fd3d2 100644
--- a/testdata/v2_full_yaml/fogg.yml
+++ b/testdata/v2_full_yaml/fogg.yml
@@ -20,6 +20,9 @@ accounts:
         aws_profile: prof
         version: 0.4.2
         aws_region: us-west-2
+    depends_on:
+      accounts: []
+      components: []
 defaults:
   backend:
     bucket: buck
@@ -42,9 +45,17 @@ envs:
       hero:
         providers:
           heroku: {}
+        depends_on:
+          accounts: []
+          components: []
       datadog:
         providers:
           datadog: {}
+        depends_on:
+          accounts: []
+          components:
+            - foo
+            - hero
       vpc:
         module_source: github.com/terraform-aws-modules/terraform-aws-vpc?ref=v1.30.0
         module_name: prod-vpc
diff --git a/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf b/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf
index 69743ffa..a2a6c17b 100644
--- a/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf
@@ -91,20 +91,6 @@ data terraform_remote_state global {
     profile = "profile"
 
 
-  }
-}
-data terraform_remote_state bar {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/accounts/bar.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
   }
 }
 provider random {
diff --git a/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf b/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf
index d70432d3..48a10af3 100644
--- a/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf
@@ -92,37 +92,9 @@ data terraform_remote_state hero {
     profile = "profile"
 
 
-  }
-}
-data terraform_remote_state vpc {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/envs/prod/components/vpc.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
   }
 }
 # remote state for accounts
-data terraform_remote_state bar {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/accounts/bar.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
-  }
-}
 data terraform_remote_state foo {
   backend = "s3"
   config = {
diff --git a/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf b/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf
index 792cd917..375774ee 100644
--- a/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf
@@ -78,65 +78,9 @@ data terraform_remote_state global {
     profile = "profile"
 
 
-  }
-}
-data terraform_remote_state datadog {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/envs/prod/components/datadog.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
-  }
-}
-data terraform_remote_state vpc {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/envs/prod/components/vpc.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
   }
 }
 # remote state for accounts
-data terraform_remote_state bar {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/accounts/bar.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
-  }
-}
-data terraform_remote_state foo {
-  backend = "s3"
-  config = {
-
-
-    bucket = "buck"
-
-    key     = "terraform/proj/accounts/foo.tfstate"
-    region  = "us-west-2"
-    profile = "profile"
-
-
-  }
-}
 # map of aws_accounts
 variable aws_accounts {
   type = map