Repository: chanzuckerberg/fogg
Pull request #466 (2020-04-22 03:23:39)

[refactor] aws provider into common template
Refactor our aws provider configurations in to a common template that is shared across global, accounts and components.

Also push the logic for generating per-region providers from the templates to the planning stage.

### Test Plan
* [x] CI
* [X] tested on 1 internal repo

### References


Patch:
diff --git a/plan/plan.go b/plan/plan.go
index 59ae7740..fa0227c3 100644
--- a/plan/plan.go
+++ b/plan/plan.go
@@ -111,23 +111,24 @@ func (c CIComponent) generateCIConfig(
 }
 
 type Providers struct {
-	AWS       *AWSProvider       `yaml:"aws"`
-	Github    *GithubProvider    `yaml:"github"`
-	Heroku    *HerokuProvider    `yaml:"heroku"`
-	Snowflake *SnowflakeProvider `yaml:"snowflake"`
-	Bless     *BlessProvider     `yaml:"bless"`
-	Okta      *OktaProvider      `yaml:"okta"`
-	Datadog   *DatadogProvider   `yaml:"datadog"`
-	Tfe       *TfeProvider       `yaml:"tfe"`
+	AWS                    *AWSProvider       `yaml:"aws"`
+	AWSAdditionalProviders []AWSProvider      `yaml:"aws_regional_providers"`
+	Github                 *GithubProvider    `yaml:"github"`
+	Heroku                 *HerokuProvider    `yaml:"heroku"`
+	Snowflake              *SnowflakeProvider `yaml:"snowflake"`
+	Bless                  *BlessProvider     `yaml:"bless"`
+	Okta                   *OktaProvider      `yaml:"okta"`
+	Datadog                *DatadogProvider   `yaml:"datadog"`
+	Tfe                    *TfeProvider       `yaml:"tfe"`
 }
 
 //AWSProvider represents AWS provider configuration
 type AWSProvider struct {
-	AccountID         json.Number `yaml:"account_id"`
-	Profile           string      `yaml:"profile"`
-	Version           string      `yaml:"version"`
-	Region            string      `yaml:"region"`
-	AdditionalRegions []string    `yaml:"additional_regions"`
+	AccountID json.Number `yaml:"account_id"`
+	Alias     *string     `yaml:"alias"`
+	Profile   string      `yaml:"profile"`
+	Region    string      `yaml:"region"`
+	Version   string      `yaml:"version"`
 }
 
 // GithubProvider represents a configuration of a github provider
@@ -437,14 +438,27 @@ func (p *Plan) buildEnvs(conf *v2.Config) (map[string]Env, error) {
 func resolveComponentCommon(commons ...v2.Common) ComponentCommon {
 	var awsPlan *AWSProvider
 	awsConfig := v2.ResolveAWSProvider(commons...)
+	additionalProviders := []AWSProvider{}
 
 	if awsConfig != nil {
 		awsPlan = &AWSProvider{
-			AccountID:         *awsConfig.AccountID,
-			Profile:           *awsConfig.Profile,
-			Version:           *awsConfig.Version,
-			Region:            *awsConfig.Region,
-			AdditionalRegions: awsConfig.AdditionalRegions,
+			AccountID: *awsConfig.AccountID,
+			Profile:   *awsConfig.Profile,
+			Version:   *awsConfig.Version,
+			Region:    *awsConfig.Region,
+		}
+
+		for _, r := range awsConfig.AdditionalRegions {
+			// we have to take a reference here otherwise it gets overwritten by the loop
+			region := r
+			additionalProviders = append(additionalProviders,
+				AWSProvider{
+					AccountID: *awsConfig.AccountID,
+					Alias:     &region,
+					Profile:   *awsConfig.Profile,
+					Version:   *awsConfig.Version,
+					Region:    region,
+				})
 		}
 	}
 
@@ -593,14 +607,15 @@ func resolveComponentCommon(commons ...v2.Common) ComponentCommon {
 	return ComponentCommon{
 		Backend: backend,
 		Providers: Providers{
-			AWS:       awsPlan,
-			Github:    githubPlan,
-			Heroku:    herokuPlan,
-			Snowflake: snowflakePlan,
-			Bless:     blessPlan,
-			Okta:      oktaPlan,
-			Datadog:   datadogPlan,
-			Tfe:       tfePlan,
+			AWS:                    awsPlan,
+			AWSAdditionalProviders: additionalProviders,
+			Github:                 githubPlan,
+			Heroku:                 herokuPlan,
+			Snowflake:              snowflakePlan,
+			Bless:                  blessPlan,
+			Okta:                   oktaPlan,
+			Datadog:                datadogPlan,
+			Tfe:                    tfePlan,
 		},
 		TfLint:          tfLintPlan,
 		ExtraVars:       v2.ResolveStringMap(v2.ExtraVarsGetter, commons...),
diff --git a/templates/account/fogg.tf.tmpl b/templates/account/fogg.tf.tmpl
index b7182faa..0979b82a 100644
--- a/templates/account/fogg.tf.tmpl
+++ b/templates/account/fogg.tf.tmpl
@@ -2,24 +2,10 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 {{if .Providers.AWS}}
-  # Default Provider
-  provider aws {
-    version = "~> {{ .Providers.AWS.Version }}"
-    region = "{{ .Providers.AWS.Region }}"
-    profile = "{{ .Providers.AWS.Profile }}"
-    allowed_account_ids = [{{ .Providers.AWS.AccountID }}]
-  }
-
+{{ template "aws_provider" .Providers.AWS}}
   # Aliased Providers (for doing things in every region).
-  {{ $outer := . -}}
-  {{ range $region := .Providers.AWS.AdditionalRegions }}
-    provider aws {
-      alias = "{{ $region }}"
-      version = "~> {{ $outer.Providers.AWS.Version }}"
-      region = "{{ $region }}"
-      profile = "{{ $outer.Providers.AWS.Profile }}"
-      allowed_account_ids = [{{ $outer.Providers.AWS.AccountID }}]
-    }
+  {{ range $p := .Providers.AWSAdditionalProviders }}
+    {{ template "aws_provider" $p}}
   {{ end }}
 {{ end }}
 
diff --git a/templates/common/aws_provider.tmpl b/templates/common/aws_provider.tmpl
new file mode 100644
index 00000000..846d8dcf
--- /dev/null
+++ b/templates/common/aws_provider.tmpl
@@ -0,0 +1,9 @@
+{{ define "aws_provider" }}
+provider aws {
+  {{ if .Alias }}alias = "{{ .Alias }}"{{ end }}
+  version = "~> {{ .Version }}"
+  region = "{{ .Region }}"
+  profile = "{{ .Profile }}"
+  allowed_account_ids = [{{ .AccountID }}]
+}
+{{ end }}
diff --git a/templates/component/terraform/fogg.tf.tmpl b/templates/component/terraform/fogg.tf.tmpl
index 1abebe6a..f2328112 100644
--- a/templates/component/terraform/fogg.tf.tmpl
+++ b/templates/component/terraform/fogg.tf.tmpl
@@ -2,23 +2,10 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 {{ if .Providers.AWS }}
-  provider aws {
-    version = "~> {{ .Providers.AWS.Version }}"
-    region = "{{ .Providers.AWS.Region }}"
-    profile = "{{ .Providers.AWS.Profile }}"
-    allowed_account_ids = [{{ .Providers.AWS.AccountID }}]
-  }
-
+{{ template "aws_provider" .Providers.AWS}}
   # Aliased Providers (for doing things in every region).
-  {{ $outer := . }}
-  {{ range $region := .Providers.AWS.AdditionalRegions }}
-    provider aws {
-      alias = "{{ $region }}"
-      version = "~> {{ $outer.Providers.AWS.Version }}"
-      region = "{{ $region }}"
-      profile = "{{ $outer.Providers.AWS.Profile }}"
-      allowed_account_ids = [{{ $outer.Providers.AWS.AccountID }}]
-    }
+  {{ range $p := .Providers.AWSAdditionalProviders }}
+    {{ template "aws_provider" $p}}
   {{ end }}
 
 {{ end }}
diff --git a/templates/global/fogg.tf.tmpl b/templates/global/fogg.tf.tmpl
index fc2d76ae..c2dc95a5 100644
--- a/templates/global/fogg.tf.tmpl
+++ b/templates/global/fogg.tf.tmpl
@@ -2,25 +2,11 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 {{ if .Providers.AWS }}
-  provider aws {
-    version = "~> {{ .Providers.AWS.Version }}"
-    region = "{{ .Providers.AWS.Region }}"
-    profile = "{{ .Providers.AWS.Profile }}"
-    {{ if .Providers.AWS.AccountID }}allowed_account_ids = [{{ .Providers.AWS.AccountID }}]{{ end }}
-  }
-
+{{ template "aws_provider" .Providers.AWS}}
   # Aliased Providers (for doing things in every region).
-  {{ $p := .Providers.AWS }}
-  {{ range $region := .Providers.AWS.AdditionalRegions }}
-    provider aws {
-      alias = "{{ $region }}"
-      version = "~> {{ $p.Version }}"
-      region = "{{ $region }}"
-      profile = "{{ $p.Profile }}"
-      {{ if $p.AccountID }}allowed_account_ids = [{{ $p.AccountID }}]{{ end }}
-    }
-  {{ end }}
-{{ end }}
+  {{ range $p := .Providers.AWSAdditionalProviders }}
+    {{ template "aws_provider" $p}}
+  {{ end }}{{ end }}
 
 {{ if .Providers.Snowflake }}
   {{ template "snowflake_provider" .Providers.Snowflake }}


Test patch:
diff --git a/testdata/v2_full_yaml/fogg.yml b/testdata/v2_full_yaml/fogg.yml
index 7f59e5e0..1556352e 100644
--- a/testdata/v2_full_yaml/fogg.yml
+++ b/testdata/v2_full_yaml/fogg.yml
@@ -3,6 +3,9 @@ accounts:
     providers:
       aws:
         account_id: 456
+        additional_regions:
+          - us-east-1
+          - us-east-2
   foo:
     providers:
       aws:
diff --git a/testdata/v2_full_yaml/terraform/accounts/bar/fogg.tf b/testdata/v2_full_yaml/terraform/accounts/bar/fogg.tf
index 571fd93a..2a0ca97b 100644
--- a/testdata/v2_full_yaml/terraform/accounts/bar/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/accounts/bar/fogg.tf
@@ -2,8 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
-# Default Provider
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -13,6 +14,26 @@ provider aws {
 # Aliased Providers (for doing things in every region).
 
 
+provider aws {
+  alias               = "us-east-1"
+  version             = "~> 0.12.0"
+  region              = "us-east-1"
+  profile             = "profile"
+  allowed_account_ids = [456]
+}
+
+
+
+provider aws {
+  alias               = "us-east-2"
+  version             = "~> 0.12.0"
+  region              = "us-east-2"
+  profile             = "profile"
+  allowed_account_ids = [456]
+}
+
+
+
 
 
 
diff --git a/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf b/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf
index 22cdd717..c62243d1 100644
--- a/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/accounts/foo/fogg.tf
@@ -2,8 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
-# Default Provider
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
diff --git a/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf b/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf
index cb93199c..5672733d 100644
--- a/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/prod/datadog/fogg.tf
@@ -2,7 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -26,7 +28,6 @@ provider aws {
 
 
 
-
 provider datadog {
 }
 
diff --git a/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf b/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf
index d895b2a0..5c694b72 100644
--- a/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/prod/hero/fogg.tf
@@ -2,7 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -24,7 +26,6 @@ provider aws {
 
 
 
-
 provider heroku {
 }
 
diff --git a/testdata/v2_full_yaml/terraform/envs/staging/comp1/fogg.tf b/testdata/v2_full_yaml/terraform/envs/staging/comp1/fogg.tf
index 522d5f97..e96b224e 100644
--- a/testdata/v2_full_yaml/terraform/envs/staging/comp1/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/staging/comp1/fogg.tf
@@ -2,7 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -29,7 +31,6 @@ provider aws {
 
 
 
-
 terraform {
   required_version = "~>0.100.0"
 
diff --git a/testdata/v2_full_yaml/terraform/envs/staging/comp2/fogg.tf b/testdata/v2_full_yaml/terraform/envs/staging/comp2/fogg.tf
index 9ba30a59..1bbe4ca3 100644
--- a/testdata/v2_full_yaml/terraform/envs/staging/comp2/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/staging/comp2/fogg.tf
@@ -2,7 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -29,7 +31,6 @@ provider aws {
 
 
 
-
 terraform {
   required_version = "~>0.100.0"
 
diff --git a/testdata/v2_full_yaml/terraform/envs/staging/vpc/fogg.tf b/testdata/v2_full_yaml/terraform/envs/staging/vpc/fogg.tf
index 501a0200..61565b3a 100644
--- a/testdata/v2_full_yaml/terraform/envs/staging/vpc/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/envs/staging/vpc/fogg.tf
@@ -2,7 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -29,7 +31,6 @@ provider aws {
 
 
 
-
 terraform {
   required_version = "~>0.100.0"
 
diff --git a/testdata/v2_full_yaml/terraform/global/fogg.tf b/testdata/v2_full_yaml/terraform/global/fogg.tf
index 7fd9713d..f59f3b06 100644
--- a/testdata/v2_full_yaml/terraform/global/fogg.tf
+++ b/testdata/v2_full_yaml/terraform/global/fogg.tf
@@ -2,7 +2,9 @@
 # Make improvements in fogg, so that everyone can benefit.
 
 
+
 provider aws {
+
   version             = "~> 0.12.0"
   region              = "us-west-2"
   profile             = "profile"
@@ -26,8 +28,6 @@ provider aws {
 
 
 
-
-
 terraform {
   required_version = "~>0.100.0"