Repository: chanzuckerberg/fogg
Pull request #422 (2020-03-09 22:58:17)

[refactor] make backend type configurable and default to 's3'
Up until now we have only supported the [s3 backend](https://www.terraform.io/docs/backends/types/s3.html). In preparation for supporting the [remote backend](https://www.terraform.io/docs/backends/types/remote.html), this PR refactors the config, plan and templates to make the type explicit, but still only all 's3'.

### Test Plan
* CI

### References
* https://www.terraform.io/docs/backends/types/s3.html
* https://www.terraform.io/docs/backends/types/remote.html

codecov[bot]: # [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=h1) Report
> Merging [#422](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=desc) into [master](https://codecov.io/gh/chanzuckerberg/fogg/commit/56b0380aab6b4c811eedb54af48f038c06b97e0b&el=desc) will **increase** coverage by `0.02%`.
> The diff coverage is `100.00%`.

[![Impacted file tree graph](https://codecov.io/gh/chanzuckerberg/fogg/pull/422/graphs/tree.svg?width=650&height=150&src=pr&token=57DhtZ7g2A)](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #422      +/-   ##
==========================================
+ Coverage   76.00%   76.03%   +0.02%     
==========================================
  Files          22       22              
  Lines        1817     1819       +2     
==========================================
+ Hits         1381     1383       +2     
  Misses        293      293              
  Partials      143      143              
```


| [Impacted Files](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [config/v2/config.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/422/diff?src=pr&el=tree#diff-Y29uZmlnL3YyL2NvbmZpZy5nbw==) | `97.08% <ø> (ø)` | |
| [plan/plan.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/422/diff?src=pr&el=tree#diff-cGxhbi9wbGFuLmdv) | `92.08% <100.00%> (+0.06%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=footer). Last update [56b0380...a219a33](https://codecov.io/gh/chanzuckerberg/fogg/pull/422?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/Makefile b/Makefile
index 9eea20ba..0202345f 100644
--- a/Makefile
+++ b/Makefile
@@ -79,9 +79,9 @@ coverage: ## run the go coverage tool, reading file coverage.out
 
 test: fmt deps packr ## run tests
  ifeq (, $(shell which gotest))
-	gotest -cover ./...
- else
 	go test -cover ./...
+ else
+	gotest -cover ./...
  endif
 .PHONY: test
 
diff --git a/config/v2/config.go b/config/v2/config.go
index 5e1be52b..519a9add 100644
--- a/config/v2/config.go
+++ b/config/v2/config.go
@@ -150,7 +150,9 @@ type HerokuProvider struct{}
 
 type DatadogProvider struct{}
 
+//Backend is used to configure a terraform backend
 type Backend struct {
+	Type        *string `json:"type,omitempty" yaml:"type,omitempty" validate:"omitempty,oneof=s3"`
 	AccountID   *string `json:"account_id,omitempty" yaml:"account_id,omitempty"`
 	Bucket      *string `json:"bucket,omitempty" yaml:"bucket,omitempty"`
 	DynamoTable *string `json:"dynamodb_table,omitempty" yaml:"dynamodb_table,omitempty"`
diff --git a/plan/plan.go b/plan/plan.go
index 4e08a136..29b0e1a1 100644
--- a/plan/plan.go
+++ b/plan/plan.go
@@ -34,7 +34,7 @@ type Common struct {
 type ComponentCommon struct {
 	Common `yaml:",inline"`
 
-	Backend   AWSBackend        `json:"backend" yaml:"backend"`
+	Backend   Backend           `json:"backend" yaml:"backend"`
 	ExtraVars map[string]string `json:"extra_vars" yaml:"extra_vars"`
 	Owner     string            `json:"owner" yaml:"owner"`
 	Project   string            `json:"project" yaml:"project"`
@@ -72,7 +72,7 @@ type CIComponent struct {
 
 // generateCIConfig generates the config for ci
 func (c CIComponent) generateCIConfig(
-	backend AWSBackend,
+	backend Backend,
 	provider *AWSProvider,
 	projName string,
 	projDir string) *CIConfig {
@@ -93,9 +93,9 @@ func (c CIComponent) generateCIConfig(
 		Command: c.Command,
 	})
 
-	if backend.AccountID != nil {
-		ciConfig.AWSProfiles[backend.Profile] = AWSRole{
-			AccountID: *backend.AccountID,
+	if backend.S3 != nil && backend.S3.AccountID != nil {
+		ciConfig.AWSProfiles[backend.S3.Profile] = AWSRole{
+			AccountID: *backend.S3.AccountID,
 			RoleName:  c.AWSRoleName,
 		}
 	}
@@ -161,8 +161,23 @@ type HerokuProvider struct{}
 
 type DatadogProvider struct{}
 
-// AWSBackend represents aws backend configuration
-type AWSBackend struct {
+// BackendType is a enum of backends we support
+type BackendType string
+
+const (
+	// BackendTypeS3 is https://www.terraform.io/docs/backends/types/s3.html
+	BackendTypeS3 BackendType = "s3"
+)
+
+//Backend represents a plan for configuring the terraform backend. Only one struct member can be
+//non-nil at a time
+type Backend struct {
+	Type BackendType `yaml:"type"`
+	S3   *S3Backend  `yaml:"s3,omitempty"`
+}
+
+// S3Backend represents aws backend configuration
+type S3Backend struct {
 	AccountID   *string `yaml:"account_id,omitempty"`
 	AccountName string  `json:"account_name" yaml:"account_name"`
 	Profile     string  `json:"profile" yaml:"profile"`
@@ -459,12 +474,14 @@ func resolveComponentCommon(commons ...v2.Common) ComponentCommon {
 	}
 
 	return ComponentCommon{
-		Backend: AWSBackend{
-			AccountID:   v2.ResolveOptionalString(v2.BackendAccountIdGetter, commons...),
-			Region:      v2.ResolveRequiredString(v2.BackendRegionGetter, commons...),
-			Profile:     v2.ResolveRequiredString(v2.BackendProfileGetter, commons...),
-			Bucket:      v2.ResolveRequiredString(v2.BackendBucketGetter, commons...),
-			DynamoTable: v2.ResolveOptionalString(v2.BackendDynamoTableGetter, commons...),
+		Backend: Backend{
+			S3: &S3Backend{
+				AccountID:   v2.ResolveOptionalString(v2.BackendAccountIdGetter, commons...),
+				Region:      v2.ResolveRequiredString(v2.BackendRegionGetter, commons...),
+				Profile:     v2.ResolveRequiredString(v2.BackendProfileGetter, commons...),
+				Bucket:      v2.ResolveRequiredString(v2.BackendBucketGetter, commons...),
+				DynamoTable: v2.ResolveOptionalString(v2.BackendDynamoTableGetter, commons...),
+			},
 		},
 		Providers: Providers{
 			AWS:       awsPlan,
diff --git a/templates/account/Makefile.tmpl b/templates/account/Makefile.tmpl
index 627ae243..6082a075 100644
--- a/templates/account/Makefile.tmpl
+++ b/templates/account/Makefile.tmpl
@@ -4,7 +4,7 @@
 export TERRAFORM_VERSION := {{ .TerraformVersion }}
 export TFLINT_ENABLED := {{ if .TfLint.Enabled }}1{{ else }}0{{ end }}
 export TF_PLUGIN_CACHE_DIR := {{ .PathToRepoRoot }}/.terraform.d/plugin-cache
-export AWS_BACKEND_PROFILE := {{ .Backend.Profile }}
+export AWS_BACKEND_PROFILE := {{ .Backend.S3.Profile }}
 {{if .Providers.AWS}}
 export AWS_PROVIDER_PROFILE := {{ .Providers.AWS.Profile }}
 {{ end }}
diff --git a/templates/account/fogg.tf.tmpl b/templates/account/fogg.tf.tmpl
index a729d8ba..041523e6 100644
--- a/templates/account/fogg.tf.tmpl
+++ b/templates/account/fogg.tf.tmpl
@@ -51,12 +51,12 @@ terraform {
   required_version = "={{ .TerraformVersion }}"
 
   backend "s3" {
-    bucket         = "{{ .Backend.Bucket }}"
-    {{ if .Backend.DynamoTable }}dynamodb_table = "{{ .Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ .Backend.S3.Bucket }}"
+    {{ if .Backend.S3.DynamoTable }}dynamodb_table = "{{ .Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ .Project }}/accounts/{{ .AccountName }}.tfstate"
     encrypt        = true
-    region         = "{{ .Backend.Region }}"
-    profile        = "{{ .Backend.Profile }}"
+    region         = "{{ .Backend.S3.Region }}"
+    profile        = "{{ .Backend.S3.Profile }}"
   }
 }
 
@@ -106,11 +106,11 @@ data "terraform_remote_state" "global" {
   backend = "s3"
 
   config = {
-    bucket         = "{{ .Global.Backend.Bucket }}"
-    {{ if .Backend.DynamoTable }}dynamodb_table = "{{ .Global.Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ .Global.Backend.S3.Bucket }}"
+    {{ if .Backend.S3.DynamoTable }}dynamodb_table = "{{ .Global.Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ .Global.Project }}/global.tfstate"
-    region         = "{{ .Global.Backend.Region }}"
-    {{ if .Global.Backend.Profile }}profile        = "{{ .Global.Backend.Profile }}"{{ end }}
+    region         = "{{ .Global.Backend.S3.Region }}"
+    {{ if .Global.Backend.S3.Profile }}profile        = "{{ .Global.Backend.S3.Profile }}"{{ end }}
   }
 }
 
@@ -121,11 +121,11 @@ data "terraform_remote_state" "{{ $accountName }}" {
   backend = "s3"
 
   config = {
-    bucket = "{{ $outer.Backend.Bucket }}"
-    {{ if $outer.Backend.DynamoTable }}dynamodb_table = "{{ $outer.Backend.DynamoTable }}"{{ end }}
+    bucket = "{{ $outer.Backend.S3.Bucket }}"
+    {{ if $outer.Backend.S3.DynamoTable }}dynamodb_table = "{{ $outer.Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ $outer.Project }}/accounts/{{ $accountName }}.tfstate"
-    region         = "{{ $outer.Backend.Region }}"
-    {{ if $outer.Backend.Profile }}profile        = "{{ $outer.Backend.Profile }}"{{ end }}
+    region         = "{{ $outer.Backend.S3.Region }}"
+    {{ if $outer.Backend.S3.Profile }}profile        = "{{ $outer.Backend.S3.Profile }}"{{ end }}
   }
 }
 {{ end }}
diff --git a/templates/component/terraform/Makefile.tmpl b/templates/component/terraform/Makefile.tmpl
index 69acb9ac..64648f6a 100644
--- a/templates/component/terraform/Makefile.tmpl
+++ b/templates/component/terraform/Makefile.tmpl
@@ -4,7 +4,7 @@
 export TERRAFORM_VERSION := {{ .TerraformVersion }}
 export TFLINT_ENABLED := {{ if .TfLint.Enabled }}1{{ else }}0{{ end }}
 export TF_PLUGIN_CACHE_DIR := {{ .PathToRepoRoot }}/.terraform.d/plugin-cache
-export AWS_BACKEND_PROFILE := {{ .Backend.Profile }}
+export AWS_BACKEND_PROFILE := {{ .Backend.S3.Profile }}
 {{if .Providers.AWS}}
 export AWS_PROVIDER_PROFILE := {{ .Providers.AWS.Profile }}
 {{ end }}
diff --git a/templates/component/terraform/fogg.tf.tmpl b/templates/component/terraform/fogg.tf.tmpl
index 533dd6e3..6786d636 100644
--- a/templates/component/terraform/fogg.tf.tmpl
+++ b/templates/component/terraform/fogg.tf.tmpl
@@ -51,8 +51,8 @@ terraform {
   required_version = "~>{{ .TerraformVersion }}"
 
   backend "s3" {
-    bucket         = "{{ .Backend.Bucket }}"
-    {{ if .Backend.DynamoTable }}dynamodb_table = "{{ .Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ .Backend.S3.Bucket }}"
+    {{ if .Backend.S3.DynamoTable }}dynamodb_table = "{{ .Backend.S3.DynamoTable }}"{{ end }}
     {{/* {%- if env is defined and component_name is defined %} */}}
     key            = "terraform/{{ .Project }}/envs/{{ .Env }}/components/{{ .Component }}.tfstate"
 
@@ -62,8 +62,8 @@ terraform {
     {%- endif %}
 */}}
     encrypt = true
-    region = "{{ .Backend.Region }}"
-    profile = "{{ .Backend.Profile }}"
+    region = "{{ .Backend.S3.Region }}"
+    profile = "{{ .Backend.S3.Profile }}"
   }
 }
 
@@ -124,11 +124,11 @@ data "terraform_remote_state" "global" {
   backend = "s3"
 
   config = {
-    bucket         = "{{ .Global.Backend.Bucket }}"
-    {{ if .Backend.DynamoTable }}dynamodb_table = "{{ .Global.Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ .Global.Backend.S3.Bucket }}"
+    {{ if .Backend.S3.DynamoTable }}dynamodb_table = "{{ .Global.Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ .Global.Project }}/global.tfstate"
-    region         = "{{ .Global.Backend.Region }}"
-    {{ if .Global.Backend.Profile }}profile        = "{{ .Global.Backend.Profile }}"{{ end }}
+    region         = "{{ .Global.Backend.S3.Region }}"
+    {{ if .Global.Backend.S3.Profile }}profile        = "{{ .Global.Backend.S3.Profile }}"{{ end }}
   }
 }
 
@@ -138,11 +138,11 @@ data "terraform_remote_state" "{{ $component }}" {
   backend = "s3"
 
   config = {
-    bucket         = "{{ $outer.Backend.Bucket }}"
-    {{ if $outer.Backend.DynamoTable }}dynamodb_table = "{{ $outer.Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ $outer.Backend.S3.Bucket }}"
+    {{ if $outer.Backend.S3.DynamoTable }}dynamodb_table = "{{ $outer.Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ $outer.Project }}/envs/{{ $outer.Env }}/components/{{ $component }}.tfstate"
-    region         = "{{ $outer.Backend.Region }}"
-    {{ if $outer.Backend.Profile }}profile        = "{{ $outer.Backend.Profile }}"{{ end }}
+    region         = "{{ $outer.Backend.S3.Region }}"
+    {{ if $outer.Backend.S3.Profile }}profile        = "{{ $outer.Backend.S3.Profile }}"{{ end }}
   }
 }
 {{ end }}
@@ -153,11 +153,11 @@ data "terraform_remote_state" "{{ $accountName }}" {
   backend = "s3"
 
   config = {
-    bucket         = "{{ $account.Backend.Bucket }}"
-    {{ if $account.Backend.DynamoTable }}dynamodb_table = "{{ $account.Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ $account.Backend.S3.Bucket }}"
+    {{ if $account.Backend.S3.DynamoTable }}dynamodb_table = "{{ $account.Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ .Project }}/accounts/{{ $account.AccountName }}.tfstate"
-    region         = "{{ $account.Backend.Region }}"
-    {{ if $account.Backend.Profile }}profile        = "{{ $account.Backend.Profile }}"{{ end }}
+    region         = "{{ $account.Backend.S3.Region }}"
+    {{ if $account.Backend.S3.Profile }}profile        = "{{ $account.Backend.S3.Profile }}"{{ end }}
   }
 }
 {{ end }}
diff --git a/templates/global/Makefile.tmpl b/templates/global/Makefile.tmpl
index 627ae243..6082a075 100644
--- a/templates/global/Makefile.tmpl
+++ b/templates/global/Makefile.tmpl
@@ -4,7 +4,7 @@
 export TERRAFORM_VERSION := {{ .TerraformVersion }}
 export TFLINT_ENABLED := {{ if .TfLint.Enabled }}1{{ else }}0{{ end }}
 export TF_PLUGIN_CACHE_DIR := {{ .PathToRepoRoot }}/.terraform.d/plugin-cache
-export AWS_BACKEND_PROFILE := {{ .Backend.Profile }}
+export AWS_BACKEND_PROFILE := {{ .Backend.S3.Profile }}
 {{if .Providers.AWS}}
 export AWS_PROVIDER_PROFILE := {{ .Providers.AWS.Profile }}
 {{ end }}
diff --git a/templates/global/fogg.tf.tmpl b/templates/global/fogg.tf.tmpl
index edece059..f2fb5ab8 100644
--- a/templates/global/fogg.tf.tmpl
+++ b/templates/global/fogg.tf.tmpl
@@ -51,8 +51,8 @@ terraform {
   required_version = "~>{{ .TerraformVersion }}"
 
   backend "s3" {
-    bucket         = "{{ .Backend.Bucket }}"
-    {{ if .Backend.DynamoTable }}dynamodb_table = "{{ .Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ .Backend.S3.Bucket }}"
+    {{ if .Backend.S3.DynamoTable }}dynamodb_table = "{{ .Backend.S3.DynamoTable }}"{{ end }}
     {{/* {%- if env is defined and component_name is defined %} */}}
     key    = "terraform/{{ .Project }}/{{ .Component }}.tfstate"
 {{/*
@@ -61,8 +61,8 @@ terraform {
     {%- endif %}
 */}}
     encrypt = true
-    region = "{{ .Backend.Region }}"
-    profile = "{{ .Backend.Profile }}"
+    region = "{{ .Backend.S3.Region }}"
+    profile = "{{ .Backend.S3.Profile }}"
   }
 }
 
@@ -124,11 +124,11 @@ data "terraform_remote_state" "{{ $component }}" {
   backend = "s3"
 
   config = {
-    bucket         = "{{ $outer.Backend.Bucket }}"
-    {{ if .Backend.DynamoTable }}dynamodb_table = "{{ .Backend.DynamoTable }}"{{ end }}
+    bucket         = "{{ $outer.Backend.S3.Bucket }}"
+    {{ if .Backend.S3.DynamoTable }}dynamodb_table = "{{ .Backend.S3.DynamoTable }}"{{ end }}
     key            = "terraform/{{ $.Project }}/envs/{{ $outer.Env }}/components/{{ $component }}.tfstate"
-    region         = "{{ $outer.Backend.Region }}"
-    profile        = "{{ $outer.Backend.Profile }}"
+    region         = "{{ $outer.Backend.S3.Region }}"
+    profile        = "{{ $outer.Backend.S3.Profile }}"
   }
 }
 {{ end }}


Test patch:
diff --git a/config/v2/validation_test.go b/config/v2/validation_test.go
index 9c361448..a297eeb7 100644
--- a/config/v2/validation_test.go
+++ b/config/v2/validation_test.go
@@ -76,6 +76,15 @@ func TestValidateOwnersComponent(t *testing.T) {
 	}
 }
 
+func TestValidateBackend(t *testing.T) {
+	r := require.New(t)
+
+	c := confBackendType()
+
+	_, err := c.Validate()
+	r.Error(err)
+}
+
 func confAcctOwner(def, acct string) Config {
 	return Config{
 		Defaults: Defaults{
@@ -103,6 +112,25 @@ func confAcctOwner(def, acct string) Config {
 	}
 }
 
+func confBackendType() Config {
+	return Config{
+		Version: 2,
+		Defaults: Defaults{
+			Common{
+				Owner: util.StrPtr("foo@example.com"),
+				Backend: &Backend{
+					Type:    util.StrPtr("invalid"),
+					Bucket:  util.StrPtr("foo"),
+					Region:  util.StrPtr("foo"),
+					Profile: util.StrPtr("foo"),
+				},
+				Project:          util.StrPtr("foo"),
+				TerraformVersion: util.StrPtr("1.1.1"),
+			},
+		},
+	}
+}
+
 func confComponentOwner(def, env, component string) Config {
 	return Config{
 		Defaults: Defaults{