Repository: chanzuckerberg/fogg
Pull request #483 (2020-06-08 22:31:55)

[fix] simplify and optimize .terraformignore
Make the configuration simpler and 10x faster at building slugs.

This is faster because the process seems to be CPU bound when you have a
large number of patterns in your .terraformignore.

And slug sizes are smaller because we now exclude modules directorties.

### Summary
A description of changes or issues addressed by this PR. Provide links to relevant PRs if any.

### Test Plan

Tested on shared-infra and runtime when from 6m -> 12s and slug size
from 75MB to 5MB.


### References
* library that builds the slugs - https://github.com/hashicorp/go-slug

codecov-commenter: # [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=h1) Report
> Merging [#483](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=desc) into [master](https://codecov.io/gh/chanzuckerberg/fogg/commit/fd7837aa80706108b2047f903c2ca0dd02269183&el=desc) will **decrease** coverage by `0.58%`.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/chanzuckerberg/fogg/pull/483/graphs/tree.svg?width=650&height=150&src=pr&token=57DhtZ7g2A)](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #483      +/-   ##
==========================================
- Coverage   59.66%   59.08%   -0.59%     
==========================================
  Files          19       19              
  Lines        1996     1982      -14     
==========================================
- Hits         1191     1171      -20     
- Misses        648      653       +5     
- Partials      157      158       +1     
```


| [Impacted Files](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [plan/plan.go](https://codecov.io/gh/chanzuckerberg/fogg/pull/483/diff?src=pr&el=tree#diff-cGxhbi9wbGFuLmdv) | `84.19% <ø> (-2.54%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=footer). Last update [fd7837a...e224f78](https://codecov.io/gh/chanzuckerberg/fogg/pull/483?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/plan/plan.go b/plan/plan.go
index 801b34c3..762029ac 100644
--- a/plan/plan.go
+++ b/plan/plan.go
@@ -22,7 +22,6 @@ type Plan struct {
 	CircleCI        CircleCIConfig        `yaml:"github_actions_ci"`
 	GitHubActionsCI GitHubActionsCIConfig `yaml:"circle_ci"`
 	Version         string                `yaml:"version"`
-	TerraformIgnore []string              `yaml:"terraform_ignore"`
 }
 
 // Common represents common fields
@@ -280,7 +279,6 @@ func Eval(c *v2.Config) (*Plan, error) {
 	p.TravisCI = p.buildTravisCIConfig(c, v)
 	p.CircleCI = p.buildCircleCIConfig(c, v)
 	p.GitHubActionsCI = p.buildGitHubActionsConfig(c, v)
-	p.TerraformIgnore = p.buildTerraformIgnore()
 
 	return p, nil
 }
@@ -680,27 +678,3 @@ func resolveAccounts(accounts map[string]v2.Account) map[string]*json.Number {
 	}
 	return a
 }
-
-func (p *Plan) buildTerraformIgnore() []string {
-	// start with a static list that applies to every repository
-	ignores := []string{
-		".git/",
-		".fogg/",
-		".terraform.d/",
-		"terraform/global/.terraform/plugins/",
-	}
-
-	for a := range p.Accounts {
-		ignores = append(ignores, fmt.Sprintf("terraform/accounts/%s/.terraform/plugins/", a))
-	}
-
-	for envName, env := range p.Envs {
-		for componentName, component := range env.Components {
-			if component.Kind.GetOrDefault() == v2.ComponentKindTerraform {
-				ignores = append(ignores, fmt.Sprintf("terraform/envs/%s/%s/.terraform/plugins/", envName, componentName))
-			}
-		}
-	}
-
-	return ignores
-}
diff --git a/templates/repo/.terraformignore.tmpl b/templates/repo/.terraformignore.tmpl
index f4040d77..7a58732c 100644
--- a/templates/repo/.terraformignore.tmpl
+++ b/templates/repo/.terraformignore.tmpl
@@ -1,4 +1,7 @@
 {{ template "fogg_header" }}
-{{ range $i := .TerraformIgnore | sortAlpha }}
-{{ $i }}{{ end }}
 
+.fogg/
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**


Test patch:
diff --git a/plan/plan_test.go b/plan/plan_test.go
index 2cac30dc..78124f5e 100644
--- a/plan/plan_test.go
+++ b/plan/plan_test.go
@@ -176,30 +176,6 @@ func buildPlan(t *testing.T, testfile string) *Plan {
 	return plan
 }
 
-func Test_buildTerraformIgnore(t *testing.T) {
-	r := require.New(t)
-
-	p := buildPlan(t, "v2_full_yaml")
-
-	r.NotNil(p)
-
-	expected := []string{
-		".git/",
-		".fogg/",
-		".terraform.d/",
-		"terraform/global/.terraform/plugins/",
-		"terraform/accounts/foo/.terraform/plugins/",
-		"terraform/accounts/bar/.terraform/plugins/",
-		"terraform/envs/prod/hero/.terraform/plugins/",
-		"terraform/envs/prod/datadog/.terraform/plugins/",
-		"terraform/envs/staging/comp1/.terraform/plugins/",
-		"terraform/envs/staging/comp2/.terraform/plugins/",
-		"terraform/envs/staging/vpc/.terraform/plugins/",
-	}
-
-	r.ElementsMatch(expected, p.TerraformIgnore)
-}
-
 func TestTfeProvider(t *testing.T) {
 	r := require.New(t)
 
diff --git a/testdata/bless_provider_yaml/.terraformignore b/testdata/bless_provider_yaml/.terraformignore
index 1402663d..60a3f017 100644
--- a/testdata/bless_provider_yaml/.terraformignore
+++ b/testdata/bless_provider_yaml/.terraformignore
@@ -3,9 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/accounts/foo/.terraform/plugins/
-terraform/envs/bar/bam/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/circleci/.terraformignore b/testdata/circleci/.terraformignore
index cb811c70..60a3f017 100644
--- a/testdata/circleci/.terraformignore
+++ b/testdata/circleci/.terraformignore
@@ -3,7 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/github_actions/.terraformignore b/testdata/github_actions/.terraformignore
index cb811c70..60a3f017 100644
--- a/testdata/github_actions/.terraformignore
+++ b/testdata/github_actions/.terraformignore
@@ -3,7 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/github_provider_yaml/.terraformignore b/testdata/github_provider_yaml/.terraformignore
index 1402663d..60a3f017 100644
--- a/testdata/github_provider_yaml/.terraformignore
+++ b/testdata/github_provider_yaml/.terraformignore
@@ -3,9 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/accounts/foo/.terraform/plugins/
-terraform/envs/bar/bam/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/okta_provider_yaml/.terraformignore b/testdata/okta_provider_yaml/.terraformignore
index 1402663d..60a3f017 100644
--- a/testdata/okta_provider_yaml/.terraformignore
+++ b/testdata/okta_provider_yaml/.terraformignore
@@ -3,9 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/accounts/foo/.terraform/plugins/
-terraform/envs/bar/bam/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/snowflake_provider_yaml/.terraformignore b/testdata/snowflake_provider_yaml/.terraformignore
index 1402663d..60a3f017 100644
--- a/testdata/snowflake_provider_yaml/.terraformignore
+++ b/testdata/snowflake_provider_yaml/.terraformignore
@@ -3,9 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/accounts/foo/.terraform/plugins/
-terraform/envs/bar/bam/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/tfe_provider_yaml/.terraformignore b/testdata/tfe_provider_yaml/.terraformignore
index 1402663d..60a3f017 100644
--- a/testdata/tfe_provider_yaml/.terraformignore
+++ b/testdata/tfe_provider_yaml/.terraformignore
@@ -3,9 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/accounts/foo/.terraform/plugins/
-terraform/envs/bar/bam/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/v2_full_yaml/.terraformignore b/testdata/v2_full_yaml/.terraformignore
index 5f2f70e4..60a3f017 100644
--- a/testdata/v2_full_yaml/.terraformignore
+++ b/testdata/v2_full_yaml/.terraformignore
@@ -3,14 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/accounts/bar/.terraform/plugins/
-terraform/accounts/foo/.terraform/plugins/
-terraform/envs/prod/datadog/.terraform/plugins/
-terraform/envs/prod/hero/.terraform/plugins/
-terraform/envs/staging/comp1/.terraform/plugins/
-terraform/envs/staging/comp2/.terraform/plugins/
-terraform/envs/staging/vpc/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/v2_minimal_valid_yaml/.terraformignore b/testdata/v2_minimal_valid_yaml/.terraformignore
index cb811c70..60a3f017 100644
--- a/testdata/v2_minimal_valid_yaml/.terraformignore
+++ b/testdata/v2_minimal_valid_yaml/.terraformignore
@@ -3,7 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**
diff --git a/testdata/v2_no_aws_provider_yaml/.terraformignore b/testdata/v2_no_aws_provider_yaml/.terraformignore
index b10c1cc4..60a3f017 100644
--- a/testdata/v2_no_aws_provider_yaml/.terraformignore
+++ b/testdata/v2_no_aws_provider_yaml/.terraformignore
@@ -3,8 +3,7 @@
 
 
 .fogg/
-.git/
-.terraform.d/
-terraform/envs/staging/vpc/.terraform/plugins/
-terraform/global/.terraform/plugins/
-
+**/terraform.d/**
+**/.terraform.d/**
+**/.terraform/**
+**/.git/**