Repository: chanzuckerberg/fogg
Pull request #410 (2020-03-03 23:09:12)

[feature] GitHub Actions CI integration
### Summary
Enables us to configure GitHub actions ci. Still missing buildevents integration. 

### Test Plan
tested in internal repos

### References

codecov[bot]: # [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/410?src=pr&el=h1) Report
> Merging [#410](https://codecov.io/gh/chanzuckerberg/fogg/pull/410?src=pr&el=desc) into [master](https://codecov.io/gh/chanzuckerberg/fogg/commit/560cfc3f8676099564a7712cc83deda750a8910c&el=desc) will **decrease** coverage by `0.17%`.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/chanzuckerberg/fogg/pull/410/graphs/tree.svg?width=650&height=150&src=pr&token=57DhtZ7g2A)](https://codecov.io/gh/chanzuckerberg/fogg/pull/410?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #410      +/-   ##
==========================================
- Coverage   78.34%   78.17%   -0.18%     
==========================================
  Files          24       24              
  Lines        1926     2007      +81     
==========================================
+ Hits         1509     1569      +60     
- Misses        276      291      +15     
- Partials      141      147       +6     
```



------

[Continue to review full report at Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/410?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/410?src=pr&el=footer). Last update [560cfc3...0172ce6](https://codecov.io/gh/chanzuckerberg/fogg/pull/410?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/apply/apply.go b/apply/apply.go
index 825b79c5..95ad75ae 100644
--- a/apply/apply.go
+++ b/apply/apply.go
@@ -63,6 +63,13 @@ func Apply(fs afero.Fs, conf *v2.Config, tmp *templates.T, upgrade bool) error {
 		}
 	}
 
+	if p.GitHubActionsCI.Enabled {
+		e = applyTree(fs, &tmp.GitHubActionsCI, &tmp.Common, ".github", p.GitHubActionsCI)
+		if e != nil {
+			return errs.WrapUser(e, "unable to apply GitHub Actions CI")
+		}
+	}
+
 	e = applyAccounts(fs, p, &tmp.Account, &tmp.Common)
 	if e != nil {
 		return errs.WrapUser(e, "unable to apply accounts")
diff --git a/config/v2/config.go b/config/v2/config.go
index 9eabd7cd..8542fa35 100644
--- a/config/v2/config.go
+++ b/config/v2/config.go
@@ -68,9 +68,10 @@ type Account struct {
 }
 
 type Tools struct {
-	TravisCI *v1.TravisCI `json:"travis_ci,omitempty" yaml:"travis_ci,omitempty"`
-	CircleCI *CircleCI    `json:"circle_ci,omitempty" yaml:"circle_ci,omitempty"`
-	TfLint   *v1.TfLint   `json:"tflint,omitempty" yaml:"tflint,omitempty"`
+	TravisCI        *v1.TravisCI     `json:"travis_ci,omitempty" yaml:"travis_ci,omitempty"`
+	CircleCI        *CircleCI        `json:"circle_ci,omitempty" yaml:"circle_ci,omitempty"`
+	GitHubActionsCI *GitHubActionsCI `json:"github_actions_ci,omitempty" yaml:"github_actions_ci,omitempty"`
+	TfLint          *v1.TfLint       `json:"tflint,omitempty" yaml:"tflint,omitempty"`
 }
 
 type CircleCI struct {
@@ -79,6 +80,10 @@ type CircleCI struct {
 	SSHKeyFingerprints []string `json:"ssh_key_fingerprints" yaml:"ssh_key_fingerprints"`
 }
 
+type GitHubActionsCI struct {
+	v1.CommonCI `json:",inline" yaml:",inline"`
+}
+
 type Env struct {
 	Common `json:",inline" yaml:",inline"`
 
diff --git a/config/v2/resolvers.go b/config/v2/resolvers.go
index 4930bd5a..8ce379fa 100644
--- a/config/v2/resolvers.go
+++ b/config/v2/resolvers.go
@@ -248,6 +248,33 @@ func ResolveTravis(commons ...Common) *v1.TravisCI {
 	}
 }
 
+func ResolveGitHubActionsCI(commons ...Common) *GitHubActionsCI {
+	enabled := false
+	buildevents := false
+	testCommand := "check"
+	for _, c := range commons {
+		if c.Tools != nil && c.Tools.GitHubActionsCI != nil && c.Tools.GitHubActionsCI.Enabled != nil {
+			enabled = *c.Tools.GitHubActionsCI.Enabled
+		}
+		if c.Tools != nil && c.Tools.GitHubActionsCI != nil && c.Tools.GitHubActionsCI.Command != nil {
+			testCommand = *c.Tools.GitHubActionsCI.Command
+		}
+		if c.Tools != nil && c.Tools.GitHubActionsCI != nil && c.Tools.GitHubActionsCI.Buildevents != nil {
+			buildevents = *c.Tools.GitHubActionsCI.Buildevents
+		}
+	}
+
+	roleName := lastNonNil(GitHubActionsRoleNameGetter, commons...)
+	return &GitHubActionsCI{
+		CommonCI: v1.CommonCI{
+			Enabled:        &enabled,
+			Buildevents:    &buildevents,
+			AWSIAMRoleName: roleName,
+			Command:        &testCommand,
+		},
+	}
+}
+
 func ResolveCircleCI(commons ...Common) *CircleCI {
 	enabled := false
 	buildevents := false
@@ -467,6 +494,13 @@ func TravisRoleNameGetter(comm Common) *string {
 	return comm.Tools.TravisCI.AWSIAMRoleName
 }
 
+func GitHubActionsRoleNameGetter(comm Common) *string {
+	if comm.Tools == nil || comm.Tools.GitHubActionsCI == nil {
+		return nil
+	}
+	return comm.Tools.GitHubActionsCI.AWSIAMRoleName
+}
+
 func CircleCIRoleNameGetter(comm Common) *string {
 	if comm.Tools == nil || comm.Tools.CircleCI == nil {
 		return nil
diff --git a/plan/ci.go b/plan/ci.go
index 73577b74..8476c1e0 100644
--- a/plan/ci.go
+++ b/plan/ci.go
@@ -27,6 +27,10 @@ type CircleCIConfig struct {
 	SSHFingerprints []string
 }
 
+type GitHubActionsCIConfig struct {
+	CIConfig
+}
+
 type TravisCIConfig struct {
 	CIConfig
 }
@@ -216,3 +220,67 @@ func (p *Plan) buildCircleCIConfig(c *v2.Config, foggVersion string) CircleCICon
 		SSHFingerprints: sshFingerprints,
 	}
 }
+
+func (p *Plan) buildGitHubActionsConfig(c *v2.Config, foggVersion string) GitHubActionsCIConfig {
+	ciConfig := &CIConfig{
+		FoggVersion: foggVersion,
+	}
+
+	globalConfig := p.Global.GitHubActionsCI.generateCIConfig(
+		p.Global.Backend,
+		p.Global.Providers.AWS,
+		"global",
+		"terraform/global")
+	ciConfig = ciConfig.merge(globalConfig)
+
+	for name, acct := range p.Accounts {
+		accountConfig := acct.GitHubActionsCI.generateCIConfig(
+			acct.Backend,
+			acct.Providers.AWS,
+			fmt.Sprintf("accounts/%s", name),
+			fmt.Sprintf("terraform/accounts/%s", name),
+		)
+		ciConfig = ciConfig.merge(accountConfig)
+	}
+
+	for envName, env := range p.Envs {
+		for cName, c := range env.Components {
+			envConfig := c.GitHubActionsCI.generateCIConfig(
+				c.Backend,
+				c.Providers.AWS,
+				fmt.Sprintf("%s/%s", envName, cName),
+				fmt.Sprintf("terraform/envs/%s/%s", envName, cName),
+			)
+			ciConfig = ciConfig.merge(envConfig)
+		}
+	}
+
+	for moduleName := range p.Modules {
+		proj := CIProject{
+			Name:    fmt.Sprintf("modules/%s", moduleName),
+			Dir:     fmt.Sprintf("terraform/modules/%s", moduleName),
+			Command: "check",
+		}
+		ciConfig = ciConfig.addProjects(proj)
+	}
+
+	numBuckets := 1
+	if c.Defaults.Tools != nil && c.Defaults.Tools.GitHubActionsCI != nil {
+
+		if c.Defaults.Tools.GitHubActionsCI.TestBuckets != nil &&
+			*c.Defaults.Tools.GitHubActionsCI.TestBuckets > 0 {
+			numBuckets = *c.Defaults.Tools.GitHubActionsCI.TestBuckets
+		}
+
+		// If aws is disabled, reset the providers
+		aws, ok := c.Defaults.Tools.GitHubActionsCI.Providers["aws"]
+		if ok && aws.Disabled {
+			ciConfig.AWSProfiles = ciAwsProfiles{}
+		}
+	}
+
+	ciConfig = ciConfig.populateBuckets(numBuckets)
+	return GitHubActionsCIConfig{
+		CIConfig: *ciConfig,
+	}
+}
diff --git a/plan/plan.go b/plan/plan.go
index 9ea8a55b..9fbac107 100644
--- a/plan/plan.go
+++ b/plan/plan.go
@@ -15,13 +15,14 @@ import (
 
 // Plan represents a set of actions to take
 type Plan struct {
-	Accounts map[string]Account `json:"account" yaml:"account"`
-	Envs     map[string]Env     `json:"envs" yaml:"envs"`
-	Global   Component          `json:"global" yaml:"global"`
-	Modules  map[string]Module  `json:"modules" yaml:"modules"`
-	TravisCI TravisCIConfig     `json:"travis_ci" yaml:"travis_ci"`
-	CircleCI CircleCIConfig     `json:"circle_ci" yaml:"circle_ci"`
-	Version  string             `json:"version" yaml:"version"`
+	Accounts        map[string]Account    `json:"account" yaml:"account"`
+	Envs            map[string]Env        `json:"envs" yaml:"envs"`
+	Global          Component             `json:"global" yaml:"global"`
+	Modules         map[string]Module     `json:"modules" yaml:"modules"`
+	TravisCI        TravisCIConfig        `json:"travis_ci" yaml:"travis_ci"`
+	CircleCI        CircleCIConfig        `json:"github_actions_ci" yaml:"github_actions_ci"`
+	GitHubActionsCI GitHubActionsCIConfig `json:"circle_ci" yaml:"circle_ci"`
+	Version         string                `json:"version" yaml:"version"`
 }
 
 // Common represents common fields
@@ -41,8 +42,9 @@ type ComponentCommon struct {
 	Providers Providers         `json:"providers" yaml:"providers"`
 	TfLint    TfLint            `json:"tf_lint" yaml:"tf_lint"`
 
-	TravisCI TravisCIComponent
-	CircleCI CircleCIComponent
+	TravisCI        TravisCIComponent
+	CircleCI        CircleCIComponent
+	GitHubActionsCI GitHubActionsComponent
 }
 
 type TravisCIComponent struct {
@@ -55,6 +57,10 @@ type CircleCIComponent struct {
 	SSHFingerprints []string
 }
 
+type GitHubActionsComponent struct {
+	CIComponent
+}
+
 type CIComponent struct {
 	Enabled     bool
 	Buildevents bool
@@ -230,6 +236,7 @@ func Eval(c *v2.Config) (*Plan, error) {
 	p.Modules = p.buildModules(c)
 	p.TravisCI = p.buildTravisCIConfig(c, v)
 	p.CircleCI = p.buildCircleCIConfig(c, v)
+	p.GitHubActionsCI = p.buildGitHubActionsConfig(c, v)
 
 	return p, nil
 }
@@ -424,6 +431,18 @@ func resolveComponentCommon(commons ...v2.Common) ComponentCommon {
 		travisPlan.Command = *travisConfig.Command
 	}
 
+	githubActionsConfig := v2.ResolveGitHubActionsCI(commons...)
+	githubActionsPlan := GitHubActionsComponent{
+		CIComponent: CIComponent{
+			Enabled:     *githubActionsConfig.Enabled,
+			Buildevents: *githubActionsConfig.Buildevents,
+		},
+	}
+	if githubActionsPlan.Enabled {
+		githubActionsPlan.AWSRoleName = *githubActionsConfig.AWSIAMRoleName
+		githubActionsPlan.Command = *githubActionsConfig.Command
+	}
+
 	circleConfig := v2.ResolveCircleCI(commons...)
 	circlePlan := CircleCIComponent{
 		CIComponent: CIComponent{
@@ -457,13 +476,14 @@ func resolveComponentCommon(commons ...v2.Common) ComponentCommon {
 			Okta:      oktaPlan,
 			Datadog:   datadogPlan,
 		},
-		TfLint:    tfLintPlan,
-		ExtraVars: v2.ResolveStringMap(v2.ExtraVarsGetter, commons...),
-		Owner:     v2.ResolveRequiredString(v2.OwnerGetter, commons...),
-		Project:   v2.ResolveRequiredString(v2.ProjectGetter, commons...),
-		Common:    Common{TerraformVersion: v2.ResolveRequiredString(v2.TerraformVersionGetter, commons...)},
-		TravisCI:  travisPlan,
-		CircleCI:  circlePlan,
+		TfLint:          tfLintPlan,
+		ExtraVars:       v2.ResolveStringMap(v2.ExtraVarsGetter, commons...),
+		Owner:           v2.ResolveRequiredString(v2.OwnerGetter, commons...),
+		Project:         v2.ResolveRequiredString(v2.ProjectGetter, commons...),
+		Common:          Common{TerraformVersion: v2.ResolveRequiredString(v2.TerraformVersionGetter, commons...)},
+		TravisCI:        travisPlan,
+		CircleCI:        circlePlan,
+		GitHubActionsCI: githubActionsPlan,
 	}
 }
 
diff --git a/templates/.github/workflows/fogg_ci.yml.tmpl b/templates/.github/workflows/fogg_ci.yml.tmpl
new file mode 100644
index 00000000..ef3098cf
--- /dev/null
+++ b/templates/.github/workflows/fogg_ci.yml.tmpl
@@ -0,0 +1,45 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+{{- $githubActionsCI := . }}
+
+on: push
+
+jobs:
+{{- range $idx, $testBucket := $githubActionsCI.TestBuckets }}
+  terraform_{{$idx}}:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+      - run: python3 -m venv venv
+      - run: . venv/bin/activate
+      - run: pip install awscli --upgrade --user
+      - run: aws --version
+      - run: FOGG_VERSION="v{{ $githubActionsCI.FoggVersion }}" make setup
+      - run: |
+          .fogg/bin/fogg apply
+          if [ "$(git status --porcelain)" != "" ]; then
+            echo "git tree is dirty after running fogg apply"
+            echo "ensure you run fogg apply on your branch"
+            git status
+            exit 1
+          fi
+{{- if not (eq (len $githubActionsCI.AWSProfiles) 0) }}
+      - run: aws configure set aws_access_key_id     $IDACCT_AWS_ACCESS_KEY_ID     --profile _idacct
+      - run: aws configure set aws_secret_access_key $IDACCT_AWS_SECRET_ACCESS_KEY --profile _idacct
+      - run: aws --profile _idacct sts get-caller-identity
+{{- end }}
+{{- range $profileName, $profile := $githubActionsCI.AWSProfiles }}
+      - run: aws configure set profile.{{ $profileName }}.role_arn arn:aws:iam::{{ $profile.AccountID }}:role/{{ $profile.RoleName}}
+      - run: aws configure set profile.{{ $profileName }}.source_profile _idacct
+      - run: aws --profile {{ $profileName }} sts get-caller-identity
+{{- end }}
+  {{- range $component := $testBucket }}
+      - run: |
+          cd {{ $component.Dir }}
+          make  {{ $component.Command }}
+  {{- end }}
+{{- end }}
diff --git a/templates/templates.go b/templates/templates.go
index d174a87a..26f8c660 100644
--- a/templates/templates.go
+++ b/templates/templates.go
@@ -16,6 +16,7 @@ type T struct {
 	Repo             packr.Box
 	TravisCI         packr.Box
 	CircleCI         packr.Box
+	GitHubActionsCI  packr.Box
 }
 
 var Templates = &T{
@@ -32,4 +33,5 @@ var Templates = &T{
 	Repo:             packr.NewBox("repo"),
 	TravisCI:         packr.NewBox("travis-ci"),
 	CircleCI:         packr.NewBox("circleci"),
+	GitHubActionsCI:  packr.NewBox(".github"),
 }


Test patch:
diff --git a/plan/plan_test.go b/plan/plan_test.go
index 93b0cd08..52d9f94e 100644
--- a/plan/plan_test.go
+++ b/plan/plan_test.go
@@ -109,6 +109,7 @@ func TestPlanBasicV2(t *testing.T) {
 	assert.NoError(t, e)
 
 	fs, _, err := util.TestFs()
+	a.NoError(err)
 	err = afero.WriteFile(fs, "fogg.json", b, 0644)
 	a.NoError(err)
 
diff --git a/testdata/v2_full/.github/workflows/fogg_ci.yml b/testdata/v2_full/.github/workflows/fogg_ci.yml
new file mode 100644
index 00000000..ea0b5cbc
--- /dev/null
+++ b/testdata/v2_full/.github/workflows/fogg_ci.yml
@@ -0,0 +1,60 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+
+on: push
+
+jobs:
+  terraform_0:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+      - run: python3 -m venv venv
+      - run: . venv/bin/activate
+      - run: pip install awscli --upgrade --user
+      - run: aws --version
+      - run: FOGG_VERSION="vundefined+undefined+dirty" make setup
+      - run: |
+          .fogg/bin/fogg apply
+          if [ "$(git status --porcelain)" != "" ]; then
+            echo "git tree is dirty after running fogg apply"
+            echo "ensure you run fogg apply on your branch"
+            git status
+            exit 1
+          fi
+      - run: aws configure set aws_access_key_id     $IDACCT_AWS_ACCESS_KEY_ID     --profile _idacct
+      - run: aws configure set aws_secret_access_key $IDACCT_AWS_SECRET_ACCESS_KEY --profile _idacct
+      - run: aws --profile _idacct sts get-caller-identity
+      - run: aws configure set profile.comp1.role_arn arn:aws:iam::1234:role/testo-role
+      - run: aws configure set profile.comp1.source_profile _idacct
+      - run: aws --profile comp1 sts get-caller-identity
+      - run: aws configure set profile.profile.role_arn arn:aws:iam::456:role/testo-role
+      - run: aws configure set profile.profile.source_profile _idacct
+      - run: aws --profile profile sts get-caller-identity
+      - run: |
+          cd terraform/accounts/bar
+          make  check
+      - run: |
+          cd terraform/accounts/foo
+          make  check
+      - run: |
+          cd terraform/global
+          make  check
+      - run: |
+          cd terraform/modules/my_module
+          make  check
+      - run: |
+          cd terraform/envs/staging/comp1
+          make  check
+      - run: |
+          cd terraform/envs/staging/comp2
+          make  check
+      - run: |
+          cd terraform/envs/staging/comp_helm_template
+          make  check
+      - run: |
+          cd terraform/envs/staging/vpc
+          make  check
diff --git a/testdata/v2_full/fogg.json b/testdata/v2_full/fogg.json
index dbb5b2cd..44fc35af 100644
--- a/testdata/v2_full/fogg.json
+++ b/testdata/v2_full/fogg.json
@@ -30,6 +30,10 @@
       "circle_ci": {
         "enabled": true,
         "aws_iam_role_name": "foobar-role"
+      },
+      "github_actions_ci": {
+        "enabled": true,
+        "aws_iam_role_name": "testo-role"
       }
     }
   },