Repository: chanzuckerberg/fogg
Pull request #500 (2020-10-09 03:53:31)

[feature] set env variables for github actions jobs
Add new config option for setting environment variales in github actions.

Also add some golden-file testing and fix a few small issues.

### Test Plan
* CI

### References
* https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#env


Patch:
diff --git a/config/v2/config.go b/config/v2/config.go
index 02e6f2dc..84b4e8dd 100644
--- a/config/v2/config.go
+++ b/config/v2/config.go
@@ -229,6 +229,7 @@ type CommonCI struct {
 	Command        *string                     `yaml:"command,omitempty"`
 	Buildevents    *bool                       `yaml:"buildevents,omitempty"`
 	Providers      map[string]CIProviderConfig `yaml:"providers,omitempty"`
+	Env            map[string]string           `yaml:"env,omitempty"`
 }
 
 type DependsOn struct {
diff --git a/config/v2/resolvers.go b/config/v2/resolvers.go
index 1f651de7..c352f4eb 100644
--- a/config/v2/resolvers.go
+++ b/config/v2/resolvers.go
@@ -391,15 +391,18 @@ func ResolveGitHubActionsCI(commons ...Common) *GitHubActionsCI {
 	enabled := false
 	buildevents := false
 	testCommand := "check"
+
 	for _, c := range commons {
-		if c.Tools != nil && c.Tools.GitHubActionsCI != nil && c.Tools.GitHubActionsCI.Enabled != nil {
-			enabled = *c.Tools.GitHubActionsCI.Enabled
-		}
-		if c.Tools != nil && c.Tools.GitHubActionsCI != nil && c.Tools.GitHubActionsCI.Command != nil {
-			testCommand = *c.Tools.GitHubActionsCI.Command
-		}
-		if c.Tools != nil && c.Tools.GitHubActionsCI != nil && c.Tools.GitHubActionsCI.Buildevents != nil {
-			buildevents = *c.Tools.GitHubActionsCI.Buildevents
+		if c.Tools != nil && c.Tools.GitHubActionsCI != nil {
+			if c.Tools.GitHubActionsCI.Enabled != nil {
+				enabled = *c.Tools.GitHubActionsCI.Enabled
+			}
+			if c.Tools.GitHubActionsCI.Command != nil {
+				testCommand = *c.Tools.GitHubActionsCI.Command
+			}
+			if c.Tools.GitHubActionsCI.Buildevents != nil {
+				buildevents = *c.Tools.GitHubActionsCI.Buildevents
+			}
 		}
 	}
 
diff --git a/config/v2/validation.go b/config/v2/validation.go
index 60d3d382..f291c41f 100644
--- a/config/v2/validation.go
+++ b/config/v2/validation.go
@@ -11,7 +11,7 @@ import (
 	validator "gopkg.in/go-playground/validator.v9"
 )
 
-var validTravisCommands = map[string]struct{}{
+var validCICommands = map[string]struct{}{
 	"check": {},
 	"lint":  {},
 }
@@ -51,6 +51,7 @@ func (c *Config) Validate() ([]string, error) {
 	errs = multierror.Append(errs, c.ValidateOktaProviders())
 	errs = multierror.Append(errs, c.validateModules())
 	errs = multierror.Append(errs, c.ValidateTravis())
+	errs = multierror.Append(errs, c.ValidateGithubActionsCI())
 
 	// refactor to make it easier to manage these
 	w, e := c.ValidateToolsTfLint()
@@ -262,7 +263,7 @@ func (c *Config) ValidateTravis() error {
 		}
 
 		if t.Command != nil {
-			_, ok := validTravisCommands[*t.Command]
+			_, ok := validCICommands[*t.Command]
 			if !ok {
 				errs = multierror.Append(errs, fmt.Errorf("unrecognized travisci command %s (%s)", *t.Command, component))
 			}
@@ -272,6 +273,29 @@ func (c *Config) ValidateTravis() error {
 	return errs
 }
 
+func (c *Config) ValidateGithubActionsCI() error {
+	var errs *multierror.Error
+	c.WalkComponents(func(component string, comms ...Common) {
+		t := ResolveGitHubActionsCI(comms...)
+		if t.Enabled == nil || !*t.Enabled {
+			return // nothing to do
+		}
+
+		if t.AWSIAMRoleName == nil || *t.AWSIAMRoleName == "" {
+			errs = multierror.Append(errs, fmt.Errorf("if github_actions_ci is enabled, aws_role_name must be set"))
+		}
+
+		if t.Command != nil {
+			_, ok := validCICommands[*t.Command]
+			if !ok {
+				errs = multierror.Append(errs, fmt.Errorf("unrecognized github_actions_ci command %s (%s)", *t.Command, component))
+			}
+		}
+	})
+
+	return errs
+}
+
 func (c *Config) ValidateToolsTfLint() ([]string, error) {
 	var warns []string
 	var errs *multierror.Error
diff --git a/plan/ci.go b/plan/ci.go
index 79bab169..8f6e640d 100644
--- a/plan/ci.go
+++ b/plan/ci.go
@@ -15,6 +15,7 @@ type CIProject struct {
 type CIConfig struct {
 	Enabled     bool
 	FoggVersion string
+	Env         map[string]string
 	projects    []CIProject
 
 	TestBuckets [][]CIProject
@@ -256,8 +257,16 @@ func (p *Plan) buildCircleCIConfig(c *v2.Config, foggVersion string) CircleCICon
 }
 
 func (p *Plan) buildGitHubActionsConfig(c *v2.Config, foggVersion string) GitHubActionsCIConfig {
+
+	var env map[string]string
+
+	if c.Defaults.Tools != nil && c.Defaults.Tools.GitHubActionsCI != nil {
+		env = c.Defaults.Tools.GitHubActionsCI.Env
+	}
+
 	ciConfig := &CIConfig{
 		FoggVersion: foggVersion,
+		Env:         env,
 	}
 
 	var awsProvider v2.CIProviderConfig
diff --git a/templates/.github/workflows/fogg_ci.yml.tmpl b/templates/.github/workflows/fogg_ci.yml.tmpl
index 2fb80700..30fb5c0a 100644
--- a/templates/.github/workflows/fogg_ci.yml.tmpl
+++ b/templates/.github/workflows/fogg_ci.yml.tmpl
@@ -4,6 +4,13 @@
 
 on: push
 
+{{- if $githubActionsCI.Env }}
+env:
+{{- range $k, $v := .Env }}
+    {{ $k }}: {{ $v }}
+{{- end }}
+{{- end }}
+
 jobs:
 {{- range $idx, $testBucket := $githubActionsCI.TestBuckets }}
   terraform_{{$idx}}:


Test patch:
diff --git a/plan/ci_test.go b/plan/ci_test.go
index 64f00fa1..55670c9e 100644
--- a/plan/ci_test.go
+++ b/plan/ci_test.go
@@ -80,7 +80,7 @@ func Test_buildTravisCI_Profiles(t *testing.T) {
 			},
 		},
 		Accounts: map[string]v2.Account{
-			"foo": v2.Account{
+			"foo": {
 				Common: v2.Common{Providers: &v2.Providers{AWS: &v2.AWSProvider{AccountID: &id1}}},
 			},
 		},
@@ -135,10 +135,10 @@ func Test_buildTravisCI_TestBuckets(t *testing.T) {
 			},
 		},
 		Accounts: map[string]v2.Account{
-			"foo": v2.Account{
+			"foo": {
 				Common: v2.Common{Providers: &v2.Providers{AWS: &v2.AWSProvider{AccountID: &id1}}},
 			},
-			"bar": v2.Account{
+			"bar": {
 				Common: v2.Common{Providers: &v2.Providers{AWS: &v2.AWSProvider{AccountID: &id2}}},
 			},
 		},
@@ -192,7 +192,7 @@ func Test_buildCircleCI_Profiles(t *testing.T) {
 			},
 		},
 		Accounts: map[string]v2.Account{
-			"foo": v2.Account{
+			"foo": {
 				Common: v2.Common{Providers: &v2.Providers{AWS: &v2.AWSProvider{AccountID: &id1}}},
 			},
 		},
@@ -243,7 +243,7 @@ func Test_buildCircleCI_ProfilesDisabled(t *testing.T) {
 							Enabled:        &tr,
 							AWSIAMRoleName: util.StrPtr("rollin"),
 							Providers: map[string]v2.CIProviderConfig{
-								"aws": v2.CIProviderConfig{
+								"aws": {
 									Disabled: true,
 								},
 							},
@@ -252,7 +252,7 @@ func Test_buildCircleCI_ProfilesDisabled(t *testing.T) {
 			},
 		},
 		Accounts: map[string]v2.Account{
-			"foo": v2.Account{
+			"foo": {
 				Common: v2.Common{Providers: &v2.Providers{AWS: &v2.AWSProvider{AccountID: &id1}}},
 			},
 		},
diff --git a/plan/plan_test.go b/plan/plan_test.go
index dd87460b..827cee6b 100644
--- a/plan/plan_test.go
+++ b/plan/plan_test.go
@@ -130,6 +130,9 @@ func TestPlanBasicV2Yaml(t *testing.T) {
 	r.Equal(BackendKindS3, plan.Accounts["bar"].AccountBackends["foo"].Kind)
 	r.NotNil(plan.Accounts["bar"].AccountBackends["foo"].S3)
 	r.Equal("terraform/proj/accounts/foo.tfstate", plan.Accounts["bar"].AccountBackends["foo"].S3.KeyPath)
+
+	r.NotNil(plan.GitHubActionsCI)
+	r.True(plan.GitHubActionsCI.Enabled)
 }
 
 func TestResolveEKSConfig(t *testing.T) {
diff --git a/testdata/v2_full_yaml/.github/workflows/fogg_ci.yml b/testdata/v2_full_yaml/.github/workflows/fogg_ci.yml
new file mode 100644
index 00000000..cd64fdd7
--- /dev/null
+++ b/testdata/v2_full_yaml/.github/workflows/fogg_ci.yml
@@ -0,0 +1,68 @@
+# Auto-generated by fogg. Do not edit
+# Make improvements in fogg, so that everyone can benefit.
+
+on: push
+env:
+    FOO: bar
+
+jobs:
+  terraform_0:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.x
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+      - run: python3 -m venv venv
+      - run: . venv/bin/activate
+      - run: pip install awscli --upgrade --user
+      - run: aws --version
+      - run: FOGG_VERSION="vundefined-pre+undefined.dirty" make setup
+      - run: |
+          .fogg/bin/fogg apply
+          if [ "$(git status --porcelain)" != "" ]; then
+            echo "git tree is dirty after running fogg apply"
+            echo "ensure you run fogg apply on your branch"
+            git status
+            exit 1
+          fi
+      - run: aws configure set aws_access_key_id     ${{ secrets.IDACCT_AWS_ACCESS_KEY_ID }}     --profile _idacct
+      - run: aws configure set aws_secret_access_key ${{ secrets.IDACCT_AWS_SECRET_ACCESS_KEY }} --profile _idacct
+      - run: aws --profile _idacct sts get-caller-identity
+      - run: aws configure set profile.profile.role_arn arn:aws:iam::456:role/foo
+      - run: aws configure set profile.profile.source_profile _idacct
+      - run: aws --profile profile sts get-caller-identity
+      - run: |
+          cd terraform/accounts/bar
+          make  lint
+      - run: |
+          cd terraform/accounts/foo
+          make  lint
+      - run: |
+          cd terraform/global
+          make  lint
+      - run: |
+          cd terraform/modules/my_module
+          make  check
+      - run: |
+          cd terraform/envs/prod/datadog
+          make  lint
+      - run: |
+          cd terraform/envs/prod/hero
+          make  lint
+      - run: |
+          cd terraform/envs/prod/vpc
+          make  lint
+      - run: |
+          cd terraform/envs/staging/comp1
+          make  lint
+      - run: |
+          cd terraform/envs/staging/comp2
+          make  lint
+      - run: |
+          cd terraform/envs/staging/comp_helm_template
+          make  lint
+      - run: |
+          cd terraform/envs/staging/vpc
+          make  lint
diff --git a/testdata/v2_full_yaml/fogg.yml b/testdata/v2_full_yaml/fogg.yml
index e30fd3d2..4c5a595d 100644
--- a/testdata/v2_full_yaml/fogg.yml
+++ b/testdata/v2_full_yaml/fogg.yml
@@ -39,6 +39,13 @@ defaults:
       region: us-west-2
       version: 0.12.0
   terraform_version: 0.100.0
+  tools:
+    github_actions_ci:
+      enabled: true
+      command: lint
+      aws_iam_role_name: foo
+      env:
+        FOO: bar
 envs:
   prod:
     components: