Repository: chanzuckerberg/fogg
Pull request #707 (2022-08-11 04:36:44)

feat: update locals.tf.json on apply
### Summary
If there exists a locals.tf.json file in the fogg project, on apply it will get updated based on any new envs/accounts or any deleted envs/accounts. For accounts or envs that are already there, it will leave them alone.

### Test Plan
Say unittests, or list out steps to verify changes.

### References
(Optional) Additional links to provide more context.



Patch:
diff --git a/apply/apply.go b/apply/apply.go
index f5fd0bb4..a764f8f2 100644
--- a/apply/apply.go
+++ b/apply/apply.go
@@ -2,6 +2,7 @@ package apply
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/fs"
@@ -13,6 +14,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	v2 "github.com/chanzuckerberg/fogg/config/v2"
 	"github.com/chanzuckerberg/fogg/errs"
 	"github.com/chanzuckerberg/fogg/plan"
@@ -38,55 +41,183 @@ func Apply(fs afero.Fs, conf *v2.Config, tmp *templates.T, upgrade bool) error {
 			return errs.NewUserf("fogg version (%s) is different than version currently used to manage repo (%s). To upgrade add --upgrade.", toolVersion, repoVersion)
 		}
 	}
-	p, err := plan.Eval(conf)
+	plan, err := plan.Eval(conf)
 	if err != nil {
 		return errs.WrapUser(err, "unable to evaluate plan")
 	}
-	e := applyRepo(fs, p, tmp.Repo, tmp.Common)
-	if e != nil {
-		return errs.WrapUser(e, "unable to apply repo")
+	err = applyRepo(fs, plan, tmp.Repo, tmp.Common)
+	if err != nil {
+		return errs.WrapUser(err, "unable to apply repo")
 	}
 
-	if p.TravisCI.Enabled {
-		e = applyTree(fs, tmp.TravisCI, tmp.Common, "", p.TravisCI)
-		if e != nil {
-			return errs.WrapUser(e, "unable to apply travis ci")
+	if plan.TravisCI.Enabled {
+		err = applyTree(fs, tmp.TravisCI, tmp.Common, "", plan.TravisCI)
+		if err != nil {
+			return errs.WrapUser(err, "unable to apply travis ci")
 		}
 	}
 
-	if p.CircleCI.Enabled {
-		e = applyTree(fs, tmp.CircleCI, tmp.Common, "", p.CircleCI)
-		if e != nil {
-			return errs.WrapUser(e, "unable to apply CircleCI")
+	if plan.CircleCI.Enabled {
+		err = applyTree(fs, tmp.CircleCI, tmp.Common, "", plan.CircleCI)
+		if err != nil {
+			return errs.WrapUser(err, "unable to apply CircleCI")
 		}
 	}
 
-	if p.GitHubActionsCI.Enabled {
-		e = applyTree(fs, tmp.GitHubActionsCI, tmp.Common, ".github", p.GitHubActionsCI)
-		if e != nil {
-			return errs.WrapUser(e, "unable to apply GitHub Actions CI")
+	if plan.GitHubActionsCI.Enabled {
+		err = applyTree(fs, tmp.GitHubActionsCI, tmp.Common, ".github", plan.GitHubActionsCI)
+		if err != nil {
+			return errs.WrapUser(err, "unable to apply GitHub Actions CI")
 		}
 	}
 
 	tfBox := tmp.Components[v2.ComponentKindTerraform]
-	e = applyAccounts(fs, p, tfBox, tmp.Common)
-	if e != nil {
-		return errs.WrapUser(e, "unable to apply accounts")
+	err = applyAccounts(fs, plan, tfBox, tmp.Common)
+	if err != nil {
+		return errs.WrapUser(err, "unable to apply accounts")
 	}
 
-	e = applyEnvs(fs, p, tmp.Env, tmp.Components, tmp.Common)
-	if e != nil {
-		return errs.WrapUser(e, "unable to apply envs")
+	err = applyEnvs(fs, plan, tmp.Env, tmp.Components, tmp.Common)
+	if err != nil {
+		return errs.WrapUser(err, "unable to apply envs")
 	}
 
 	tfBox = tmp.Components[v2.ComponentKindTerraform]
-	e = applyGlobal(fs, p.Global, tfBox, tmp.Common)
-	if e != nil {
-		return errs.WrapUser(e, "unable to apply global")
+	err = applyGlobal(fs, plan.Global, tfBox, tmp.Common)
+	if err != nil {
+		return errs.WrapUser(err, "unable to apply global")
+	}
+
+	err = applyModules(fs, plan.Modules, tmp.Module, tmp.Common)
+	if err != nil {
+		return errs.WrapUser(err, "unable to apply modules")
+	}
+
+	return errs.WrapUser(applyTFE(fs, plan), "unable to apply TFE locals.tf.json")
+}
+
+type LocalsTFE struct {
+	Locals *Locals `json:"locals,omitempty"`
+}
+
+type Locals struct {
+	Accounts         map[string]*TFEWorkspace            `json:"accounts,omitempty"`
+	Envs             map[string]map[string]*TFEWorkspace `json:"envs,omitempty"`
+	DefaultTFVersion *string                             `json:"default_terraform_version,omitempty"`
+}
+type ExtraTeamPermissions struct {
+	Plan *[]string `json:"plan,omitempty"`
+	Read *[]string `json:"read,omitempty"`
+}
+type TFEWorkspace struct {
+	TriggerPrefixes      *[]string             `json:"trigger_prefixes"`
+	WorkingDirectory     *string               `json:"working_directory,omitempty"`
+	TerraformVersion     *string               `json:"terraform_version,omitempty"`
+	ExtraTeamPermissions *ExtraTeamPermissions `json:"extra_team_permissions,omitempty"`
+	GithubBranch         *string               `json:"branch,omitempty"`
+	AutoApply            *bool                 `json:"auto_apply,omitempty"`
+	RemoteApply          *bool                 `json:"remote_apply,omitempty"`
+}
+
+func MakeTFEWorkspace() *TFEWorkspace {
+	defaultTriggerPrefixes := make([]string, 0)
+	defaultTerraformVersion := "1.2.6"
+	defaultGithubBranch := "main"
+	defaultAutoApply := true
+	defaultRemoteApply := false
+	return &TFEWorkspace{
+		TriggerPrefixes:  &defaultTriggerPrefixes,
+		TerraformVersion: &defaultTerraformVersion,
+		GithubBranch:     &defaultGithubBranch,
+		AutoApply:        &defaultAutoApply,
+		RemoteApply:      &defaultRemoteApply,
+	}
+}
+
+func updateLocalsFromPlan(locals *LocalsTFE, plan *plan.Plan) {
+	// if there is a planned env or account that isn't in the locals, add it
+	for accountName := range plan.Accounts {
+		if _, ok := locals.Locals.Accounts[accountName]; !ok {
+			locals.Locals.Accounts[accountName] = MakeTFEWorkspace()
+		}
+	}
+	for envName := range plan.Envs {
+		if _, ok := locals.Locals.Envs[envName]; !ok {
+			locals.Locals.Envs[envName] = make(map[string]*TFEWorkspace, 0)
+		}
+		for componentName := range plan.Envs[envName].Components {
+			if _, ok := locals.Locals.Envs[envName][componentName]; !ok {
+				locals.Locals.Envs[envName][componentName] = MakeTFEWorkspace()
+			}
+		}
+	}
+
+	// if there is a locals env or account that isn't in the plan, delete it
+	for account := range locals.Locals.Accounts {
+		shouldDelete := func() bool {
+			for plannedAccount := range plan.Accounts {
+				if account == plannedAccount {
+					return false
+				}
+			}
+			return true
+		}()
+		if shouldDelete {
+			delete(locals.Locals.Accounts, account)
+		}
+	}
+	for envName, component := range locals.Locals.Envs {
+		for componentName := range component {
+			shouldDelete := func() bool {
+				for plannedComponent := range plan.Envs[envName].Components {
+					if plannedComponent == componentName {
+						return false
+					}
+				}
+
+				return true
+			}()
+			if shouldDelete {
+				delete(locals.Locals.Envs[envName], componentName)
+			}
+		}
 	}
+}
 
-	e = applyModules(fs, p.Modules, tmp.Module, tmp.Common)
-	return errs.WrapUser(e, "unable to apply modules")
+func applyTFE(fs afero.Fs, plan *plan.Plan) error {
+	tfePath := filepath.Join("terraform", "tfe", "locals.tf.json")
+	_, err := fs.Stat(tfePath)
+	// if the repo doesn't have a locals.tf.json, don't worry about it
+	if errors.Is(err, os.ErrNotExist) {
+		return nil
+	} else if err != nil {
+		return err
+	}
+	read, err := fs.Open(tfePath)
+	if err != nil {
+		return errors.Wrapf(err, "unable to open locals.tf.json file %s for unmarshalling", tfePath)
+	}
+	defer read.Close()
+	locals := LocalsTFE{}
+	err = json.NewDecoder(read).Decode(&locals)
+	if err != nil {
+		return errors.Wrapf(err, "unable to decode locals.tf.json from %s", tfePath)
+	}
+
+	updateLocalsFromPlan(&locals, plan)
+	write, err := fs.OpenFile(tfePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+	if err != nil {
+		return errors.Wrapf(err, "unable to open locals.tf.json file %s for marshaling", tfePath)
+	}
+	defer write.Close()
+	encoder := json.NewEncoder(write)
+	encoder.SetIndent("", "  ")
+	err = encoder.Encode(locals)
+	if err != nil {
+		return errors.Wrap(err, "unable to marhsal locals.tf.json")
+	}
+
+	return nil
 }
 
 func checkToolVersions(fs afero.Fs, current string) (bool, string, error) {


Test patch:
diff --git a/apply/apply_test.go b/apply/apply_test.go
index d0b1a153..c6c8290d 100644
--- a/apply/apply_test.go
+++ b/apply/apply_test.go
@@ -1,6 +1,8 @@
 package apply
 
 import (
+	"errors"
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"os"
@@ -9,6 +11,7 @@ import (
 	"testing"
 
 	v2 "github.com/chanzuckerberg/fogg/config/v2"
+	"github.com/chanzuckerberg/fogg/plan"
 	"github.com/chanzuckerberg/fogg/templates"
 	"github.com/chanzuckerberg/fogg/util"
 	"github.com/sirupsen/logrus"
@@ -46,6 +49,151 @@ func getNonExistentDirectoryName() string {
 	}
 }
 
+type UploadLocalsTestCase struct {
+	locals                     LocalsTFE
+	plan                       plan.Plan
+	resultEnvs, resultAccounts []string
+}
+
+type UploadLocalsTestCaseSimple struct {
+	localsAccounts, localEnvs  []string
+	planAccounts, planEnvs     []string
+	resultEnvs, resultAccounts []string
+}
+
+func makeTestCases(tests []UploadLocalsTestCaseSimple) ([]UploadLocalsTestCase, error) {
+	testcases := []UploadLocalsTestCase{}
+	for _, test := range tests {
+		testcase := UploadLocalsTestCase{}
+
+		localAccounts := make(map[string]*TFEWorkspace, 0)
+		for _, local := range test.localsAccounts {
+			localAccounts[local] = MakeTFEWorkspace()
+		}
+		localEnvs := make(map[string]map[string]*TFEWorkspace, 0)
+		for _, local := range test.localEnvs {
+			splits := strings.Split(local, "/")
+			if len(splits) != 2 {
+				return nil, errors.New("env needs to be of the form env/component")
+			}
+			component := map[string]*TFEWorkspace{}
+			component[splits[1]] = MakeTFEWorkspace()
+			localEnvs[splits[0]] = component
+		}
+		locals := LocalsTFE{
+			Locals: &Locals{
+				Accounts: localAccounts,
+				Envs:     localEnvs,
+			},
+		}
+
+		planAccounts := map[string]plan.Account{}
+		for _, local := range test.planAccounts {
+			planAccounts[local] = plan.Account{}
+		}
+		planEnvs := map[string]plan.Env{}
+		for _, local := range test.planEnvs {
+			splits := strings.Split(local, "/")
+			if len(splits) != 2 {
+				return nil, errors.New("env needs to be of the form env/component")
+			}
+			component := map[string]plan.Component{}
+			component[splits[1]] = plan.Component{}
+			planEnvs[splits[0]] = plan.Env{
+				Components: component,
+			}
+		}
+		plan := plan.Plan{
+			Accounts: planAccounts,
+			Envs:     planEnvs,
+		}
+		testcase.locals = locals
+		testcase.plan = plan
+		testcase.resultAccounts = test.resultAccounts
+		testcase.resultEnvs = test.resultEnvs
+		testcases = append(testcases, testcase)
+	}
+	return testcases, nil
+}
+
+func TestUpdateLocalsFromPlan(t *testing.T) {
+	r := require.New(t)
+	testcases, err := makeTestCases([]UploadLocalsTestCaseSimple{
+		{
+			localsAccounts: []string{"a", "b"},
+			localEnvs:      []string{"playground/a", "dev/b"},
+			planAccounts:   []string{"a", "b"},
+			planEnvs:       []string{"playground/a", "dev/b"},
+			resultAccounts: []string{"a", "b"},
+			resultEnvs:     []string{"playground/a", "dev/b"},
+		},
+		{
+			localsAccounts: []string{"b"},
+			localEnvs:      []string{"playground/a", "dev/b"},
+			planAccounts:   []string{"a", "b"},
+			planEnvs:       []string{"playground/a", "dev/b"},
+			resultAccounts: []string{"a", "b"},
+			resultEnvs:     []string{"playground/a", "dev/b"},
+		},
+		{
+			localsAccounts: []string{},
+			localEnvs:      []string{"playground/a", "dev/b"},
+			planAccounts:   []string{"a", "b"},
+			planEnvs:       []string{"playground/a", "dev/b"},
+			resultAccounts: []string{"a", "b"},
+			resultEnvs:     []string{"playground/a", "dev/b"},
+		},
+		{
+			localsAccounts: []string{},
+			localEnvs:      []string{},
+			planAccounts:   []string{"a", "b"},
+			planEnvs:       []string{"playground/a", "dev/b"},
+			resultAccounts: []string{"a", "b"},
+			resultEnvs:     []string{"playground/a", "dev/b"},
+		},
+		{
+			localsAccounts: []string{"a", "b"},
+			localEnvs:      []string{"playground/a", "dev/b"},
+			planAccounts:   []string{"b"},
+			planEnvs:       []string{"playground/a", "dev/b"},
+			resultAccounts: []string{"b"},
+			resultEnvs:     []string{"playground/a", "dev/b"},
+		},
+		{
+			localsAccounts: []string{"a", "b"},
+			localEnvs:      []string{"playground/a", "dev/b"},
+			planAccounts:   []string{"b"},
+			planEnvs:       []string{"dev/b"},
+			resultAccounts: []string{"b"},
+			resultEnvs:     []string{"dev/b"},
+		},
+		{
+			localsAccounts: []string{"a", "b"},
+			localEnvs:      []string{"playground/a", "dev/b"},
+			planAccounts:   []string{"b"},
+			planEnvs:       []string{"dev2/b"},
+			resultAccounts: []string{"b"},
+			resultEnvs:     []string{"dev2/b"},
+		},
+	})
+	r.NoError(err)
+	for _, testcase := range testcases {
+		updateLocalsFromPlan(&testcase.locals, &testcase.plan)
+		resultsAccounts := []string{}
+		for key := range testcase.locals.Locals.Accounts {
+			resultsAccounts = append(resultsAccounts, key)
+		}
+		resultEnvs := []string{}
+		for env, envLocal := range testcase.locals.Locals.Envs {
+			for component := range envLocal {
+				resultEnvs = append(resultEnvs, fmt.Sprintf("%s/%s", env, component))
+			}
+		}
+		r.ElementsMatch(testcase.resultAccounts, resultsAccounts)
+		r.ElementsMatch(testcase.resultEnvs, resultEnvs)
+	}
+}
+
 func TestRemoveExtension(t *testing.T) {
 	r := require.New(t)
 	x := removeExtension("foo")