Repository: chanzuckerberg/fogg (go)

Issue:
don't run refresh each time
Basic idea is that in the component.mk generated by fogg we do this–

1.  pull 'refresh' out to its own target and make it a dependency of plan & apply (and maybe others?)
1.  change the plan, apply, etc targets to -refresh=false
1.  In the refresh target, do something like
  1. read a cache file which stores the time of the last refresh (which is .gitignored)
  1. if now - last refresh > some threshold (60 min?)
    1. run refresh



Pull request:
[feature] avoid running fresh on every plan
In order to improve development cycles, this diff makes a change that
will only on refresh before a plan once an hour. Also when `terraform
plan` runs a refresh, the updated state does not get persisted whereas
`terraform refresh` does.

Fixes #411 

### Test Plan
* CI
* tested manually on one repo already

### References
* https://www.terraform.io/docs/commands/refresh.html
* https://www.terraform.io/docs/commands/plan.html#refresh-true

Patch:
diff --git a/templates/repo/scripts/component.mk b/templates/repo/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/templates/repo/scripts/component.mk
+++ b/templates/repo/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \


Test patch:
diff --git a/testdata/bless_provider/scripts/component.mk b/testdata/bless_provider/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/bless_provider/scripts/component.mk
+++ b/testdata/bless_provider/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/github_provider/scripts/component.mk b/testdata/github_provider/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/github_provider/scripts/component.mk
+++ b/testdata/github_provider/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/okta_provider/scripts/component.mk b/testdata/okta_provider/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/okta_provider/scripts/component.mk
+++ b/testdata/okta_provider/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/snowflake_provider/scripts/component.mk b/testdata/snowflake_provider/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/snowflake_provider/scripts/component.mk
+++ b/testdata/snowflake_provider/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/v1_full/scripts/component.mk b/testdata/v1_full/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/v1_full/scripts/component.mk
+++ b/testdata/v1_full/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/v2_full/scripts/component.mk b/testdata/v2_full/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/v2_full/scripts/component.mk
+++ b/testdata/v2_full/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/v2_minimal_valid/scripts/component.mk b/testdata/v2_minimal_valid/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/v2_minimal_valid/scripts/component.mk
+++ b/testdata/v2_minimal_valid/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \
diff --git a/testdata/v2_no_aws_provider/scripts/component.mk b/testdata/v2_no_aws_provider/scripts/component.mk
index b7acf472..c7b8d430 100644
--- a/testdata/v2_no_aws_provider/scripts/component.mk
+++ b/testdata/v2_no_aws_provider/scripts/component.mk
@@ -61,12 +61,28 @@ ifeq ($(HEROKU_PROVIDER),1)
 endif
 .PHONY: check-auth-heroku
 
-plan: check-auth init fmt ## run a terraform plan
-	@$(terraform_command) plan $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -input=false
+refresh:
+	@$(terraform_command) refresh $(TF_ARGS)
+	@date +%s > .terraform/refreshed_at
+.PHONY: refresh
+
+refresh-cached:
+	@last_refresh=`cat .terraform/refreshed_at || echo '0'`; \
+	current_time=`date +%s`; \
+	if (( current_time - last_refresh > 600 )); then \
+		echo "It has been awhile since the last refresh. It is time."; \
+		$(MAKE) refresh; \
+	else \
+		echo "Not time to refresh yet."; \
+	fi;
+.PHONY: refresh-cached
+
+plan: check-auth init fmt refresh-cached ## run a terraform plan
+	@$(terraform_command) plan $(TF_ARGS) -refresh=false -input=false
 .PHONY: plan
 
-apply: check-auth init ## run a terraform apply
-	@$(terraform_command) apply $(TF_ARGS) -refresh=$(REFRESH_ENABLED) -auto-approve=$(AUTO_APPROVE)
+apply: check-auth init refresh ## run a terraform apply
+	@$(terraform_command) apply $(TF_ARGS) -refresh=false -auto-approve=$(AUTO_APPROVE)
 .PHONY: apply
 
 docs:
@@ -85,7 +101,7 @@ init: terraform check-auth ## run terraform init for this component
 	@$(terraform_command) init -input=false
 .PHONY: init
 
-check-plan: init check-auth ## run a terraform plan and check that it does not fail
+check-plan: check-auth init refresh-cached ## run a terraform plan and check that it does not fail
 	@$(terraform_command) plan $(TF_ARGS) -detailed-exitcode -lock=false -out=$(CHECK_PLANFILE_PATH) ; \
 	ERR=$$?; \
 	if [ $$ERR -eq 0 ] ; then \