Repository: chanzuckerberg/fogg
Pull request #407 (2020-02-19 18:01:18)

fix: CircleCI config proper parallelism/bucketing
We were not properly doing parallelism/workflows. CircleCI config generated could be DRYer, we can make improvements later.

codecov[bot]: # [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/407?src=pr&el=h1) Report
> Merging [#407](https://codecov.io/gh/chanzuckerberg/fogg/pull/407?src=pr&el=desc) into [master](https://codecov.io/gh/chanzuckerberg/fogg/commit/4db119574c40d0247f82be3ec50caae9f765088b?src=pr&el=desc) will **not change** coverage.
> The diff coverage is `n/a`.

[![Impacted file tree graph](https://codecov.io/gh/chanzuckerberg/fogg/pull/407/graphs/tree.svg?width=650&token=57DhtZ7g2A&height=150&src=pr)](https://codecov.io/gh/chanzuckerberg/fogg/pull/407?src=pr&el=tree)

```diff
@@           Coverage Diff           @@
##           master     #407   +/-   ##
=======================================
  Coverage   78.15%   78.15%           
=======================================
  Files          24       24           
  Lines        1909     1909           
=======================================
  Hits         1492     1492           
  Misses        276      276           
  Partials      141      141
```



------

[Continue to review full report at Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/407?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/chanzuckerberg/fogg/pull/407?src=pr&el=footer). Last update [4db1195...c40c1ac](https://codecov.io/gh/chanzuckerberg/fogg/pull/407?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/templates/circleci/.circleci/config.yml.tmpl b/templates/circleci/.circleci/config.yml.tmpl
index 81a20dd9..f3b8d89b 100644
--- a/templates/circleci/.circleci/config.yml.tmpl
+++ b/templates/circleci/.circleci/config.yml.tmpl
@@ -2,30 +2,29 @@
 # Make improvements in fogg, so that everyone can benefit.
 {{- $circleCI := . }}
 version: 2
-
 working_directory: &working_directory ~/fogg
 
 jobs:
-  build:
+{{- range $idx, $testBucket := $circleCI.TestBuckets }}
+  terraform_{{$idx}}:
     docker:
       - image: circleci/python:3.8
     working_directory: *working_directory
-    parallelism: {{ len .TestBuckets }}
     steps:
       - checkout
-{{- if not (eq (len .SSHFingerprints) 0) }}
+{{- if not (eq (len $circleCI.SSHFingerprints) 0) }}
       # HACK(el):Switch to key that reads private tf modules after checking out our code
       - add_ssh_keys:
-          fingerprints:
-          {{- range $sshFingerprint := .SSHFingerprints }}
-            - {{ $sshFingerprint }}
-          {{- end }}
+        fingerprints:
+        {{- range $sshFingerprint := $circleCI.SSHFingerprints }}
+          - {{ $sshFingerprint }}
+        {{- end }}
 {{- end }}
       - run: python3 -m venv venv
       - run: . venv/bin/activate
       - run: pip install awscli --upgrade --user
       - run: aws --version
-      - run: FOGG_VERSION="v{{ .FoggVersion }}" make setup
+      - run: FOGG_VERSION="v{{ $circleCI.FoggVersion }}" make setup
       - run:
           command: |
             .fogg/bin/fogg apply
@@ -35,18 +34,16 @@ jobs:
               git status
               exit 1
             fi
-{{- if not (eq (len .AWSProfiles) 0) }}
-      - aws configure set aws_access_key_id     $IDACCT_AWS_ACCESS_KEY_ID     --profile _idacct
-      - aws configure set aws_secret_access_key $IDACCT_AWS_SECRET_ACCESS_KEY --profile _idacct
-      - aws --profile _idacct sts get-caller-identity
+{{- if not (eq (len $circleCI.AWSProfiles) 0) }}
+      - run: aws configure set aws_access_key_id     $IDACCT_AWS_ACCESS_KEY_ID     --profile _idacct
+      - run: aws configure set aws_secret_access_key $IDACCT_AWS_SECRET_ACCESS_KEY --profile _idacct
+      - run: aws --profile _idacct sts get-caller-identity
 {{- end }}
-{{- range $profileName, $profile := .AWSProfiles }}
-      - aws configure set profile.{{ $profileName }}.role_arn arn:aws:iam::{{ $profile.AccountID }}:role/{{ $profile.RoleName}}
-      - aws configure set profile.{{ $profileName }}.source_profile _idacct
-      - aws --profile {{ $profileName }} sts get-caller-identity
+{{- range $profileName, $profile := $circleCI.AWSProfiles }}
+      - run: aws configure set profile.{{ $profileName }}.role_arn arn:aws:iam::{{ $profile.AccountID }}:role/{{ $profile.RoleName}}
+      - run: aws configure set profile.{{ $profileName }}.source_profile _idacct
+      - run: aws --profile {{ $profileName }} sts get-caller-identity
 {{- end }}
-
-{{- range $testBucket := .TestBuckets }}
   {{- range $component := $testBucket }}
       - run:
           command: |
@@ -54,3 +51,11 @@ jobs:
             make  {{ $component.Command }}
   {{- end }}
 {{- end }}
+
+workflows:
+  version: 2
+  terraform:
+    jobs:
+{{- range $idx, $testBucket := $circleCI.TestBuckets }}
+      - terraform_{{$idx}}
+{{- end }}


Test patch:
diff --git a/testdata/v2_full/.circleci/config.yml b/testdata/v2_full/.circleci/config.yml
index 651712a4..fec2de1b 100644
--- a/testdata/v2_full/.circleci/config.yml
+++ b/testdata/v2_full/.circleci/config.yml
@@ -1,15 +1,13 @@
 # Auto-generated by fogg. Do not edit
 # Make improvements in fogg, so that everyone can benefit.
 version: 2
-
 working_directory: &working_directory ~/fogg
 
 jobs:
-  build:
+  terraform_0:
     docker:
       - image: circleci/python:3.8
     working_directory: *working_directory
-    parallelism: 1
     steps:
       - checkout
       - run: python3 -m venv venv
@@ -26,15 +24,15 @@ jobs:
               git status
               exit 1
             fi
-      - aws configure set aws_access_key_id     $IDACCT_AWS_ACCESS_KEY_ID     --profile _idacct
-      - aws configure set aws_secret_access_key $IDACCT_AWS_SECRET_ACCESS_KEY --profile _idacct
-      - aws --profile _idacct sts get-caller-identity
-      - aws configure set profile.comp1.role_arn arn:aws:iam::1234:role/foobar-role
-      - aws configure set profile.comp1.source_profile _idacct
-      - aws --profile comp1 sts get-caller-identity
-      - aws configure set profile.profile.role_arn arn:aws:iam::456:role/foobar-role
-      - aws configure set profile.profile.source_profile _idacct
-      - aws --profile profile sts get-caller-identity
+      - run: aws configure set aws_access_key_id     $IDACCT_AWS_ACCESS_KEY_ID     --profile _idacct
+      - run: aws configure set aws_secret_access_key $IDACCT_AWS_SECRET_ACCESS_KEY --profile _idacct
+      - run: aws --profile _idacct sts get-caller-identity
+      - run: aws configure set profile.comp1.role_arn arn:aws:iam::1234:role/foobar-role
+      - run: aws configure set profile.comp1.source_profile _idacct
+      - run: aws --profile comp1 sts get-caller-identity
+      - run: aws configure set profile.profile.role_arn arn:aws:iam::456:role/foobar-role
+      - run: aws configure set profile.profile.source_profile _idacct
+      - run: aws --profile profile sts get-caller-identity
       - run:
           command: |
             cd terraform/accounts/bar
@@ -67,3 +65,9 @@ jobs:
           command: |
             cd terraform/envs/staging/vpc
             make  check
+
+workflows:
+  version: 2
+  terraform:
+    jobs:
+      - terraform_0