Repository: facebook/react
Pull request: [CI] cache yarn instead of node_modules

The current caching of steps of `node_modules` doesn't work reliable as is because it includes `arch` in the cache key. `arch` might be different across workers in the same commit.

I couldn't find a way to optionally restore caches, so what this PR does is:
- remove the setup step that ran before all other steps and essentially just populates a circle CI cache key
- all other steps now do: restore yarn cache, `yarn install`, save yarn cache (fast if already exists)

With this change the initial batch of jobs all race to populate the cache, but any subsequent jobs should find the existing cache. The expected downside would be slightly more worker CPU time with all the parallel jobs, but wall time might be shorter (1 step less in the critical path) and we should be more reliable as we no longer have the failure with multiple archs.


## Alternative 1
Remove the `{arch}` from the cache key.

Downside: this might run into weird issues with native dependencies.

## Alternative 2
Somehow check if the cache was restored and only then run a yarn install.

Downside: couldn't figure out if it's possible to only restore the yarn cache if restoring the node_modules cache failed. Without that we'd either always restore both the yarn and node_modules cache or do yarn installs w/o cache which are prone to failure in the past.

Changed files:
filename: .circleci/config.yml
patch: @@ -7,32 +7,6 @@ aliases:
   - &environment
     TZ: /usr/share/zoneinfo/America/Los_Angeles
 
-  - &restore_yarn_cache
-    restore_cache:
-      name: Restore yarn cache
-      keys:
-        - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
-        - v1-yarn_cache-{{ arch }}-
-        - v1-yarn_cache-
-
-  - &yarn_install
-    run:
-      name: Install dependencies
-      command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
-
-  - &yarn_install_retry
-    run:
-      name: Install dependencies (retry)
-      when: on_fail
-      command: yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
-
-  - &save_yarn_cache
-    save_cache:
-      name: Save yarn cache
-      key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
-      paths:
-        - ~/.cache/yarn
-
   - &restore_yarn_cache_fixtures_dom
     restore_cache:
       name: Restore yarn cache for fixtures/dom
@@ -62,47 +36,34 @@ aliases:
       paths:
         - ~/.cache/yarn
 
-  - &save_node_modules
-    save_cache:
-      name: Save node_modules cache
-      # Cache only for the current revision to prevent cache injections from
-      # malicious PRs.
-      key: v1-node_modules-{{ arch }}-{{ .Revision }}
-      paths:
-        - node_modules
-        - packages/eslint-plugin-react-hooks/node_modules
-        - packages/react-art/node_modules
-        - packages/react-client/node_modules
-        - packages/react-devtools-core/node_modules
-        - packages/react-devtools-extensions/node_modules
-        - packages/react-devtools-inline/node_modules
-        - packages/react-devtools-shared/node_modules
-        - packages/react-devtools-shell/node_modules
-        - packages/react-devtools-timeline/node_modules
-        - packages/react-devtools/node_modules
-        - packages/react-dom/node_modules
-        - packages/react-interactions/node_modules
-        - packages/react-native-renderer/node_modules
-        - packages/react-reconciler/node_modules
-        - packages/react-server-dom-relay/node_modules
-        - packages/react-server-dom-webpack/node_modules
-        - packages/react-server-native-relay/node_modules
-        - packages/react-server/node_modules
-        - packages/react-test-renderer/node_modules
-        - packages/react/node_modules
-        - packages/scheduler/node_modules
-
-  - &restore_node_modules
-    restore_cache:
-      name: Restore node_modules cache
-      keys:
-        - v1-node_modules-{{ arch }}-{{ .Revision }}
-
   - &TEST_PARALLELISM 20
 
   - &attach_workspace
     at: build
 
+commands:
+  setup_node_modules:
+    description: "Restore node_modules"
+    steps:
+      - restore_cache:
+          name: Restore yarn cache
+          keys:
+            - v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
+            - v1-yarn_cache-{{ arch }}-
+            - v1-yarn_cache-
+      - run:
+          name: Install dependencies
+          command: |
+            yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+            if [ $? -ne 0 ]; then
+              yarn install --frozen-lockfile --cache-folder ~/.cache/yarn
+            fi
+      - save_cache:
+          name: Save yarn cache
+          key: v1-yarn_cache-{{ arch }}-{{ checksum "yarn.lock" }}
+          paths:
+            - ~/.cache/yarn
+
 # The CircleCI API doesn't yet support triggering a specific workflow, but it
 # does support triggering a pipeline. So as a workaround you can triggger the
 # entire pipeline and use parameters to disable everything except the workflow
@@ -115,28 +76,13 @@ parameters:
     default: ''
 
 jobs:
-  setup:
-    docker: *docker
-    environment: *environment
-    steps:
-      - checkout
-      - run:
-          name: NodeJS Version
-          command: node --version
-      - *restore_yarn_cache
-      - *restore_node_modules
-      - *yarn_install
-      - *yarn_install_retry
-      - *save_yarn_cache
-      - *save_node_modules
-
   yarn_lint:
     docker: *docker
     environment: *environment
 
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run: node ./scripts/prettier/index
       - run: node ./scripts/tasks/eslint
       - run: ./scripts/circleci/check_license.sh
@@ -150,7 +96,7 @@ jobs:
 
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run: node ./scripts/tasks/flow-ci
 
   scrape_warning_messages:
@@ -159,7 +105,7 @@ jobs:
 
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           command: |
             mkdir -p ./build
@@ -175,7 +121,7 @@ jobs:
     parallelism: 40
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run: yarn build-combined
       - persist_to_workspace:
           root: .
@@ -190,7 +136,7 @@ jobs:
         type: string
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Download artifacts for revision
           command: |
@@ -207,7 +153,7 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Download artifacts for base revision
           command: |
@@ -235,7 +181,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run: echo "<< pipeline.git.revision	>>" >> build/COMMIT_SHA
         # Compress build directory into a single tarball for easy download
       - run: tar -zcvf ./build.tgz ./build
@@ -254,7 +200,7 @@ jobs:
       - attach_workspace:
           at: .
       - run: echo "<< pipeline.git.revision	>>" >> build/COMMIT_SHA
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           command: node ./scripts/tasks/danger
 
@@ -265,7 +211,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           environment:
             RELEASE_CHANNEL: experimental
@@ -280,7 +226,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Playwright install deps
           command: |
@@ -302,7 +248,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run: ./scripts/circleci/download_devtools_regression_build.js << parameters.version >> --replaceBuild
       - run: node ./scripts/jest/jest-cli.js --build --project devtools --release-channel=experimental --reactVersion << parameters.version >> --ci
 
@@ -317,7 +263,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Playwright install deps
           command: |
@@ -341,7 +287,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run: yarn lint-build
 
   yarn_check_release_dependencies:
@@ -351,7 +297,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run: yarn check-release-dependencies
 
 
@@ -361,7 +307,7 @@ jobs:
     steps:
       - checkout
       - attach_workspace: *attach_workspace
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Search build artifacts for unminified errors
           command: |
@@ -374,7 +320,7 @@ jobs:
     steps:
       - checkout
       - attach_workspace: *attach_workspace
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Confirm generated inline Fizz runtime is up to date
           command: |
@@ -390,7 +336,7 @@ jobs:
         type: string
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run: yarn test <<parameters.args>> --ci
 
   yarn_test_build:
@@ -404,7 +350,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - run: yarn test --build <<parameters.args>> --ci
 
   RELEASE_CHANNEL_stable_yarn_test_dom_fixtures:
@@ -414,7 +360,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: .
-      - *restore_node_modules
+      - setup_node_modules
       - *restore_yarn_cache_fixtures_dom
       - *yarn_install_fixtures_dom
       - *yarn_install_fixtures_dom_retry
@@ -433,7 +379,7 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Run fuzz tests
           command: |
@@ -452,7 +398,7 @@ jobs:
     environment: *environment
     steps:
       - checkout
-      - *restore_node_modules
+      - setup_node_modules
       - run:
           name: Run publish script
           command: |
@@ -465,27 +411,29 @@ jobs:
 workflows:
   version: 2
 
-  # New workflow that will replace "stable" and "experimental"
   build_and_test:
     unless: << pipeline.parameters.prerelease_commit_sha >>
     jobs:
-      - setup:
+      - yarn_flow:
           filters:
             branches:
               ignore:
                 - builds/facebook-www
-      - yarn_flow:
-          requires:
-            - setup
       - check_generated_fizz_runtime:
-          requires:
-            - setup
+          filters:
+            branches:
+              ignore:
+                - builds/facebook-www
       - yarn_lint:
-          requires:
-            - setup
+          filters:
+            branches:
+              ignore:
+                - builds/facebook-www
       - yarn_test:
-          requires:
-            - setup
+          filters:
+            branches:
+              ignore:
+                - builds/facebook-www
           matrix:
             parameters:
               args:
@@ -509,11 +457,15 @@ workflows:
                 - '-r=stable --env=development --persistent'
                 - '-r=experimental --env=development --persistent'
       - yarn_build_combined:
-          requires:
-            - setup
+          filters:
+            branches:
+              ignore:
+                - builds/facebook-www
       - scrape_warning_messages:
-          requires:
-            - setup
+          filters:
+            branches:
+              ignore:
+                - builds/facebook-www
       - process_artifacts_combined:
           requires:
             - scrape_warning_messages
@@ -551,8 +503,7 @@ workflows:
             branches:
               ignore:
                 - main
-          requires:
-            - setup
+                - builds/facebook-www
       - sizebot:
           filters:
             branches:
@@ -591,10 +542,7 @@ workflows:
               only:
                 - main
     jobs:
-      - setup
-      - test_fuzz:
-          requires:
-            - setup
+      - test_fuzz
 
   devtools_regression_tests:
     unless: << pipeline.parameters.prerelease_commit_sha >>
@@ -607,10 +555,7 @@ workflows:
               only:
                 - main
     jobs:
-      - setup
       - download_build:
-          requires:
-            - setup
           revision: << pipeline.git.revision >>
       - build_devtools_and_process_artifacts:
           requires:
@@ -642,11 +587,8 @@ workflows:
   publish_preleases:
     when: << pipeline.parameters.prerelease_commit_sha >>
     jobs:
-      - setup
       - publish_prerelease:
           name: Publish to Next channel
-          requires:
-            - setup
           commit_sha: << pipeline.parameters.prerelease_commit_sha >>
           release_channel: stable
           dist_tag: "next"
@@ -674,11 +616,8 @@ workflows:
               only:
                 - main
     jobs:
-      - setup
       - publish_prerelease:
           name: Publish to Next channel
-          requires:
-            - setup
           commit_sha: << pipeline.git.revision >>
           release_channel: stable
           dist_tag: "next"