Repository: solana-labs/solana-program-library
Pull request #5841 (2023-11-14 19:16:42)

ci: Refactor JS tests to avoid running so often
#### Problem

The JS CI that kicks off for any change to pnpm-lock.yaml triggers essentially *everything* to run, when very often we just need to make sure that the packages build.

#### Solution

Refactor the CI jobs a bit:

* Make JS test scripts build the programs they rely on with `build:program`
* Avoid running main program-level workflows if there's just a JS change
* Add new workflow if there's only a JS change

While I was at it, I killed off the fee-specific token-swap JS tests, since they were more annoying than anything else to manage.

Note: Marking this a draft since it might take some time to get things right in CI.

joncinque: Merging this early so that it can be part of the dependabot runs tomorrow and avoid using all of our runners at once. I'll happily integrate any feedback afterwards!

Patch:
diff --git a/.github/workflows/pull-request-js.yml b/.github/workflows/pull-request-js.yml
new file mode 100644
index 000000000..ed747a7e1
--- /dev/null
+++ b/.github/workflows/pull-request-js.yml
@@ -0,0 +1,54 @@
+name: Pull Request JS
+
+on:
+  pull_request:
+    paths:
+      - 'libraries/type-length-value/js/**'
+      - 'memo/js/**'
+      - 'name-service/js/**'
+      - 'single-pool/js/**'
+      - 'stake-pool/js/**'
+      - 'token/js/**'
+      - 'token-lending/js/**'
+      - 'token-metadata/js/**'
+      - 'token-swap/js/**'
+      - 'pnpm-lock.yaml'
+      - '.github/workflows/pull-request-js.yml'
+  push:
+    branches: [master]
+    paths:
+      - 'libraries/type-length-value/js/**'
+      - 'memo/js/**'
+      - 'single-pool/js/**'
+      - 'stake-pool/js/**'
+      - 'token/js/**'
+      - 'token-lending/js/**'
+      - 'token-metadata/js/**'
+      - 'token-swap/js/**'
+      - 'pnpm-lock.yaml'
+      - '.github/workflows/pull-request-js.yml'
+
+jobs:
+  js-test:
+    strategy:
+      matrix:
+        package: [libraries, memo, name-service, single-pool, stake-pool, token, token-lending, token-metadata, token-swap]
+    runs-on: ubuntu-latest
+    env:
+      NODE_VERSION: 20.5
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ env.NODE_VERSION }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - uses: pnpm/action-setup@v2
+        with:
+          version: 8
+      - uses: actions/cache@v2
+        with:
+          path: ~/.npm
+          key: node-${{ hashFiles('pnpm-lock.yaml') }}
+          restore-keys: |
+            node-
+      - run: ./ci/js-test-${{ matrix.package }}.sh
diff --git a/.github/workflows/pull-request-libraries.yml b/.github/workflows/pull-request-libraries.yml
index aa839defe..53226ba3c 100644
--- a/.github/workflows/pull-request-libraries.yml
+++ b/.github/workflows/pull-request-libraries.yml
@@ -5,13 +5,13 @@ on:
     paths:
     - 'libraries/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!libraries/**/js/**'
   push:
     branches: [master]
     paths:
     - 'libraries/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!libraries/**/js/**'
 
 jobs:
   cargo-test-sbf:
diff --git a/.github/workflows/pull-request-memo.yml b/.github/workflows/pull-request-memo.yml
index dac3b5927..78c0d62bf 100644
--- a/.github/workflows/pull-request-memo.yml
+++ b/.github/workflows/pull-request-memo.yml
@@ -5,13 +5,13 @@ on:
     paths:
     - 'memo/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!memo/js/**'
   push:
     branches: [master]
     paths:
     - 'memo/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!memo/js/**'
 
 jobs:
   cargo-test-sbf:
diff --git a/.github/workflows/pull-request-name-service.yml b/.github/workflows/pull-request-name-service.yml
index b1a666a2c..f3d2ce06f 100644
--- a/.github/workflows/pull-request-name-service.yml
+++ b/.github/workflows/pull-request-name-service.yml
@@ -5,13 +5,13 @@ on:
     paths:
     - 'name-service/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!name-service/js/**'
   push:
     branches: [master]
     paths:
     - 'name-service/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!name-service/js/**'
 
 jobs:
   cargo-test-sbf:
@@ -70,7 +70,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NODE_VERSION: 16.x
-    needs: cargo-test-sbf
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ env.NODE_VERSION }}
@@ -86,9 +85,4 @@ jobs:
           key: node-${{ hashFiles('pnpm-lock.yaml') }}
           restore-keys: |
             node-
-      - name: Download programs
-        uses: actions/download-artifact@v2
-        with:
-          name: name-service-programs
-          path: target/deploy
       - run: ./ci/js-test-name-service.sh
diff --git a/.github/workflows/pull-request-single-pool.yml b/.github/workflows/pull-request-single-pool.yml
index c85dd0a90..c1ab2f6a3 100644
--- a/.github/workflows/pull-request-single-pool.yml
+++ b/.github/workflows/pull-request-single-pool.yml
@@ -8,7 +8,7 @@ on:
     - 'associated-token-account/**'
     - 'ci/*-version.sh'
     - '.github/workflows/pull-request-single-pool.yml'
-    - 'pnpm-lock.yaml'
+    - '!single-pool/js/**'
   push:
     branches: [master]
     paths:
@@ -17,7 +17,7 @@ on:
     - 'associated-token-account/**'
     - 'ci/*-version.sh'
     - '.github/workflows/pull-request-single-pool.yml'
-    - 'pnpm-lock.yaml'
+    - '!single-pool/js/**'
 
 jobs:
   cargo-test-sbf:
@@ -121,7 +121,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NODE_VERSION: 20.5
-    needs: cargo-test-sbf
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ env.NODE_VERSION }}
@@ -137,9 +136,4 @@ jobs:
           key: node-${{ hashFiles('pnpm-lock.yaml') }}
           restore-keys: |
             node-
-      - name: Download programs
-        uses: actions/download-artifact@v2
-        with:
-          name: single-pool-programs
-          path: target/deploy
       - run: ./ci/js-test-single-pool.sh
diff --git a/.github/workflows/pull-request-stake-pool.yml b/.github/workflows/pull-request-stake-pool.yml
index 54f0e41e8..a912b5b33 100644
--- a/.github/workflows/pull-request-stake-pool.yml
+++ b/.github/workflows/pull-request-stake-pool.yml
@@ -8,7 +8,7 @@ on:
     - 'ci/*-version.sh'
     - 'ci/warning/purge-ubuntu-runner.sh'
     - '.github/workflows/pull-request-stake-pool.yml'
-    - 'pnpm-lock.yaml'
+    - '!stake-pool/js/**'
   push:
     branches: [master]
     paths:
@@ -17,7 +17,7 @@ on:
     - 'ci/*-version.sh'
     - 'ci/warning/purge-ubuntu-runner.sh'
     - '.github/workflows/pull-request-stake-pool.yml'
-    - 'pnpm-lock.yaml'
+    - '!stake-pool/js/**'
 
 jobs:
   cargo-test-sbf:
diff --git a/.github/workflows/pull-request-token-lending.yml b/.github/workflows/pull-request-token-lending.yml
index bcf83fe82..dd0441721 100644
--- a/.github/workflows/pull-request-token-lending.yml
+++ b/.github/workflows/pull-request-token-lending.yml
@@ -6,14 +6,14 @@ on:
     - 'token-lending/**'
     - 'token/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!token-lending/js/**'
   push:
     branches: [master]
     paths:
     - 'token-lending/**'
     - 'token/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!token-lending/js/**'
 
 jobs:
   cargo-test-sbf:
diff --git a/.github/workflows/pull-request-token-metadata.yml b/.github/workflows/pull-request-token-metadata.yml
index d2af42bcf..ef91340a5 100644
--- a/.github/workflows/pull-request-token-metadata.yml
+++ b/.github/workflows/pull-request-token-metadata.yml
@@ -7,7 +7,7 @@ on:
     - 'token/program-2022/**'
     - 'ci/*-version.sh'
     - '.github/workflows/pull-request-token-metadata.yml'
-    - 'pnpm-lock.yaml'
+    - '!token-metadata/js/**'
   push:
     branches: [master]
     paths:
@@ -15,7 +15,7 @@ on:
     - 'token/program-2022/**'
     - 'ci/*-version.sh'
     - '.github/workflows/pull-request-token-metadata.yml'
-    - 'pnpm-lock.yaml'
+    - '!token-metadata/js/**'
 
 jobs:
   cargo-test-sbf:
diff --git a/.github/workflows/pull-request-token-swap.yml b/.github/workflows/pull-request-token-swap.yml
index 74e28b2cb..f2149696b 100644
--- a/.github/workflows/pull-request-token-swap.yml
+++ b/.github/workflows/pull-request-token-swap.yml
@@ -7,7 +7,7 @@ on:
     - 'token/**'
     - 'libraries/math/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!token-swap/js/**'
   push:
     branches: [master]
     paths:
@@ -15,7 +15,7 @@ on:
     - 'token/**'
     - 'libraries/math/**'
     - 'ci/*-version.sh'
-    - 'pnpm-lock.yaml'
+    - '!token-swap/js/**'
 
 jobs:
   cargo-test-sbf:
@@ -87,7 +87,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NODE_VERSION: 16.x
-    needs: cargo-test-sbf
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ env.NODE_VERSION }}
@@ -103,11 +102,6 @@ jobs:
           key: node-${{ hashFiles('pnpm-lock.yaml') }}
           restore-keys: |
             node-
-      - name: Download programs
-        uses: actions/download-artifact@v2
-        with:
-          name: token-swap-programs
-          path: target/deploy
       - run: ./ci/js-test-token-swap.sh
 
   fuzz:
diff --git a/.github/workflows/pull-request-token.yml b/.github/workflows/pull-request-token.yml
index 43104fd42..325f7861e 100644
--- a/.github/workflows/pull-request-token.yml
+++ b/.github/workflows/pull-request-token.yml
@@ -7,7 +7,7 @@ on:
     - 'token/**'
     - 'ci/*-version.sh'
     - '.github/workflows/pull-request-token.yml'
-    - 'pnpm-lock.yaml'
+    - '!token/js/**'
   push:
     branches: [master]
     paths:
@@ -15,7 +15,7 @@ on:
     - 'token/**'
     - 'ci/*-version.sh'
     - '.github/workflows/pull-request-token.yml'
-    - 'pnpm-lock.yaml'
+    - '!token/js/**'
 
 jobs:
   cargo-test-sbf:
@@ -255,7 +255,6 @@ jobs:
     runs-on: ubuntu-latest
     env:
       NODE_VERSION: 16.x
-    needs: [cargo-test-sbf, cargo-test-sbf-associated-token-account, cargo-test-sbf-transfer-hook]
     steps:
       - uses: actions/checkout@v2
       - name: Use Node.js ${{ env.NODE_VERSION }}
@@ -271,21 +270,6 @@ jobs:
           key: node-${{ hashFiles('pnpm-lock.yaml') }}
           restore-keys: |
             node-
-      - name: Download programs
-        uses: actions/download-artifact@v2
-        with:
-          name: token-programs
-          path: target/deploy
-      - name: Download associated-token-account program
-        uses: actions/download-artifact@v2
-        with:
-          name: associated-token-account-program
-          path: target/deploy
-      - name: Download spl-transfer-hook-example program
-        uses: actions/download-artifact@v2
-        with:
-          name: spl-transfer-hook-example
-          path: target/deploy
       - run: ./ci/js-test-token.sh
 
   cargo-build-test-cli:
diff --git a/name-service/js/package.json b/name-service/js/package.json
index 2fc2144d4..56d27cac1 100644
--- a/name-service/js/package.json
+++ b/name-service/js/package.json
@@ -20,6 +20,7 @@
   "scripts": {
     "dev": "tsc && node --trace-warnings dist/transfer.js",
     "build": "tsc",
+    "build:program": "cargo build-sbf --manifest-path=../program/Cargo.toml",
     "prepublish": "tsc",
     "lint": "npm run pretty && eslint --max-warnings 0 'src/*.ts'",
     "lint:fix": "npm run pretty:fix && eslint 'src/*.ts' --fix",
diff --git a/package.json b/package.json
index f43c544cb..3d0e83e2b 100644
--- a/package.json
+++ b/package.json
@@ -12,6 +12,7 @@
     ],
     "scripts": {
         "build": "turbo run build",
+        "build:program": "turbo run build:program",
         "clean": "turbo run clean",
         "lint": "turbo run lint",
         "lint:fix": "turbo run lint:fix",
diff --git a/single-pool/js/packages/classic/package.json b/single-pool/js/packages/classic/package.json
index 400d698ce..cfd75055e 100644
--- a/single-pool/js/packages/classic/package.json
+++ b/single-pool/js/packages/classic/package.json
@@ -12,6 +12,7 @@
   "scripts": {
     "clean": "rm -rf dist/*",
     "build": "tsc -p tsconfig.json && tsc -p tsconfig-cjs.json && ./ts-fixup.sh",
+    "build:program": "cargo build-sbf --manifest-path=../../../program/Cargo.toml",
     "lint": "eslint --max-warnings 0 .",
     "lint:fix": "eslint . --fix",
     "test": "sed -i '1s/.*/{ \"type\": \"module\",/' package.json && NODE_OPTIONS='--loader=tsx' ava ; ret=$?; sed -i '1s/.*/{/' package.json && exit $ret"
diff --git a/stake-pool/js/package.json b/stake-pool/js/package.json
index a394f6a97..d2dbae8fb 100644
--- a/stake-pool/js/package.json
+++ b/stake-pool/js/package.json
@@ -4,6 +4,7 @@
   "description": "SPL Stake Pool Program JS API",
   "scripts": {
     "build": "tsc && cross-env NODE_ENV=production rollup -c",
+    "build:program": "cargo build-sbf --manifest-path=../program/Cargo.toml",
     "lint": "eslint --max-warnings 0 .",
     "lint:fix": "eslint . --fix",
     "test": "jest",
diff --git a/token-swap/js/package.json b/token-swap/js/package.json
index 6ee03608c..c04c0e9a8 100644
--- a/token-swap/js/package.json
+++ b/token-swap/js/package.json
@@ -34,8 +34,9 @@
     "clean": "rm -rf lib/*",
     "build": "tsc --build --verbose tsconfig.all.json",
     "postbuild": "echo '{\"type\":\"commonjs\"}' > lib/cjs/package.json && echo '{\"type\":\"module\"}' > lib/esm/package.json",
-    "test": "mocha test",
-    "start-with-test-validator": "start-server-and-test 'solana-test-validator --bpf-program SwapsVeCiPHMUAtzQWZw7RjsKjgCjhwU55QGu4U1Szw ../../target/deploy/spl_token_swap.so --reset --quiet' http://127.0.0.1:8899/health test",
+    "build:program": "cargo build-sbf --manifest-path=../program/Cargo.toml",
+    "test:js": "mocha test",
+    "test": "start-server-and-test 'solana-test-validator --bpf-program SwapsVeCiPHMUAtzQWZw7RjsKjgCjhwU55QGu4U1Szw ../../target/deploy/spl_token_swap.so --reset --quiet' http://127.0.0.1:8899/health test:js",
     "lint": "npm run pretty && eslint --max-warnings 0 .",
     "lint:fix": "npm run pretty:fix && eslint . --fix",
     "build:program": "cargo build-sbf --manifest-path ../program/Cargo.toml",
diff --git a/token/js/package.json b/token/js/package.json
index de5741a68..2570b7622 100644
--- a/token/js/package.json
+++ b/token/js/package.json
@@ -33,6 +33,7 @@
         "clean": "shx rm -rf lib **/*.tsbuildinfo || true",
         "build": "tsc --build --verbose tsconfig.all.json",
         "postbuild": "shx echo '{ \"type\": \"commonjs\" }' > lib/cjs/package.json",
+        "build:program": "cargo build-sbf --manifest-path=../program/Cargo.toml && cargo build-sbf --manifest-path=../program-2022/Cargo.toml && cargo build-sbf --manifest-path=../../associated-token-account/program/Cargo.toml && cargo build-sbf --manifest-path=../transfer-hook/example/Cargo.toml",
         "watch": "tsc --build --verbose --watch tsconfig.all.json",
         "release": "npm run clean && npm run build",
         "fmt": "prettier --write '{*,**/*}.{ts,tsx,js,jsx,json}'",
diff --git a/turbo.json b/turbo.json
index 8fa5de6b1..6c981fcab 100644
--- a/turbo.json
+++ b/turbo.json
@@ -5,6 +5,9 @@
             "dependsOn": ["^build"],
             "outputs": ["dist/**", "lib/**"]
         },
+        "build:program": {
+            "dependsOn": ["^build:program"]
+        },
         "clean": {},
         "lint:fix": {
             "inputs": ["src/**", "test/**"],


Test patch:
diff --git a/ci/js-test-tlv.sh b/ci/js-test-libraries.sh
similarity index 100%
rename from ci/js-test-tlv.sh
rename to ci/js-test-libraries.sh
diff --git a/ci/js-test-name-service.sh b/ci/js-test-name-service.sh
index 55c3f9d3c..f854b5fae 100755
--- a/ci/js-test-name-service.sh
+++ b/ci/js-test-name-service.sh
@@ -9,5 +9,6 @@ cd name-service/js
 
 pnpm install
 pnpm lint
+pnpm build:program
 pnpm build
 pnpm test
diff --git a/ci/js-test-single-pool.sh b/ci/js-test-single-pool.sh
index da794c9df..3842116df 100755
--- a/ci/js-test-single-pool.sh
+++ b/ci/js-test-single-pool.sh
@@ -12,6 +12,7 @@ pnpm lint
 pnpm build
 
 cd ../classic
+pnpm build:program
 pnpm lint
 pnpm build
 pnpm test
diff --git a/ci/js-test-token-swap.sh b/ci/js-test-token-swap.sh
index d49c7636d..f08a7ea72 100755
--- a/ci/js-test-token-swap.sh
+++ b/ci/js-test-token-swap.sh
@@ -8,7 +8,6 @@ pnpm install
 pnpm build
 
 cd token-swap/js
+pnpm build:program
 pnpm lint
-pnpm start-with-test-validator
-(cd ../../target/deploy && mv spl_token_swap_production.so spl_token_swap.so)
-SWAP_PROGRAM_OWNER_FEE_ADDRESS="HfoTxFR1Tm6kGmWgYWD6J7YHVy1UwqSULUGVLXkJqaKN" pnpm start-with-test-validator
+pnpm test
diff --git a/ci/js-test-token.sh b/ci/js-test-token.sh
index 3f71c204b..f7d0db98f 100755
--- a/ci/js-test-token.sh
+++ b/ci/js-test-token.sh
@@ -9,5 +9,6 @@ pnpm install
 pnpm build
 
 cd token/js
+pnpm build:program
 pnpm lint
 pnpm test