Repository: vercel/nft (javascript)

Issue:
Tracing .lua files
There are some libraries with .lua files in core, e.g. [bull](https://github.com/OptimalBits/bull), [bullmq](https://github.com/taskforcesh/bullmq), [bee-queue](https://github.com/bee-queue/bee-queue). And these files are ignored by default in `nft`.
I see support for bull in [special-cases](https://github.com/vercel/nft/blob/main/src/utils/special-cases.ts) and I can add the same for other libs. But this way looks difficult to maintain:
- if the library structure changes, need to add a condition in `nft`
- if new libraries appear, need to add them to `nft`

And then wait for next.js release...

Unfortunately, next.js doesn't have support for "custom" files in API-routes, only in pages (via `unstable_includeFiles`). And there is no support from Vercel also ([includeFiles doesn't work with next.js](https://vercel.com/support/articles/how-can-i-use-files-in-serverless-functions)). So current solutions to work with e.g. `bee-queue` look uncomfortable:
1. monkey patch for next.js
2. or mix `/pages/api` with `/api` (and use `vercel dev` instead of `next dev`)

Related issues:
https://github.com/vercel/nft/issues/35
https://github.com/taskforcesh/bullmq/issues/469
https://github.com/vercel/next.js/discussions/14807


Thank you!


Discussion:
All assets are traced, regardless of extension, as long as the files are read in a statically analyzable way.

For example, you should be able to do `const file = fs.readFile(path.join(__dirname, 'file.lua'))`.

The best way to solve this problem is to make changes upstream to `bull` so it doesn't need to be special cased in `nft`.

I believe changing this line from `dir` to `__dirname` should be sufficient https://github.com/OptimalBits/bull/blob/65972184e7dbf156bbf4a4edd8d70a6bf3019b53/lib/commands/index.js#L34
I created a PR https://github.com/OptimalBits/bull/pull/2285
Thank you for the reply @styfle!

>For example, you should be able to do const file = fs.readFile(path.join(__dirname, 'file.lua'))

It doesn't work on the application layer because files from `node_modules` are ignored.

I do the following in my application (next 12.0.10), just for example:
`const file = fs.readFileSync(path.join(__dirname, 'src/addJob.lua'))` ok
`const file = fs.readFileSync(path.join(__dirname, 'node_modules/bee-queue/lib/lua/addJob.lua'))` not ok, file is missing

>I believe changing this line from dir to __dirname should be sufficient

Sorry, but this solution doesn't seem to be working. I made the same changes as in your PR, removed special case from nft, and rebuilt the application. After that .lua files are missing.

And your PR won't solve this issue anyway, because there are several libraries with .lua files, not only `bull`. I mentioned some of them in the first comment.

I can create a repo to help reproduce the problem if needed ;)
@voronin-ivan Ah yes you're right, since we have the bull special condition.

I had to rename the package to reproduce.

```
mkdir example
cd example
yarn init -y
yarn add bull
mv node_modules/bull node_modules/bull-example # ensure special case is not hit
echo "require('bull-example')" > index.js
nft print index.js | grep lua
```

Then I applied my patch and it still didn't work.

It seems like the case with `__dirname` is not detected.

Once we get that fixed, then the change to `bull` will work.

@guybedford Would you agree that https://github.com/vercel/nft/pull/267 is a bug?

Patch:
diff --git a/package.json b/package.json
--- a/package.json
+++ b/package.json
@@ -55,6 +55,7 @@
     "bcrypt": "^5.0.1",
     "browserify-middleware": "^8.1.1",
     "bull": "^3.10.0",
+    "bullmq": "^1.74.2",
     "camaro": "^6.1.0",
     "canvas": "^2.9.0",
     "chromeless": "^1.5.2",
@@ -109,6 +110,7 @@
     "redis": "^3.1.1",
     "remark-parse": "^10.0.0",
     "remark-prism": "^1.3.6",
+    "rimraf": "^3.0.2",
     "rxjs": "^6.5.2",
     "saslprep": "^1.0.3",
     "semver": "^7.1.1",
diff --git a/yarn.lock b/yarn.lock
--- a/yarn.lock
+++ b/yarn.lock
@@ -3837,6 +3837,21 @@ bull@^3.10.0:
     util.promisify "^1.0.1"
     uuid "^8.3.0"
 
+bullmq@^1.74.2:
+  version "1.74.2"
+  resolved "https://registry.yarnpkg.com/bullmq/-/bullmq-1.74.2.tgz#56310618df4b6eeadd9ce4ffb9add81cbc9189e2"
+  integrity sha512-qf0xjF3NDbZqi6ovxAA9QPrPdEgl3BrdYzQO5Pr+ECNIfscLpSXz87kVXI0oTPeDqxw5EJx4sh8EFf2RvVPYTg==
+  dependencies:
+    cron-parser "^2.18.0"
+    get-port "^5.1.1"
+    glob "^7.2.0"
+    ioredis "^4.28.2"
+    lodash "^4.17.21"
+    msgpackr "^1.4.6"
+    semver "^6.3.0"
+    tslib "^1.14.1"
+    uuid "^8.3.2"
+
 busboy@^0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.1.tgz#170899274c5bf38aae27d5c62b71268cd585fd1b"
@@ -4820,7 +4835,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
-cron-parser@^2.13.0:
+cron-parser@^2.13.0, cron-parser@^2.18.0:
   version "2.18.0"
   resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.18.0.tgz#de1bb0ad528c815548371993f81a54e5a089edcf"
   integrity sha512-s4odpheTyydAbTBQepsqd2rNWGa2iV3cyo8g7zbI2QQYGLVsfbhmwukayS1XHppe02Oy1fg7mg6xoaraVJeEcg==
@@ -7061,6 +7076,18 @@ glob@^7.0.5, glob@^7.1.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4, gl
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+glob@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
+  integrity sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^3.0.4"
+    once "^1.3.0"
+    path-is-absolute "^1.0.0"
+
 global-dirs@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/global-dirs/-/global-dirs-0.1.1.tgz#b319c0dd4607f353f3be9cca4c72fc148c49f445"
@@ -7938,6 +7965,23 @@ ioredis@^4.11.1, ioredis@^4.22.0:
     redis-parser "^3.0.0"
     standard-as-callback "^2.1.0"
 
+ioredis@^4.28.2:
+  version "4.28.5"
+  resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-4.28.5.tgz#5c149e6a8d76a7f8fa8a504ffc85b7d5b6797f9f"
+  integrity sha512-3GYo0GJtLqgNXj4YhrisLaNNvWSNwSS2wS4OELGfGxH8I69+XfNdnmV1AyN+ZqMh0i7eX+SWjrwFKDBDgfBC1A==
+  dependencies:
+    cluster-key-slot "^1.1.0"
+    debug "^4.3.1"
+    denque "^1.1.0"
+    lodash.defaults "^4.2.0"
+    lodash.flatten "^4.4.0"
+    lodash.isarguments "^3.1.0"
+    p-map "^2.1.0"
+    redis-commands "1.7.0"
+    redis-errors "^1.2.0"
+    redis-parser "^3.0.0"
+    standard-as-callback "^2.1.0"
+
 ip-regex@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9"
@@ -9922,6 +9966,11 @@ lodash.includes@^4.3.0:
   resolved "https://registry.yarnpkg.com/lodash.includes/-/lodash.includes-4.3.0.tgz#60bb98a87cb923c68ca1e51325483314849f553f"
   integrity sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=
 
+lodash.isarguments@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a"
+  integrity sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo=
+
 lodash.isboolean@^3.0.3:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz#6c2e171db2a257cd96802fd43b01b20d5f5870f6"
@@ -11005,6 +11054,21 @@ msgpack5@^4.2.0:
     readable-stream "^2.3.6"
     safe-buffer "^5.1.2"
 
+msgpackr-extract@^1.0.14:
+  version "1.0.16"
+  resolved "https://registry.yarnpkg.com/msgpackr-extract/-/msgpackr-extract-1.0.16.tgz#701c4f6e6f25c100ae84557092274e8fffeefe45"
+  integrity sha512-fxdRfQUxPrL/TizyfYfMn09dK58e+d65bRD/fcaVH4052vj30QOzzqxcQIS7B0NsqlypEQ/6Du3QmP2DhWFfCA==
+  dependencies:
+    nan "^2.14.2"
+    node-gyp-build "^4.2.3"
+
+msgpackr@^1.4.6:
+  version "1.5.4"
+  resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.5.4.tgz#2b6ea6cb7d79c0ad98fc76c68163c48eda50cf0d"
+  integrity sha512-Z7w5Jg+2Q9z9gJxeM68d7tSuWZZGnFIRhZnyqcZCa/1dKkhOCNvR1TUV3zzJ3+vj78vlwKRzUgVDlW4jiSOeDA==
+  optionalDependencies:
+    msgpackr-extract "^1.0.14"
+
 mutationobserver-shim@^0.3.2:
   version "0.3.7"
   resolved "https://registry.yarnpkg.com/mutationobserver-shim/-/mutationobserver-shim-0.3.7.tgz#8bf633b0c0b0291a1107255ed32c13088a8c5bf3"
@@ -11056,7 +11120,7 @@ nan@^2.12.1, nan@^2.13.2, nan@^2.14.0:
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.2.tgz#f5376400695168f4cc694ac9393d0c9585eeea19"
   integrity sha512-M2ufzIiINKCuDfBSAUr1vWQ+vuVcA9kqx8JJUsbQi6yf1uGRyb7HfpdfUr5qLXf3B/t8dPvcjhKMmlfnP47EzQ==
 
-nan@^2.15.0:
+nan@^2.14.2, nan@^2.15.0:
   version "2.15.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.15.0.tgz#3f34a473ff18e15c1b5626b62903b5ad6e665fee"
   integrity sha512-8ZtvEnA2c5aYCZYd1cvgdnU6cqwixRoYg70xPLWUws5ORTa/lnw+u4amixRS/Ac5U5mQVgp9pnlSUnbNWFaWZQ==
@@ -11176,10 +11240,10 @@ node-forge@^0.9.0:
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.2.tgz#b35a44c28889b2ea55cabf8c79e3563f9676190a"
   integrity sha512-naKSScof4Wn+aoHU6HBsifh92Zeicm1GDQKd1vp3Y/kOi8ub0DozCa9KpvYNCXslFHYRmLNiqRopGdTGwNLpNw==
 
-node-gyp-build@^4.2.2:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.2.3.tgz#ce6277f853835f718829efb47db20f3e4d9c4739"
-  integrity sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==
+node-gyp-build@^4.2.2, node-gyp-build@^4.2.3:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.3.0.tgz#9f256b03e5826150be39c764bf51e993946d71a3"
+  integrity sha512-iWjXZvmboq0ja1pUGULQBexmxq8CV4xBhX7VDOTbL7ZR4FOowwY/VOtRxBN/yKxmdGoIp4j5ysNT4u3S2pDQ3Q==
 
 node-gyp-build@~4.1.0:
   version "4.1.1"
@@ -15584,7 +15648,7 @@ ts-invariant@^0.4.0:
   dependencies:
     tslib "^1.9.3"
 
-tslib@^1.10.0, tslib@^1.11.1, tslib@^1.9.0, tslib@^1.9.3:
+tslib@^1.10.0, tslib@^1.11.1, tslib@^1.14.1, tslib@^1.9.0, tslib@^1.9.3:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"
   integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==
@@ -16125,7 +16189,7 @@ uuid@^7.0.0, uuid@^7.0.3:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-7.0.3.tgz#c5c9f2c8cf25dc0a372c4df1441c41f5bd0c680b"
   integrity sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==
 
-uuid@^8.0.0, uuid@^8.3.0:
+uuid@^8.0.0, uuid@^8.3.0, uuid@^8.3.2:
   version "8.3.2"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.3.2.tgz#80d5b5ced271bb9af6c445f21a1a04c606cefbe2"
   integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==


Test patch:
diff --git a/test/integration/bullmq.js b/test/integration/bullmq.js
new file mode 100644
--- /dev/null
+++ b/test/integration/bullmq.js
@@ -0,0 +1,26 @@
+const { BULL_REDIS_CONNECTION } = process.env;
+
+if (!BULL_REDIS_CONNECTION) {
+  console.log('Skipping bullmq integration test');
+  console.log('Create cache on redislabs.com and export BULL_REDIS_CONNECTION="redis://:password@hostname:port"');
+  return;
+}
+
+const url = new URL(BULL_REDIS_CONNECTION);
+const connection = {
+  username: url.username,
+  password: url.password,
+  host: url.hostname,
+  port: Number(url.port),
+}
+
+const { Queue } = require('bullmq');
+
+async function main() {
+  const queue = new Queue('foo', { connection });
+  await queue.add('job', { id: 'one' });
+  await queue.add('job', { id: 'two' });
+  await queue.close();
+}
+
+main().then(() => console.log('bullmq success')).catch(console.error);