curl-impersonate: add zstd compression options

backend/functions/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get update \
     && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
     && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
     && apt-get update \
-    && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
+    && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 zstd \
     --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 

backend/functions/package-lock.json

@@ -40,6 +40,7 @@
         "puppeteer-page-proxy": "^1.3.0",
         "robots-parser": "^3.0.1",
         "set-cookie-parser": "^2.6.0",
+        "simple-zstd": "^1.4.2",
         "stripe": "^11.11.0",
         "tiktoken": "^1.0.16",
         "tld-extract": "^2.1.0",
@@ -3701,9 +3702,7 @@
     "node_modules/buffer-from": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
-      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
-      "dev": true,
-      "peer": true
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ=="
     },
     "node_modules/busboy": {
       "version": "1.6.0",
@@ -4761,6 +4760,12 @@
         "url": "https://github.com/fb55/domutils?sponsor=1"
       }
     },
+    "node_modules/duplex-maker": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/duplex-maker/-/duplex-maker-1.0.0.tgz",
+      "integrity": "sha512-KoHuzggxg7f+vvjqOHfXxaQYI1POzBm+ah0eec7YDssZmbt6QFBI8d1nl5GQwAgR2f+VQCPvyvZtmWWqWuFtlA==",
+      "license": "MIT"
+    },
     "node_modules/duplexify": {
       "version": "4.1.3",
       "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz",
@@ -7288,6 +7293,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-zst": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-zst/-/is-zst-1.0.0.tgz",
+      "integrity": "sha512-ZA5lvshKAl8z30dX7saXLpVhpsq3d2EHK9uf7qtUjnOtdw4XBpAoWb2RvZ5kyoaebdoidnGI0g2hn9Z7ObPbww==",
+      "license": "MIT"
+    },
     "node_modules/isarray": {
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz",
@@ -10043,6 +10054,75 @@
         "path2d": "^0.2.0"
       }
     },
+    "node_modules/peek-stream": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/peek-stream/-/peek-stream-1.1.3.tgz",
+      "integrity": "sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==",
+      "license": "MIT",
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "duplexify": "^3.5.0",
+        "through2": "^2.0.3"
+      }
+    },
+    "node_modules/peek-stream/node_modules/duplexify": {
+      "version": "3.7.1",
+      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
+      "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==",
+      "license": "MIT",
+      "dependencies": {
+        "end-of-stream": "^1.0.0",
+        "inherits": "^2.0.1",
+        "readable-stream": "^2.0.0",
+        "stream-shift": "^1.0.0"
+      }
+    },
+    "node_modules/peek-stream/node_modules/isarray": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
+      "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==",
+      "license": "MIT"
+    },
+    "node_modules/peek-stream/node_modules/readable-stream": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz",
+      "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==",
+      "license": "MIT",
+      "dependencies": {
+        "core-util-is": "~1.0.0",
+        "inherits": "~2.0.3",
+        "isarray": "~1.0.0",
+        "process-nextick-args": "~2.0.0",
+        "safe-buffer": "~5.1.1",
+        "string_decoder": "~1.1.1",
+        "util-deprecate": "~1.0.1"
+      }
+    },
+    "node_modules/peek-stream/node_modules/safe-buffer": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
+      "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
+      "license": "MIT"
+    },
+    "node_modules/peek-stream/node_modules/string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "license": "MIT",
+      "dependencies": {
+        "safe-buffer": "~5.1.0"
+      }
+    },
+    "node_modules/peek-stream/node_modules/through2": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz",
+      "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==",
+      "license": "MIT",
+      "dependencies": {
+        "readable-stream": "~2.3.6",
+        "xtend": "~4.0.1"
+      }
+    },
     "node_modules/pend": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
@@ -10345,6 +10425,19 @@
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
       "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
     },
+    "node_modules/process-streams": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/process-streams/-/process-streams-1.0.3.tgz",
+      "integrity": "sha512-xkIaM5vYnyekB88WyET78YEqXiaJRy0xcvIdE22n+myhvBT7LlLmX6iAtq7jDvVH8CUx2rqQsd32JdRyJMV3NA==",
+      "funding": [
+        "https://www.paypal.com/donate/?hosted_button_id=GB656ZSAEQEXN",
+        "https://de.liberapay.com/nils.knappmeier/"
+      ],
+      "license": "MIT",
+      "dependencies": {
+        "duplex-maker": "^1.0.0"
+      }
+    },
     "node_modules/process-warning": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-3.0.0.tgz",
@@ -11576,6 +11669,18 @@
         "simple-concat": "^1.0.0"
       }
     },
+    "node_modules/simple-zstd": {
+      "version": "1.4.2",
+      "resolved": "https://registry.npmjs.org/simple-zstd/-/simple-zstd-1.4.2.tgz",
+      "integrity": "sha512-kGYEvT33M5XfyQvvW4wxl3eKcWbdbCc1V7OZzuElnaXft0qbVzoIIXHXiCm3JCUki+MZKKmvjl8p2VGLJc5Y/A==",
+      "license": "MIT",
+      "dependencies": {
+        "is-zst": "^1.0.0",
+        "peek-stream": "^1.1.3",
+        "process-streams": "^1.0.1",
+        "through2": "^4.0.2"
+      }
+    },
     "node_modules/sisteransi": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
@@ -12932,6 +13037,15 @@
         "node": ">=4.0"
       }
     },
+    "node_modules/xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4"
+      }
+    },
     "node_modules/y18n": {
       "version": "5.0.8",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

backend/functions/package.json

@@ -60,6 +60,7 @@
     "puppeteer-page-proxy": "^1.3.0",
     "robots-parser": "^3.0.1",
     "set-cookie-parser": "^2.6.0",
+    "simple-zstd": "^1.4.2",
     "stripe": "^11.11.0",
     "tiktoken": "^1.0.16",
     "tld-extract": "^2.1.0",

backend/functions/src/services/curl.ts

@@ -11,6 +11,7 @@ import { TempFileManager } from '../shared';
 import { readFile } from 'fs/promises';
 import { pathToFileURL } from 'url';
 import { createBrotliDecompress, createInflate, createGunzip } from 'zlib';
+import { ZSTDDecompress } from 'simple-zstd';
 
 @singleton()
 export class CurlControl extends AsyncService {
@@ -43,7 +44,7 @@ export class CurlControl extends AsyncService {
             'Sec-Fetch-Mode': 'navigate',
             'Sec-Fetch-User': '?1',
             'Sec-Fetch-Dest': 'document',
-            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Encoding': 'gzip, deflate, br, zstd',
             'Accept-Language': 'en-US,en;q=0.9',
         };
 
@@ -157,6 +158,12 @@ export class CurlControl extends AsyncService {
                         stream = decompressed;
                         break;
                     }
+                    case 'zstd': {
+                        const decompressed = ZSTDDecompress();
+                        stream.pipe(decompressed);
+                        stream = decompressed;
+                        break;
+                    }
                     default: {
                         break;
                     }

backend/functions/src/types.d.ts

@@ -14,8 +14,15 @@ declare module 'jsdom' {
         constructor(html: string, options?: any);
         window: typeof window;
     }
-    export class VirtualConsole extends EventEmitter{
+    export class VirtualConsole extends EventEmitter {
         constructor();
         sendTo(console: any, options?: any);
     }
 }
+
+declare module 'simple-zstd' {
+    import { Duplex } from 'stream';
+    export function ZSTDCompress(lvl: Number): Duplex;
+    export function ZSTDDecompress(): Duplex;
+    export function ZSTDDecompressMaybe(): Duplex;
+}