Upload 376 files

.gitattributes

@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/bettertogether.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/catmakeup.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/catphonestand.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/catsleep.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/catswitchboxes.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/cattired.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/developers.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/meowthball.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/meowthcooking.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/meowthpolishegg.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/meowthproductions.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/meowthsnap.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/meowthstrong.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/millionusers.webp filter=lfs diff=lfs merge=lfs -text
+web/static/update-banners/valentines.webp filter=lfs diff=lfs merge=lfs -text

api/README.md

@@ -1,4 +1,64 @@
 # cobalt api
+this directory includes the source code for cobalt api. it's made with [express.js](https://www.npmjs.com/package/express) and love!
+
+## running your own instance
+if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
+we recommend to use docker compose unless you intend to run cobalt for developing/debugging purposes.
+
+## accessing the api
+there is currently no publicly available pre-hosted api.
+we recommend [deploying your own instance](/docs/run-an-instance.md) if you wish to use the cobalt api.
+
+you can read [the api documentation here](/docs/api.md).
+
+## supported services
+this list is not final and keeps expanding over time!
+if the desired service isn't supported yet, feel free to create an appropriate issue (or a pull request 👀).
+
+| service           | video + audio | only audio | only video | metadata | rich file names |
+| :--------         | :-----------: | :--------: | :--------: | :------: | :-------------: |
+| bilibili          | ✅            | ✅         | ✅         | ➖         | ➖              |
+| bluesky           | ✅            | ✅         | ✅         | ➖         | ➖              |
+| dailymotion       | ✅            | ✅         | ✅         | ✅         | ✅              |
+| instagram         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| facebook          | ✅            | ❌         | ✅         | ➖         | ➖              |
+| loom              | ✅            | ❌         | ✅         | ✅         | ➖              |
+| ok.ru             | ✅            | ❌         | ✅         | ✅         | ✅              |
+| pinterest         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| reddit            | ✅            | ✅         | ✅         | ❌         | ❌              |
+| rutube            | ✅            | ✅         | ✅         | ✅         | ✅              |
+| snapchat          | ✅            | ✅         | ✅         | ➖         | ➖              |
+| soundcloud        | ➖            | ✅         | ➖         | ✅         | ✅              |
+| streamable        | ✅            | ✅         | ✅         | ➖         | ➖              |
+| tiktok            | ✅            | ✅         | ✅         | ❌         | ❌              |
+| tumblr            | ✅            | ✅         | ✅         | ➖         | ➖              |
+| twitch clips      | ✅            | ✅         | ✅         | ✅         | ✅              |
+| twitter/x         | ✅            | ✅         | ✅         | ➖         | ➖              |
+| vimeo             | ✅            | ✅         | ✅         | ✅         | ✅              |
+| vk videos & clips | ✅            | ❌         | ✅         | ✅         | ✅              |
+| xiaohongshu       | ✅            | ✅         | ✅         | ➖         | ➖              |
+| youtube           | ✅            | ✅         | ✅         | ✅         | ✅              |
+
+| emoji   | meaning                 |
+| :-----: | :---------------------- |
+| ✅      | supported               |
+| ➖      | unreasonable/impossible |
+| ❌      | not supported           |
+
+### additional notes or features (per service)
+| service    | notes or features                                                                                                    |
+| :--------  | :-----                                                                                                               |
+| instagram  | supports reels, photos, and videos. lets you pick what to save from multi-media posts.                               |
+| facebook   | supports public accessible videos content only.                                                                      |
+| pinterest  | supports photos, gifs, videos and stories.                                                                           |
+| reddit     | supports gifs and videos.                                                                                            |
+| snapchat   | supports spotlights and stories. lets you pick what to save from stories.                                            |
+| rutube     | supports yappy & private links.                                                                                      |
+| soundcloud | supports private links.                                                                                              |
+| tiktok     | supports videos with or without watermark, images from slideshow without watermark, and full (original) audios.      |
+| twitter/x  | lets you pick what to save from multi-media posts. may not be 100% reliable due to current management.               |
+| vimeo      | audio downloads are only available for dash.                                                                         |
+| youtube    | supports videos, music, and shorts. 8K, 4K, HDR, VR, and high FPS videos. rich metadata & dubs. h264/av1/vp9 codecs. |
 
 ## license
 cobalt api code is licensed under [AGPL-3.0](LICENSE).
@@ -9,14 +69,35 @@ as long as you:
 - provide a link to the license and indicate if changes to the code were made, and
 - release the code under the **same license**
 
-## running your own instance
-if you want to run your own instance for whatever purpose, [follow this guide](/docs/run-an-instance.md).
-it's *highly* recommended to use a docker compose method unless you run for developing/debugging purposes.
+## open source acknowledgements
+### ffmpeg
+cobalt relies on ffmpeg for muxing and encoding media files. ffmpeg is absolutely spectacular and we're privileged to have an ability to use it for free, just like anyone else. we believe it should be way more recognized.
 
-## accessing the api
-currently, there is no publicly accessible main api. we plan on providing a public api for
-cobalt 10 in some form in the future. we recommend deploying your own instance if you wish
-to use the latest api. you can access [the documentation](/docs/api.md) for it here.
+you can [support ffmpeg here](https://ffmpeg.org/donations.html)!
+
+### youtube.js
+cobalt relies on **[youtube.js](https://github.com/LuanRT/YouTube.js)** for interacting with youtube's innertube api, it wouldn't have been possible without this package.
+
+you can support the developer via various methods listed on their github page!
+(linked above)
+
+### many others
+cobalt-api also depends on:
+
+- **[content-disposition-header](https://www.npmjs.com/package/content-disposition-header)** to simplify the provision of `content-disposition` headers.
+- **[cors](https://www.npmjs.com/package/cors)** to manage cross-origin resource sharing within expressjs.
+- **[dotenv](https://www.npmjs.com/package/dotenv)** to load environment variables from the `.env` file.
+- **[express](https://www.npmjs.com/package/express)** as the backbone of cobalt servers.
+- **[express-rate-limit](https://www.npmjs.com/package/express-rate-limit)** to rate limit api endpoints.
+- **[ffmpeg-static](https://www.npmjs.com/package/ffmpeg-static)** to get binaries for ffmpeg depending on the platform.
+- **[hls-parser](https://www.npmjs.com/package/hls-parser)** to parse HLS playlists according to spec (very impressive stuff).
+- **[ipaddr.js](https://www.npmjs.com/package/ipaddr.js)** to parse ip addresses (used for rate limiting).
+- **[nanoid](https://www.npmjs.com/package/nanoid)** to generate unique identifiers for each requested tunnel.
+- **[set-cookie-parser](https://www.npmjs.com/package/set-cookie-parser)** to parse cookies that cobalt receives from certain services.
+- **[undici](https://www.npmjs.com/package/undici)** for making http requests.
+- **[url-pattern](https://www.npmjs.com/package/url-pattern)** to match provided links with supported patterns.
+- **[zod](https://www.npmjs.com/package/zod)** to lock down the api request schema.
+- **[@datastructures-js/priority-queue](https://www.npmjs.com/package/@datastructures-js/priority-queue)** for sorting stream caches for future clean up (without redis).
+- **[@imput/psl](https://www.npmjs.com/package/@imput/psl)** as the domain name parser, our fork of [psl](https://www.npmjs.com/package/psl).
 
-if you are looking for the documentation for the old (7.x) api, you can find
-it [here](https://github.com/imputnet/cobalt/blob/7/docs/api.md)
+...and many other packages that these packages rely on.

api/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@imput/cobalt-api",
     "description": "save what you love",
-    "version": "10.1.0",
+    "version": "10.9.1",
     "author": "imput",
     "exports": "./src/cobalt.js",
     "type": "module",
@@ -10,9 +10,8 @@
     },
     "scripts": {
         "start": "node src/cobalt",
-        "setup": "node src/util/setup",
         "test": "node src/util/test",
-        "token:youtube": "node src/util/generate-youtube-tokens"
+        "token:jwt": "node src/util/generate-jwt-secret"
     },
     "repository": {
         "type": "git",
@@ -24,26 +23,27 @@
     },
     "homepage": "https://github.com/imputnet/cobalt#readme",
     "dependencies": {
+        "@datastructures-js/priority-queue": "^6.3.1",
+        "@imput/psl": "^2.0.4",
         "@imput/version-info": "workspace:^",
         "content-disposition-header": "0.6.0",
         "cors": "^2.8.5",
         "dotenv": "^16.0.1",
-        "esbuild": "^0.14.51",
-        "express": "^4.18.1",
-        "express-rate-limit": "^6.3.0",
+        "express": "^4.21.2",
+        "express-rate-limit": "^7.4.1",
         "ffmpeg-static": "^5.1.0",
         "hls-parser": "^0.10.7",
-        "ipaddr.js": "2.1.0",
-        "nanoid": "^4.0.2",
-        "node-cache": "^5.1.2",
-        "psl": "1.9.0",
+        "ipaddr.js": "2.2.0",
+        "nanoid": "^5.0.9",
         "set-cookie-parser": "2.6.0",
         "undici": "^5.19.1",
         "url-pattern": "1.0.3",
-        "youtubei.js": "^10.3.0",
+        "youtubei.js": "^13.3.0",
         "zod": "^3.23.8"
     },
     "optionalDependencies": {
-        "freebind": "^0.2.2"
+        "freebind": "^0.2.2",
+        "rate-limit-redis": "^4.2.0",
+        "redis": "^4.7.0"
     }
 }

api/src/cobalt.js

@@ -1,27 +1,32 @@
 import "dotenv/config";
 
 import express from "express";
+import cluster from "node:cluster";
 
-import path from 'path';
-import { fileURLToPath } from 'url';
+import path from "path";
+import { fileURLToPath } from "url";
 
-import { env } from "./config.js"
-import { Bright, Green, Red } from "./misc/console-text.js";
+import { env, isCluster } from "./config.js"
+import { Red } from "./misc/console-text.js";
+import { initCluster } from "./misc/cluster.js";
 
 const app = express();
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename).slice(0, -4);
 
-app.disable('x-powered-by');
+app.disable("x-powered-by");
 
 if (env.apiURL) {
-    const { runAPI } = await import('./core/api.js');
-    runAPI(express, app, __dirname)
+    const { runAPI } = await import("./core/api.js");
+
+    if (isCluster) {
+       await initCluster();
+    }
+
+    runAPI(express, app, __dirname, cluster.isPrimary);
 } else {
     console.log(
-        Red(`cobalt wasn't configured yet or configuration is invalid.\n`)
-        + Bright(`please run the setup script to fix this: `)
-        + Green(`npm run setup`)
+        Red("API_URL env variable is missing, cobalt api can't start.")
     )
 }

api/src/config.js

@@ -1,5 +1,7 @@
+import { Constants } from "youtubei.js";
 import { getVersion } from "@imput/version-info";
 import { services } from "./processing/service-config.js";
+import { supportsReusePort } from "./misc/cluster.js";
 
 const version = await getVersion();
 
@@ -13,6 +15,7 @@ const enabledServices = new Set(Object.keys(services).filter(e => {
 const env = {
     apiURL: process.env.API_URL || '',
     apiPort: process.env.API_PORT || 9000,
+    tunnelPort: process.env.API_PORT || 9000,
 
     listenAddress: process.env.API_LISTEN_ADDRESS,
     freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
@@ -25,8 +28,11 @@ const env = {
     rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
     rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
 
+    sessionRateLimitWindow: (process.env.SESSION_RATELIMIT_WINDOW && parseInt(process.env.SESSION_RATELIMIT_WINDOW)) || 60,
+    sessionRateLimit: (process.env.SESSION_RATELIMIT && parseInt(process.env.SESSION_RATELIMIT)) || 10,
+
     durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
-    streamLifespan: 90,
+    streamLifespan: (process.env.TUNNEL_LIFESPAN && parseInt(process.env.TUNNEL_LIFESPAN)) || 90,
 
     processingPriority: process.platform !== 'win32'
         && process.env.PROCESSING_PRIORITY
@@ -43,12 +49,46 @@ const env = {
                         && process.env.TURNSTILE_SECRET
                         && process.env.JWT_SECRET,
 
+    apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL),
+    authRequired: process.env.API_AUTH_REQUIRED === '1',
+    redisURL: process.env.API_REDIS_URL,
+    instanceCount: (process.env.API_INSTANCE_COUNT && parseInt(process.env.API_INSTANCE_COUNT)) || 1,
+    keyReloadInterval: 900,
+
     enabledServices,
+
+    customInnertubeClient: process.env.CUSTOM_INNERTUBE_CLIENT,
+    ytSessionServer: process.env.YOUTUBE_SESSION_SERVER,
+    ytSessionReloadInterval: 300,
+    ytSessionInnertubeClient: process.env.YOUTUBE_SESSION_INNERTUBE_CLIENT,
 }
 
 const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
 const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
 
+export const setTunnelPort = (port) => env.tunnelPort = port;
+export const isCluster = env.instanceCount > 1;
+
+if (env.sessionEnabled && env.jwtSecret.length < 16) {
+    throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
+}
+
+if (env.instanceCount > 1 && !env.redisURL) {
+    throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
+} else if (env.instanceCount > 1 && !await supportsReusePort()) {
+    console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
+    console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
+    console.error('(or other OS that supports it). for more info, see `reusePort` option on');
+    console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback');
+    throw new Error('SO_REUSEPORT is not supported');
+}
+
+if (env.customInnertubeClient && !Constants.SUPPORTED_CLIENTS.includes(env.customInnertubeClient)) {
+    console.error("CUSTOM_INNERTUBE_CLIENT is invalid. Provided client is not supported.");
+    console.error(`Supported clients are: ${Constants.SUPPORTED_CLIENTS.join(', ')}\n`);
+    throw new Error("Invalid CUSTOM_INNERTUBE_CLIENT");
+}
+
 export {
     env,
     genericUserAgent,

api/src/core/api.js

@@ -1,4 +1,5 @@
 import cors from "cors";
+import http from "node:http";
 import rateLimit from "express-rate-limit";
 import { setGlobalDispatcher, ProxyAgent } from "undici";
 import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
@@ -7,17 +8,21 @@ import jwt from "../security/jwt.js";
 import stream from "../stream/stream.js";
 import match from "../processing/match.js";
 
-import { env } from "../config.js";
+import { env, isCluster, setTunnelPort } from "../config.js";
 import { extract } from "../processing/url.js";
-import { languageCode } from "../misc/utils.js";
-import { Bright, Cyan } from "../misc/console-text.js";
-import { generateHmac, generateSalt } from "../misc/crypto.js";
+import { Green, Bright, Cyan } from "../misc/console-text.js";
+import { hashHmac } from "../security/secrets.js";
+import { createStore } from "../store/redis-ratelimit.js";
 import { randomizeCiphers } from "../misc/randomize-ciphers.js";
 import { verifyTurnstileToken } from "../security/turnstile.js";
 import { friendlyServiceName } from "../processing/service-alias.js";
 import { verifyStream, getInternalStream } from "../stream/manage.js";
 import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
 
+import * as APIKeys from "../security/api-keys.js";
+import * as Cookies from "../processing/cookie/manager.js";
+import * as YouTubeSession from "../processing/helpers/youtube-session.js";
+
 const git = {
     branch: await getBranch(),
     commit: await getCommit(),
@@ -28,7 +33,6 @@ const version = await getVersion();
 
 const acceptRegex = /^application\/json(; charset=utf-8)?$/;
 
-const ipSalt = generateSalt();
 const corsConfig = env.corsWildcard ? {} : {
     origin: env.corsURL,
     optionsSuccessStatus: 200
@@ -39,7 +43,7 @@ const fail = (res, code, context) => {
     res.status(status).json(body);
 }
 
-export const runAPI = (express, app, __dirname) => {
+export const runAPI = async (express, app, __dirname, isPrimary = true) => {
     const startTime = new Date();
     const startTimestamp = startTime.getTime();
 
@@ -57,38 +61,49 @@ export const runAPI = (express, app, __dirname) => {
         git,
     })
 
+    const handleRateExceeded = (_, res) => {
+        const { status, body } = createResponse("error", {
+            code: "error.api.rate_exceeded",
+            context: {
+                limit: env.rateLimitWindow
+            }
+        });
+        return res.status(status).json(body);
+    };
+
+    const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
+
+    const sessionLimiter = rateLimit({
+        windowMs: env.sessionRateLimitWindow * 1000,
+        limit: env.sessionRateLimit,
+        standardHeaders: 'draft-6',
+        legacyHeaders: false,
+        keyGenerator,
+        store: await createStore('session'),
+        handler: handleRateExceeded
+    });
+
     const apiLimiter = rateLimit({
         windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
         legacyHeaders: false,
-        keyGenerator: req => {
-            if (req.authorized) {
-                return generateHmac(req.header("Authorization"), ipSalt);
-            }
-            return generateHmac(getIP(req), ipSalt);
-        },
-        handler: (req, res) => {
-            const { status, body } = createResponse("error", {
-                code: "error.api.rate_exceeded",
-                context: {
-                    limit: env.rateLimitWindow
-                }
-            });
-            return res.status(status).json(body);
-        }
-    })
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('api'),
+        handler: handleRateExceeded
+    });
 
-    const apiLimiterStream = rateLimit({
+    const apiTunnelLimiter = rateLimit({
         windowMs: env.rateLimitWindow * 1000,
-        max: env.rateLimitMax,
-        standardHeaders: true,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
         legacyHeaders: false,
-        keyGenerator: req => generateHmac(getIP(req), ipSalt),
-        handler: (req, res) => {
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('tunnel'),
+        handler: (_, res) => {
             return res.sendStatus(429)
         }
-    })
+    });
 
     app.set('trust proxy', ['loopback', 'uniquelocal']);
 
@@ -103,9 +118,6 @@ export const runAPI = (express, app, __dirname) => {
         ...corsConfig,
     }));
 
-    app.post('/', apiLimiter);
-    app.use('/tunnel', apiLimiterStream);
-
     app.post('/', (req, res, next) => {
         if (!acceptRegex.test(req.header('Accept'))) {
             return fail(res, "error.api.header.accept");
@@ -117,7 +129,34 @@ export const runAPI = (express, app, __dirname) => {
     });
 
     app.post('/', (req, res, next) => {
-        if (!env.sessionEnabled) {
+        if (!env.apiKeyURL) {
+            return next();
+        }
+
+        const { success, error } = APIKeys.validateAuthorization(req);
+        if (!success) {
+            // We call next() here if either if:
+            // a) we have user sessions enabled, meaning the request
+            //    will still need a Bearer token to not be rejected, or
+            // b) we do not require the user to be authenticated, and
+            //    so they can just make the request with the regular
+            //    rate limit configuration;
+            // otherwise, we reject the request.
+            if (
+                (env.sessionEnabled || !env.authRequired)
+                && ['missing', 'not_api_key'].includes(error)
+            ) {
+                return next();
+            }
+
+            return fail(res, `error.api.auth.key.${error}`);
+        }
+
+        return next();
+    });
+
+    app.post('/', (req, res, next) => {
+        if (!env.sessionEnabled || req.rateLimitKey) {
             return next();
         }
 
@@ -127,26 +166,29 @@ export const runAPI = (express, app, __dirname) => {
                 return fail(res, "error.api.auth.jwt.missing");
             }
 
-            if (!authorization.startsWith("Bearer ") || authorization.length > 256) {
+            if (authorization.length >= 256) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            const verifyJwt = jwt.verify(
-                authorization.split("Bearer ", 2)[1]
-            );
+            const [ type, token, ...rest ] = authorization.split(" ");
+            if (!token || type.toLowerCase() !== 'bearer' || rest.length) {
+                return fail(res, "error.api.auth.jwt.invalid");
+            }
 
-            if (!verifyJwt) {
+            if (!jwt.verify(token, getIP(req, 32))) {
                 return fail(res, "error.api.auth.jwt.invalid");
             }
 
-            req.authorized = true;
+            req.rateLimitKey = hashHmac(token, 'rate');
         } catch {
             return fail(res, "error.api.generic");
         }
         next();
     });
 
+    app.post('/', apiLimiter);
     app.use('/', express.json({ limit: 1024 }));
+
     app.use('/', (err, _, res, next) => {
         if (err) {
             const { status, body } = createResponse("error", {
@@ -158,7 +200,7 @@ export const runAPI = (express, app, __dirname) => {
         next();
     });
 
-    app.post("/session", async (req, res) => {
+    app.post("/session", sessionLimiter, async (req, res) => {
         if (!env.sessionEnabled) {
             return fail(res, "error.api.auth.not_configured")
         }
@@ -179,7 +221,7 @@ export const runAPI = (express, app, __dirname) => {
         }
 
         try {
-            res.json(jwt.generate());
+            res.json(jwt.generate(getIP(req, 32)));
         } catch {
             return fail(res, "error.api.generic");
         }
@@ -187,16 +229,11 @@ export const runAPI = (express, app, __dirname) => {
 
     app.post('/', async (req, res) => {
         const request = req.body;
-        const lang = languageCode(req);
 
         if (!request.url) {
             return fail(res, "error.api.link.missing");
         }
 
-        if (request.youtubeDubBrowserLang) {
-            request.youtubeDubLang = lang;
-        }
-
         const { success, data: normalizedRequest } = await normalizeRequest(request);
         if (!success) {
             return fail(res, "error.api.invalid_body");
@@ -228,7 +265,7 @@ export const runAPI = (express, app, __dirname) => {
         }
     })
 
-    app.get('/tunnel', (req, res) => {
+    app.get('/tunnel', apiTunnelLimiter, async (req, res) => {
         const id = String(req.query.id);
         const exp = String(req.query.exp);
         const sig = String(req.query.sig);
@@ -247,7 +284,7 @@ export const runAPI = (express, app, __dirname) => {
             return res.status(200).end();
         }
 
-        const streamInfo = verifyStream(id, sig, exp, sec, iv);
+        const streamInfo = await verifyStream(id, sig, exp, sec, iv);
         if (!streamInfo?.service) {
             return res.status(streamInfo.status).end();
         }
@@ -259,7 +296,7 @@ export const runAPI = (express, app, __dirname) => {
         return stream(res, streamInfo);
     })
 
-    app.get('/itunnel', (req, res) => {
+    const itunnelHandler = (req, res) => {
         if (!req.ip.endsWith('127.0.0.1')) {
             return res.sendStatus(403);
         }
@@ -278,8 +315,10 @@ export const runAPI = (express, app, __dirname) => {
             ...Object.entries(req.headers)
         ]);
 
-        return stream(res, { type: 'internal', ...streamInfo });
-    })
+        return stream(res, { type: 'internal', data: streamInfo });
+    };
+
+    app.get('/itunnel', itunnelHandler);
 
     app.get('/', (_, res) => {
         res.type('json');
@@ -310,20 +349,52 @@ export const runAPI = (express, app, __dirname) => {
         setGlobalDispatcher(new ProxyAgent(env.externalProxy))
     }
 
-    app.listen(env.apiPort, env.listenAddress, () => {
-        console.log(`\n` +
-            Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +
-
-            "~~~~~~\n" +
-            Bright("version: ") + version + "\n" +
-            Bright("commit: ") + git.commit + "\n" +
-            Bright("branch: ") + git.branch + "\n" +
-            Bright("remote: ") + git.remote + "\n" +
-            Bright("start time: ") + startTime.toUTCString() + "\n" +
-            "~~~~~~\n" +
-
-            Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
-            Bright("port: ") + env.apiPort + "\n"
-        )
-    })
+    http.createServer(app).listen({
+        port: env.apiPort,
+        host: env.listenAddress,
+        reusePort: env.instanceCount > 1 || undefined
+    }, () => {
+        if (isPrimary) {
+            console.log(`\n` +
+                Bright(Cyan("cobalt ")) + Bright("API ^ω^") + "\n" +
+
+                "~~~~~~\n" +
+                Bright("version: ") + version + "\n" +
+                Bright("commit: ") + git.commit + "\n" +
+                Bright("branch: ") + git.branch + "\n" +
+                Bright("remote: ") + git.remote + "\n" +
+                Bright("start time: ") + startTime.toUTCString() + "\n" +
+                "~~~~~~\n" +
+
+                Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
+                Bright("port: ") + env.apiPort + "\n"
+            );
+        }
+
+        if (env.apiKeyURL) {
+            APIKeys.setup(env.apiKeyURL);
+        }
+
+        if (env.cookiePath) {
+            Cookies.setup(env.cookiePath);
+        }
+
+        if (env.ytSessionServer) {
+            YouTubeSession.setup();
+        }
+    });
+
+    if (isCluster) {
+        const istreamer = express();
+        istreamer.get('/itunnel', itunnelHandler);
+        const server = istreamer.listen({
+            port: 0,
+            host: '127.0.0.1',
+            exclusive: true
+        }, () => {
+            const { port } = server.address();
+            console.log(`${Green('[✓]')} cobalt sub-instance running on 127.0.0.1:${port}`);
+            setTunnelPort(port);
+        });
+    }
 }

api/src/misc/cluster.js

@@ -0,0 +1,71 @@
+import cluster from "node:cluster";
+import net from "node:net";
+import { syncSecrets } from "../security/secrets.js";
+import { env, isCluster } from "../config.js";
+
+export { isPrimary, isWorker } from "node:cluster";
+
+export const supportsReusePort = async () => {
+    try {
+        await new Promise((resolve, reject) => {
+            const server = net.createServer().listen({ port: 0, reusePort: true });
+            server.on('listening', () => server.close(resolve));
+            server.on('error', (err) => (server.close(), reject(err)));
+        });
+
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+export const initCluster = async () => {
+    if (cluster.isPrimary) {
+        for (let i = 1; i < env.instanceCount; ++i) {
+            cluster.fork();
+        }
+    }
+
+    await syncSecrets();
+}
+
+export const broadcast = (message) => {
+    if (!isCluster || !cluster.isPrimary || !cluster.workers) {
+        return;
+    }
+
+    for (const worker of Object.values(cluster.workers)) {
+        worker.send(message);
+    }
+}
+
+export const send = (message) => {
+    if (!isCluster) {
+        return;
+    }
+
+    if (cluster.isPrimary) {
+        return broadcast(message);
+    } else {
+        return process.send(message);
+    }
+}
+
+export const waitFor = (key) => {
+    return new Promise(resolve => {
+        const listener = (message) => {
+            if (key in message) {
+                process.off('message', listener);
+                return resolve(message);
+            }
+        }
+
+        process.on('message', listener);
+    });
+}
+
+export const mainOnMessage = (cb) => {
+    for (const worker of Object.values(cluster.workers)) {
+        worker.on('message', cb);
+    }
+}

api/src/misc/console-text.js

@@ -1,16 +1,36 @@
-function t(color, tt) {
-    return color + tt + "\x1b[0m"
+const ANSI = {
+    RESET: "\x1b[0m",
+    BRIGHT: "\x1b[1m",
+    RED: "\x1b[31m",
+    GREEN: "\x1b[32m",
+    CYAN: "\x1b[36m",
+    YELLOW: "\x1b[93m"
 }
 
-export function Bright(tt) {
-    return t("\x1b[1m", tt)
+function wrap(color, text) {
+    if (!ANSI[color.toUpperCase()]) {
+        throw "invalid color";
+    }
+
+    return ANSI[color.toUpperCase()] + text + ANSI.RESET;
+}
+
+export function Bright(text) {
+    return wrap('bright', text);
+}
+
+export function Red(text) {
+    return wrap('red', text);
 }
-export function Red(tt) {
-    return t("\x1b[31m", tt)
+
+export function Green(text) {
+    return wrap('green', text);
 }
-export function Green(tt) {
-    return t("\x1b[32m", tt)
+
+export function Cyan(text) {
+    return wrap('cyan', text);
 }
-export function Cyan(tt) {
-    return t("\x1b[36m", tt)
+
+export function Yellow(text) {
+    return wrap('yellow', text);
 }

api/src/misc/crypto.js

@@ -1,15 +1,7 @@
-import { createHmac, createCipheriv, createDecipheriv, randomBytes } from "crypto";
+import { createCipheriv, createDecipheriv } from "crypto";
 
 const algorithm = "aes256";
 
-export function generateSalt() {
-    return randomBytes(64).toString('hex');
-}
-
-export function generateHmac(str, salt) {
-    return createHmac("sha256", salt).update(str).digest("base64url");
-}
-
 export function encryptStream(plaintext, iv, secret) {
     const buff = Buffer.from(JSON.stringify(plaintext));
     const key = Buffer.from(secret, "base64url");

api/src/misc/run-test.js

@@ -23,6 +23,15 @@ export async function runTest(url, params, expect) {
     if (expect.status !== result.body.status) {
         const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
         error.push(`status mismatch: ${detail}`);
+
+        if (result.body.status === 'error') {
+            error.push(`error code: ${result.body?.error?.code}`);
+        }
+    }
+
+    if (expect.errorCode && expect.errorCode !== result.body?.error?.code) {
+        const detail = `${expect.errorCode} (expected) != ${result.body.error.code} (actual)`
+        error.push(`error mismatch: ${detail}`);
     }
 
     if (expect.code !== result.status) {
@@ -41,4 +50,4 @@ export async function runTest(url, params, expect) {
     if (result.body.status === 'tunnel') {
         // TODO: stream testing
     }
-}
+}

api/src/misc/utils.js

@@ -1,55 +1,27 @@
-const forbiddenCharsString = ['}', '{', '%', '>', '<', '^', ';', '`', '$', '"', "@", '='];
-
-export function metadataManager(obj) {
-    const keys = Object.keys(obj);
-    const tags = [
-        "album",
-        "copyright",
-        "title",
-        "artist",
-        "track",
-        "date"
-    ]
-    let commands = []
-
-    for (const i in keys) {
-        if (tags.includes(keys[i]))
-            commands.push('-metadata', `${keys[i]}=${obj[keys[i]]}`)
+import { request } from 'undici';
+const redirectStatuses = new Set([301, 302, 303, 307, 308]);
+
+export async function getRedirectingURL(url, dispatcher, headers) {
+    const params = {
+        dispatcher,
+        method: 'HEAD',
+        headers,
+        redirect: 'manual'
+    };
+
+    let location = await request(url, params).then(r => {
+        if (redirectStatuses.has(r.statusCode) && r.headers['location']) {
+            return r.headers['location'];
         }
-    return commands;
-}
-
-export function cleanString(string) {
-    for (const i in forbiddenCharsString) {
-        string = string.replaceAll("/", "_")
-                       .replaceAll(forbiddenCharsString[i], '')
-    }
-    return string;
-}
-export function verifyLanguageCode(code) {
-    const langCode = String(code.slice(0, 2).toLowerCase());
-    if (RegExp(/[a-z]{2}/).test(code)) {
-        return langCode
-    }
-    return "en"
-}
-export function languageCode(req) {
-    if (req.header('Accept-Language')) {
-        return verifyLanguageCode(req.header('Accept-Language'))
-    }
-    return "en"
-}
-export function cleanHTML(html) {
-    let clean = html.replace(/ {4}/g, '');
-    clean = clean.replace(/\n/g, '');
-    return clean
-}
+    }).catch(() => null);
 
-export function getRedirectingURL(url) {
-    return fetch(url, { redirect: 'manual' }).then((r) => {
-        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
+    location ??= await fetch(url, params).then(r => {
+        if (redirectStatuses.has(r.status) && r.headers.has('location')) {
             return r.headers.get('location');
+        }
     }).catch(() => null);
+
+    return location;
 }
 
 export function merge(a, b) {
@@ -76,3 +48,7 @@ export function splitFilenameExtension(filename) {
         return [ parts.join('.'), ext ]
     }
 }
+
+export function zip(a, b) {
+    return a.map((value, i) => [ value, b[i] ]);
+}

api/src/processing/cookie/cookie.js

@@ -4,16 +4,24 @@ export default class Cookie {
     constructor(input) {
         assert(typeof input === 'object');
         this._values = {};
-        this.set(input)
+
+        for (const [ k, v ] of Object.entries(input))
+            this.set(k, v);
     }
-    set(values) {
-        Object.entries(values).forEach(
-            ([ key, value ]) => this._values[key] = value
-        )
+
+    set(key, value) {
+        const old = this._values[key];
+        if (old === value)
+            return false;
+
+        this._values[key] = value;
+        return true;
     }
+
     unset(keys) {
         for (const key of keys) delete this._values[key]
     }
+
     static fromString(str) {
         const obj = {};
 
@@ -25,12 +33,15 @@ export default class Cookie {
 
         return new Cookie(obj)
     }
+
     toString() {
         return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ')
     }
+
     toJSON() {
         return this.toString()
     }
+
     values() {
         return Object.freeze({ ...this._values })
     }

api/src/processing/cookie/manager.js

@@ -1,50 +1,144 @@
 import Cookie from './cookie.js';
+
 import { readFile, writeFile } from 'fs/promises';
+import { Red, Green, Yellow } from '../../misc/console-text.js';
 import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
-import { env } from '../../config.js';
+import * as cluster from '../../misc/cluster.js';
+import { isCluster } from '../../config.js';
 
-const WRITE_INTERVAL = 60000,
-      cookiePath = env.cookiePath,
-      COUNTER = Symbol('counter');
+const WRITE_INTERVAL = 60000;
+const VALID_SERVICES = new Set([
+    'instagram',
+    'instagram_bearer',
+    'reddit',
+    'twitter',
+    'youtube',
+]);
 
+const invalidCookies = {};
 let cookies = {}, dirty = false, intervalId;
 
-const setup = async () => {
-    try {
-        if (!cookiePath) return;
+function writeChanges(cookiePath) {
+    if (!dirty) return;
+    dirty = false;
+
+    const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4);
+    writeFile(cookiePath, cookieData).catch((e) => {
+        console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`);
+        console.warn(e);
+        clearInterval(intervalId);
+        intervalId = null;
+    })
+}
 
+const setupMain = async (cookiePath) => {
+    try {
         cookies = await readFile(cookiePath, 'utf8');
         cookies = JSON.parse(cookies);
-        intervalId = setInterval(writeChanges, WRITE_INTERVAL)
-    } catch { /* no cookies for you */ }
+        for (const serviceName in cookies) {
+            if (!VALID_SERVICES.has(serviceName)) {
+                console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`);
+            } else if (!Array.isArray(cookies[serviceName])) {
+                console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`);
+            } else if (cookies[serviceName].some(c => typeof c !== 'string')) {
+                console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`);
+            } else continue;
+
+            invalidCookies[serviceName] = cookies[serviceName];
+            delete cookies[serviceName];
+        }
+
+        if (!intervalId) {
+            intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL);
+        }
+
+        cluster.broadcast({ cookies });
+
+        console.log(`${Green('[✓]')} cookies loaded successfully!`);
+    } catch (e) {
+        console.error(`${Yellow('[!]')} failed to load cookies.`);
+        console.error('error:', e);
+    }
 }
 
-setup();
+const setupWorker = async () => {
+    cookies = (await cluster.waitFor('cookies')).cookies;
+}
+
+export const loadFromFile = async (path) => {
+    if (cluster.isPrimary) {
+        await setupMain(path);
+    } else if (cluster.isWorker) {
+        await setupWorker();
+    }
 
-function writeChanges() {
-    if (!dirty) return;
     dirty = false;
+}
 
-    writeFile(cookiePath, JSON.stringify(cookies, null, 4)).catch(() => {
-        clearInterval(intervalId)
-    })
+export const setup = async (path) => {
+    await loadFromFile(path);
+
+    if (isCluster) {
+        const messageHandler = (message) => {
+            if ('cookieUpdate' in message) {
+                const { cookieUpdate } = message;
+
+                if (cluster.isPrimary) {
+                    dirty = true;
+                    cluster.broadcast({ cookieUpdate });
+                }
+
+                const { service, idx, cookie } = cookieUpdate;
+                cookies[service][idx] = cookie;
+            }
+        }
+
+        if (cluster.isPrimary) {
+            cluster.mainOnMessage(messageHandler);
+        } else {
+            process.on('message', messageHandler);
+        }
+    }
 }
 
 export function getCookie(service) {
+    if (!VALID_SERVICES.has(service)) {
+        console.error(
+            `${Red('[!]')} ${service} not in allowed services list for cookies.`
+            + ' if adding a new cookie type, include it there.'
+        );
+        return;
+    }
+
     if (!cookies[service] || !cookies[service].length) return;
 
-    let n;
-    if (cookies[service][COUNTER] === undefined) {
-        n = cookies[service][COUNTER] = 0
-    } else {
-        ++cookies[service][COUNTER]
-        n = (cookies[service][COUNTER] %= cookies[service].length)
+    const idx = Math.floor(Math.random() * cookies[service].length);
+
+    const cookie = cookies[service][idx];
+    if (typeof cookie === 'string') {
+        cookies[service][idx] = Cookie.fromString(cookie);
     }
 
-    const cookie = cookies[service][n];
-    if (typeof cookie === 'string') cookies[service][n] = Cookie.fromString(cookie);
+    cookies[service][idx].meta = { service, idx };
+    return cookies[service][idx];
+}
 
-    return cookies[service][n]
+export function updateCookieValues(cookie, values) {
+    let changed = false;
+
+    for (const [ key, value ] of Object.entries(values)) {
+        changed = cookie.set(key, value) || changed;
+    }
+
+    if (changed && cookie.meta) {
+        dirty = true;
+        if (isCluster) {
+            const message = { cookieUpdate: { ...cookie.meta, cookie } };
+            cluster.send(message);
+        }
+    }
+
+    return changed;
 }
 
 export function updateCookie(cookie, headers) {
@@ -57,10 +151,6 @@ export function updateCookie(cookie, headers) {
 
     cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
     parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value);
-    updateCookieValues(cookie, values);
-}
 
-export function updateCookieValues(cookie, values) {
-    cookie.set(values);
-    if (Object.keys(values).length) dirty = true
+    updateCookieValues(cookie, values);
 }

api/src/processing/create-filename.js

@@ -1,3 +1,13 @@
+const illegalCharacters = ['}', '{', '%', '>', '<', '^', ';', ':', '`', '$', '"', "@", '=', '?', '|', '*'];
+
+const sanitizeString = (string) => {
+    for (const i in illegalCharacters) {
+        string = string.replaceAll("/", "_").replaceAll("\\", "_")
+                       .replaceAll(illegalCharacters[i], '')
+    }
+    return string;
+}
+
 export default (f, style, isAudioOnly, isAudioMuted) => {
     let filename = '';
 
@@ -5,7 +15,11 @@ export default (f, style, isAudioOnly, isAudioMuted) => {
     let classicTags = [...infoBase];
     let basicTags = [];
 
-    const title = `${f.title} - ${f.author}`;
+    let title = sanitizeString(f.title);
+
+    if (f.author) {
+        title += ` - ${sanitizeString(f.author)}`;
+    }
 
     if (f.resolution) {
         classicTags.push(f.resolution);

api/src/processing/helpers/youtube-session.js

@@ -0,0 +1,81 @@
+import * as cluster from "../../misc/cluster.js";
+
+import { Agent } from "undici";
+import { env } from "../../config.js";
+import { Green, Yellow } from "../../misc/console-text.js";
+
+const defaultAgent = new Agent();
+
+let session;
+
+const validateSession = (sessionResponse) => {
+    if (!sessionResponse.potoken) {
+        throw "no poToken in session response";
+    }
+
+    if (!sessionResponse.visitor_data) {
+        throw "no visitor_data in session response";
+    }
+
+    if (!sessionResponse.updated) {
+        throw "no last update timestamp in session response";
+    }
+
+    // https://github.com/iv-org/youtube-trusted-session-generator/blob/c2dfe3f/potoken_generator/main.py#L25
+    if (sessionResponse.potoken.length < 160) {
+        console.error(`${Yellow('[!]')} poToken is too short and might not work (${new Date().toISOString()})`);
+    }
+}
+
+const updateSession = (newSession) => {
+    session = newSession;
+}
+
+const loadSession = async () => {
+    const sessionServerUrl = new URL(env.ytSessionServer);
+    sessionServerUrl.pathname = "/token";
+
+    const newSession = await fetch(
+        sessionServerUrl,
+        { dispatcher: defaultAgent }
+    ).then(a => a.json());
+
+    validateSession(newSession);
+
+    if (!session || session.updated < newSession?.updated) {
+        cluster.broadcast({ youtube_session: newSession });
+        updateSession(newSession);
+    }
+}
+
+const wrapLoad = (initial = false) => {
+    loadSession()
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} poToken & visitor_data loaded successfully!`);
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading poToken & visitor_data at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+export const getYouTubeSession = () => {
+    return session;
+}
+
+export const setup = () => {
+    if (cluster.isPrimary) {
+        wrapLoad(true);
+        if (env.ytSessionReloadInterval > 0) {
+            setInterval(wrapLoad, env.ytSessionReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('youtube_session' in message) {
+                updateSession(message.youtube_session);
+            }
+        });
+    }
+}

api/src/processing/match-action.js

@@ -9,13 +9,14 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
     let action,
         responseType = "tunnel",
         defaultParams = {
-            u: r.urls,
+            url: r.urls,
             headers: r.headers,
             service: host,
             filename: r.filenameAttributes ?
                     createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
             fileMetadata: !disableMetadata ? r.fileMetadata : false,
-            requestIP
+            requestIP,
+            originalRequest: r.originalRequest
         },
         params = {};
 
@@ -24,7 +25,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
     else if (r.isGif && twitterGif) action = "gif";
     else if (isAudioOnly) action = "audio";
     else if (isAudioMuted) action = "muteVideo";
-    else if (r.isM3U8) action = "m3u8";
+    else if (r.isHLS) action = "hls";
     else action = "video";
 
     if (action === "picker" || action === "audio") {
@@ -47,27 +48,29 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
             });
 
         case "photo":
-            responseType = "redirect";
+            params = { type: "proxy" };
             break;
 
         case "gif":
             params = { type: "gif" };
             break;
 
-        case "m3u8":
+        case "hls":
             params = {
-                type: Array.isArray(r.urls) ? "merge" : "remux"
+                type: Array.isArray(r.urls) ? "merge" : "remux",
+                isHLS: true,
             }
             break;
 
         case "muteVideo":
             let muteType = "mute";
-            if (Array.isArray(r.urls) && !r.isM3U8) {
+            if (Array.isArray(r.urls) && !r.isHLS) {
                 muteType = "proxy";
             }
             params = {
                 type: muteType,
-                u: Array.isArray(r.urls) ? r.urls[0] : r.urls
+                url: Array.isArray(r.urls) ? r.urls[0] : r.urls,
+                isHLS: r.isHLS
             }
             if (host === "reddit" && r.typeId === "redirect") {
                 responseType = "redirect";
@@ -81,6 +84,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 case "twitter":
                 case "snapchat":
                 case "bsky":
+                case "xiaohongshu":
                     params = { picker: r.picker };
                     break;
 
@@ -92,14 +96,15 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                     }
                     params = {
                         picker: r.picker,
-                        u: createStream({
+                        url: createStream({
                             service: "tiktok",
                             type: audioStreamType,
-                            u: r.urls,
+                            url: r.urls,
                             headers: r.headers,
-                            filename: r.audioFilename,
+                            filename: `${r.audioFilename}.${audioFormat}`,
                             isAudioOnly: true,
                             audioFormat,
+                            audioBitrate
                         })
                     }
                     break;
@@ -137,13 +142,14 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                     }
                     break;
 
+                case "ok":
                 case "vk":
                 case "tiktok":
+                case "xiaohongshu":
                     params = { type: "proxy" };
                     break;
 
                 case "facebook":
-                case "vine":
                 case "instagram":
                 case "tumblr":
                 case "pinterest":
@@ -159,7 +165,7 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
         case "audio":
             if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
                 return createResponse("error", {
-                    code: "error.api.fetch.empty"
+                    code: "error.api.service.audio_not_supported"
                 })
             }
 
@@ -183,18 +189,20 @@ export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disab
                 }
             }
 
-            if (r.isM3U8 || host === "vimeo") {
+            if (r.isHLS || host === "vimeo") {
                 copy = false;
                 processType = "audio";
             }
 
             params = {
                 type: processType,
-                u: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+                url: Array.isArray(r.urls) ? r.urls[1] : r.urls,
 
                 audioBitrate,
                 audioCopy: copy,
                 audioFormat,
+
+                isHLS: r.isHLS,
             }
             break;
     }

api/src/processing/match.js

@@ -19,7 +19,6 @@ import tumblr from "./services/tumblr.js";
 import vimeo from "./services/vimeo.js";
 import soundcloud from "./services/soundcloud.js";
 import instagram from "./services/instagram.js";
-import vine from "./services/vine.js";
 import pinterest from "./services/pinterest.js";
 import streamable from "./services/streamable.js";
 import twitch from "./services/twitch.js";
@@ -29,6 +28,7 @@ import snapchat from "./services/snapchat.js";
 import loom from "./services/loom.js";
 import facebook from "./services/facebook.js";
 import bluesky from "./services/bluesky.js";
+import xiaohongshu from "./services/xiaohongshu.js";
 
 let freebind;
 
@@ -78,8 +78,9 @@ export default async function({ host, patternMatch, params }) {
 
             case "vk":
                 r = await vk({
-                    userId: patternMatch.userId,
+                    ownerId: patternMatch.ownerId,
                     videoId: patternMatch.videoId,
+                    accessKey: patternMatch.accessKey,
                     quality: params.videoQuality
                 });
                 break;
@@ -97,17 +98,18 @@ export default async function({ host, patternMatch, params }) {
 
             case "youtube":
                 let fetchInfo = {
+                    dispatcher,
                     id: patternMatch.id.slice(0, 11),
                     quality: params.videoQuality,
                     format: params.youtubeVideoCodec,
                     isAudioOnly,
                     isAudioMuted,
                     dubLang: params.youtubeDubLang,
-                    dispatcher
+                    youtubeHLS: params.youtubeHLS,
                 }
 
                 if (url.hostname === "music.youtube.com" || isAudioOnly) {
-                    fetchInfo.quality = "max";
+                    fetchInfo.quality = "1080";
                     fetchInfo.format = "vp9";
                     fetchInfo.isAudioOnly = true;
                     fetchInfo.isAudioMuted = false;
@@ -118,16 +120,15 @@ export default async function({ host, patternMatch, params }) {
 
             case "reddit":
                 r = await reddit({
-                    sub: patternMatch.sub,
-                    id: patternMatch.id,
-                    user: patternMatch.user
+                    ...patternMatch,
+                    dispatcher,
                 });
                 break;
 
             case "tiktok":
                 r = await tiktok({
                     postId: patternMatch.postId,
-                    id: patternMatch.id,
+                    shortLink: patternMatch.shortLink,
                     fullAudio: params.tiktokFullAudio,
                     isAudioOnly,
                     h265: params.tiktokH265,
@@ -174,12 +175,6 @@ export default async function({ host, patternMatch, params }) {
                 })
                 break;
 
-            case "vine":
-                r = await vine({
-                    id: patternMatch.id
-                });
-                break;
-
             case "pinterest":
                 r = await pinterest({
                     id: patternMatch.id,
@@ -232,14 +227,25 @@ export default async function({ host, patternMatch, params }) {
 
             case "facebook":
                 r = await facebook({
-                    ...patternMatch
+                    ...patternMatch,
+                    dispatcher
                 });
                 break;
 
             case "bsky":
                 r = await bluesky({
                     ...patternMatch,
-                    alwaysProxy: params.alwaysProxy
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                });
+                break;
+
+            case "xiaohongshu":
+                r = await xiaohongshu({
+                    ...patternMatch,
+                    h265: params.tiktokH265,
+                    isAudioOnly,
+                    dispatcher,
                 });
                 break;
 

api/src/processing/request.js

@@ -37,7 +37,7 @@ export function createResponse(responseType, responseData) {
 
             case "redirect":
                 response = {
-                    url: responseData?.u,
+                    url: responseData?.url,
                     filename: responseData?.filename
                 }
                 break;
@@ -52,7 +52,7 @@ export function createResponse(responseType, responseData) {
             case "picker":
                 response = {
                     picker: responseData?.picker,
-                    audio: responseData?.u,
+                    audio: responseData?.url,
                     audioFilename: responseData?.filename
                 }
                 break;
@@ -82,14 +82,13 @@ export function normalizeRequest(request) {
     ));
 }
 
-export function getIP(req) {
+export function getIP(req, prefix = 56) {
     const strippedIP = req.ip.replace(/^::ffff:/, '');
     const ip = ipaddr.parse(strippedIP);
     if (ip.kind() === 'ipv4') {
         return strippedIP;
     }
 
-    const prefix = 56;
     const v6Bytes = ip.toByteArray();
           v6Bytes.fill(0, prefix / 8);
 

api/src/processing/schema.js

@@ -1,7 +1,5 @@
 import { z } from "zod";
-
 import { normalizeURL } from "./url.js";
-import { verifyLanguageCode } from "../misc/utils.js";
 
 export const apiSchema = z.object({
     url: z.string()
@@ -33,15 +31,21 @@ export const apiSchema = z.object({
     ).default("1080"),
 
     youtubeDubLang: z.string()
-                     .length(2)
-                     .transform(verifyLanguageCode)
+                     .min(2)
+                     .max(8)
+                     .regex(/^[0-9a-zA-Z\-]+$/)
                      .optional(),
 
+    // TODO: remove this variable as it's no longer used
+    // and is kept for schema compatibility reasons
+    youtubeDubBrowserLang: z.boolean().default(false),
+
     alwaysProxy: z.boolean().default(false),
     disableMetadata: z.boolean().default(false),
     tiktokFullAudio: z.boolean().default(false),
     tiktokH265: z.boolean().default(false),
     twitterGif: z.boolean().default(true),
-    youtubeDubBrowserLang: z.boolean().default(false),
+
+    youtubeHLS: z.boolean().default(false),
 })
 .strict();

api/src/processing/service-config.js

@@ -1,7 +1,7 @@
 import UrlPattern from "url-pattern";
 
 export const audioIgnore = ["vk", "ok", "loom"];
-export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky"];
+export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky", "youtube"];
 
 export const services = {
     bilibili: {
@@ -30,23 +30,35 @@ export const services = {
             "reel/:id",
             "share/:shareType/:id"
         ],
-        subdomains: ["web"],
+        subdomains: ["web", "m"],
         altDomains: ["fb.watch"],
     },
     instagram: {
         patterns: [
-            "reels/:postId",
-            ":username/reel/:postId",
-            "reel/:postId",
             "p/:postId",
-            ":username/p/:postId",
             "tv/:postId",
-            "stories/:username/:storyId"
+            "reel/:postId",
+            "reels/:postId",
+            "stories/:username/:storyId",
+
+            /*
+                share & username links use the same url pattern,
+                so we test the share pattern first, cuz id type is different.
+                however, if someone has the "share" username and the user
+                somehow gets a link of this ancient style, it's joever.
+            */
+
+            "share/:shareId",
+            "share/p/:shareId",
+            "share/reel/:shareId",
+
+            ":username/p/:postId",
+            ":username/reel/:postId",
         ],
         altDomains: ["ddinstagram.com"],
     },
     loom: {
-        patterns: ["share/:id"],
+        patterns: ["share/:id", "embed/:id"],
     },
     ok: {
         patterns: [
@@ -64,8 +76,23 @@ export const services = {
     },
     reddit: {
         patterns: [
+            "comments/:id",
+
+            "r/:sub/comments/:id",
             "r/:sub/comments/:id/:title",
-            "user/:user/comments/:id/:title"
+            "r/:sub/comments/:id/comment/:commentId",
+
+            "user/:user/comments/:id",
+            "user/:user/comments/:id/:title",
+            "user/:user/comments/:id/comment/:commentId",
+
+            "r/u_:user/comments/:id",
+            "r/u_:user/comments/:id/:title",
+            "r/u_:user/comments/:id/comment/:commentId",
+
+            "r/:sub/s/:shareId",
+
+            "video/:shortId",
         ],
         subdomains: "*",
     },
@@ -111,12 +138,13 @@ export const services = {
     tiktok: {
         patterns: [
             ":user/video/:postId",
-            ":id",
-            "t/:id",
+            "i18n/share/video/:postId",
+            ":shortLink",
+            "t/:shortLink",
             ":user/photo/:postId",
-            "v/:id.html"
+            "v/:postId.html"
         ],
-        subdomains: ["vt", "vm", "m"],
+        subdomains: ["vt", "vm", "m", "t"],
     },
     tumblr: {
         patterns: [
@@ -137,15 +165,12 @@ export const services = {
             ":user/status/:id/video/:index",
             ":user/status/:id/photo/:index",
             ":user/status/:id/mediaviewer",
-            ":user/status/:id/mediaViewer"
+            ":user/status/:id/mediaViewer",
+            "i/bookmarks?post_id=:id"
         ],
         subdomains: ["mobile"],
         altDomains: ["x.com", "vxtwitter.com", "fixvx.com"],
     },
-    vine: {
-        patterns: ["v/:id"],
-        tld: "co",
-    },
     vimeo: {
         patterns: [
             ":id",
@@ -157,11 +182,25 @@ export const services = {
     },
     vk: {
         patterns: [
-            "video:userId_:videoId",
-            "clip:userId_:videoId",
-            "clips:duplicate?z=clip:userId_:videoId"
+            "video:ownerId_:videoId",
+            "clip:ownerId_:videoId",
+            "clips:duplicate?z=clip:ownerId_:videoId",
+            "videos:duplicate?z=video:ownerId_:videoId",
+            "video:ownerId_:videoId_:accessKey",
+            "clip:ownerId_:videoId_:accessKey",
+            "clips:duplicate?z=clip:ownerId_:videoId_:accessKey",
+            "videos:duplicate?z=video:ownerId_:videoId_:accessKey"
         ],
         subdomains: ["m"],
+        altDomains: ["vkvideo.ru", "vk.ru"],
+    },
+    xiaohongshu: {
+        patterns: [
+            "explore/:id?xsec_token=:token",
+            "discovery/item/:id?xsec_token=:token",
+            "a/:shareId"
+        ],
+        altDomains: ["xhslink.com"],
     },
     youtube: {
         patterns: [

api/src/processing/service-patterns.js

@@ -6,7 +6,8 @@ export const testers = {
     "dailymotion": pattern => pattern.id?.length <= 32,
 
     "instagram": pattern =>
-        pattern.postId?.length <= 12
+        pattern.postId?.length <= 48
+        || pattern.shareId?.length <= 16
         || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
 
     "loom": pattern =>
@@ -19,8 +20,11 @@ export const testers = {
         pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
 
     "reddit": pattern =>
-        (pattern.sub?.length <= 22 && pattern.id?.length <= 10)
-        || (pattern.user?.length <= 22 && pattern.id?.length <= 10),
+        pattern.id?.length <= 16 && !pattern.sub && !pattern.user
+        || (pattern.sub?.length <= 22 && pattern.id?.length <= 16)
+        || (pattern.user?.length <= 22 && pattern.id?.length <= 16)
+        || (pattern.sub?.length <= 22 && pattern.shareId?.length <= 16)
+        || (pattern.shortId?.length <= 16),
 
     "rutube": pattern =>
         (pattern.id?.length === 32 && pattern.key?.length <= 32) ||
@@ -36,10 +40,10 @@ export const testers = {
         || pattern.shortLink?.length <= 16,
 
     "streamable": pattern =>
-        pattern.id?.length === 6,
+        pattern.id?.length <= 6,
 
     "tiktok": pattern =>
-        pattern.postId?.length <= 21 || pattern.id?.length <= 13,
+        pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13,
 
     "tumblr": pattern =>
         pattern.id?.length < 21
@@ -55,11 +59,9 @@ export const testers = {
         pattern.id?.length <= 11
         && (!pattern.password || pattern.password.length < 16),
 
-    "vine": pattern =>
-        pattern.id?.length <= 12,
-
     "vk": pattern =>
-        pattern.userId?.length <= 10 && pattern.videoId?.length <= 10,
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
 
     "youtube": pattern =>
         pattern.id?.length <= 11,
@@ -73,4 +75,8 @@ export const testers = {
 
     "bsky": pattern =>
         pattern.user?.length <= 128 && pattern.post?.length <= 128,
+
+    "xiaohongshu": pattern =>
+        pattern.id?.length <= 24 && pattern.token?.length <= 64
+        || pattern.shareId?.length <= 12,
 }

api/src/processing/services/bilibili.js

@@ -1,19 +1,8 @@
 import { genericUserAgent, env } from "../../config.js";
+import { resolveRedirectingURL } from "../url.js";
 
 // TO-DO: higher quality downloads (currently requires an account)
 
-function com_resolveShortlink(shortId) {
-    return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
-            .then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
-            .then(url => {
-                if (!url) return;
-                const path = new URL(url).pathname;
-                if (path.startsWith('/video/'))
-                    return path.split('/')[2];
-            })
-            .catch(() => {})
-}
-
 function getBest(content) {
     return content?.filter(v => v.baseUrl || v.url)
                 .map(v => (v.baseUrl = v.baseUrl || v.url, v))
@@ -99,7 +88,8 @@ async function tv_download(id) {
 
 export default async function({ comId, tvId, comShortLink }) {
     if (comShortLink) {
-        comId = await com_resolveShortlink(comShortLink);
+        const patternMatch = await resolveRedirectingURL(`https://b23.tv/${comShortLink}`);
+        comId = patternMatch?.comId;
     }
 
     if (comId) {

api/src/processing/services/bluesky.js

@@ -2,12 +2,19 @@ import HLS from "hls-parser";
 import { cobaltUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 
-const extractVideo = async ({ media, filename }) => {
-    const urlMasterHLS = media?.playlist;
-    if (!urlMasterHLS) return { error: "fetch.empty" };
-    if (!urlMasterHLS.startsWith("https://video.bsky.app/")) return { error: "fetch.empty" };
+const extractVideo = async ({ media, filename, dispatcher }) => {
+    let urlMasterHLS = media?.playlist;
 
-    const masterHLS = await fetch(urlMasterHLS)
+    if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) {
+        return { error: "fetch.empty" };
+    }
+
+    urlMasterHLS = urlMasterHLS.replace(
+        "video.bsky.app/watch/",
+        "video.cdn.bsky.app/hls/"
+    );
+
+    const masterHLS = await fetch(urlMasterHLS, { dispatcher })
         .then(r => {
             if (r.status !== 200) return;
             return r.text();
@@ -26,7 +33,7 @@ const extractVideo = async ({ media, filename }) => {
         urls: videoURL,
         filename: `${filename}.mp4`,
         audioFilename: `${filename}_audio`,
-        isM3U8: true,
+        isHLS: true,
     }
 }
 
@@ -48,7 +55,7 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
         let proxiedImage = createStream({
             service: "bluesky",
             type: "proxy",
-            u: url,
+            url,
             filename: `${filename}_${i + 1}.jpg`,
         });
 
@@ -64,7 +71,25 @@ const extractImages = ({ getPost, filename, alwaysProxy }) => {
     return { picker };
 }
 
-export default async function ({ user, post, alwaysProxy }) {
+const extractGif = ({ url, filename }) => {
+    const gifUrl = new URL(url);
+
+    if (!gifUrl || gifUrl.hostname !== "media.tenor.com") {
+        return { error: "fetch.empty" };
+    }
+
+    // remove downscaling params from gif url
+    // such as "?hh=498&ww=498"
+    gifUrl.search = "";
+
+    return {
+        urls: gifUrl,
+        isPhoto: true,
+        filename: `${filename}.gif`,
+    }
+}
+
+export default async function ({ user, post, alwaysProxy, dispatcher }) {
     const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
     apiEndpoint.searchParams.set(
         "uri",
@@ -73,8 +98,9 @@ export default async function ({ user, post, alwaysProxy }) {
 
     const getPost = await fetch(apiEndpoint, {
         headers: {
-            "user-agent": cobaltUserAgent
-        }
+            "user-agent": cobaltUserAgent,
+        },
+        dispatcher
     }).then(r => r.json()).catch(() => {});
 
     if (!getPost) return { error: "fetch.empty" };
@@ -87,29 +113,44 @@ export default async function ({ user, post, alwaysProxy }) {
             case "InvalidRequest":
                 return { error: "link.unsupported" };
             default:
-                return { error: "fetch.empty" };
+                return { error: "content.post.unavailable" };
         }
     }
 
     const embedType = getPost?.thread?.post?.embed?.$type;
     const filename = `bluesky_${user}_${post}`;
 
-    if (embedType === "app.bsky.embed.video#view") {
-        return extractVideo({
-            media: getPost.thread?.post?.embed,
-            filename,
-        })
-    }
-
-    if (embedType === "app.bsky.embed.recordWithMedia#view") {
-        return extractVideo({
-            media: getPost.thread?.post?.embed?.media,
-            filename,
-        })
-    }
-
-    if (embedType === "app.bsky.embed.images#view") {
-        return extractImages({ getPost, filename, alwaysProxy });
+    switch (embedType) {
+        case "app.bsky.embed.video#view":
+            return extractVideo({
+                media: getPost.thread?.post?.embed,
+                filename,
+            });
+
+        case "app.bsky.embed.images#view":
+            return extractImages({
+                getPost,
+                filename,
+                alwaysProxy
+            });
+
+        case "app.bsky.embed.external#view":
+            return extractGif({
+                url: getPost?.thread?.post?.embed?.external?.uri,
+                filename,
+            });
+
+        case "app.bsky.embed.recordWithMedia#view":
+            if (getPost?.thread?.post?.embed?.media?.$type === "app.bsky.embed.external#view") {
+                return extractGif({
+                    url: getPost?.thread?.post?.embed?.media?.external?.uri,
+                    filename,
+                });
+            }
+            return extractVideo({
+                media: getPost.thread?.post?.embed?.media,
+                filename,
+            });
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/dailymotion.js

@@ -92,7 +92,7 @@ export default async function({ id }) {
 
     return {
         urls: bestQuality.uri,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             service: 'dailymotion',
             id: media.xid,

api/src/processing/services/facebook.js

@@ -8,8 +8,8 @@ const headers = {
     'Sec-Fetch-Site': 'none',
 }
 
-const resolveUrl = (url) => {
-    return fetch(url, { headers })
+const resolveUrl = (url, dispatcher) => {
+    return fetch(url, { headers, dispatcher })
         .then(r => {
             if (r.headers.get('location')) {
                 return decodeURIComponent(r.headers.get('location'));
@@ -23,13 +23,13 @@ const resolveUrl = (url) => {
         .catch(() => false);
 }
 
-export default async function({ id, shareType, shortLink }) {
+export default async function({ id, shareType, shortLink, dispatcher }) {
     let url = `https://web.facebook.com/i/videos/${id}`;
 
     if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
-    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`, dispatcher);
 
-    const html = await fetch(url, { headers })
+    const html = await fetch(url, { headers, dispatcher })
         .then(r => r.text())
         .catch(() => false);
 

api/src/processing/services/instagram.js

@@ -1,3 +1,5 @@
+import { randomBytes } from "node:crypto";
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
 import { getCookie, updateCookie } from "../cookie/manager.js";
@@ -8,6 +10,7 @@ const commonHeaders = {
     "sec-fetch-site": "same-origin",
     "x-ig-app-id": "936619743392459"
 }
+
 const mobileHeaders = {
     "x-ig-app-locale": "en_US",
     "x-ig-device-locale": "en_US",
@@ -19,6 +22,7 @@ const mobileHeaders = {
     "x-fb-server-cluster": "True",
     "content-length": "0",
 }
+
 const embedHeaders = {
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
     "Accept-Language": "en-GB,en;q=0.9",
@@ -33,7 +37,7 @@ const embedHeaders = {
     "Sec-Fetch-Site": "none",
     "Sec-Fetch-User": "?1",
     "Upgrade-Insecure-Requests": "1",
-    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "User-Agent": genericUserAgent,
 }
 
 const cachedDtsg = {
@@ -41,7 +45,17 @@ const cachedDtsg = {
     expiry: 0
 }
 
-export default function(obj) {
+const getNumberFromQuery = (name, data) => {
+    const s = data?.match(new RegExp(name + '=(\\d+)'))?.[1];
+    if (+s) return +s;
+}
+
+const getObjectFromEntries = (name, data) => {
+    const obj = data?.match(new RegExp('\\["' + name + '",.*?,({.*?}),\\d+\\]'))?.[1];
+    return obj && JSON.parse(obj);
+}
+
+export default function instagram(obj) {
     const dispatcher = obj.dispatcher;
 
     async function findDtsgId(cookie) {
@@ -91,6 +105,7 @@ export default function(obj) {
         updateCookie(cookie, data.headers);
         return data.json();
     }
+
     async function getMediaId(id, { cookie, token } = {}) {
         const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
         oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
@@ -119,6 +134,7 @@ export default function(obj) {
 
         return mediaInfo?.items?.[0];
     }
+
     async function requestHTML(id, cookie) {
         const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, {
             headers: {
@@ -136,40 +152,167 @@ export default function(obj) {
 
         return embedData;
     }
+
+    async function getGQLParams(id, cookie) {
+        const req = await fetch(`https://www.instagram.com/p/${id}/`, {
+            headers: {
+                ...embedHeaders,
+                cookie
+            },
+            dispatcher
+        });
+
+        const html = await req.text();
+        const siteData = getObjectFromEntries('SiteData', html);
+        const polarisSiteData = getObjectFromEntries('PolarisSiteData', html);
+        const webConfig = getObjectFromEntries('DGWWebConfig', html);
+        const pushInfo = getObjectFromEntries('InstagramWebPushInfo', html);
+        const lsd = getObjectFromEntries('LSD', html)?.token || randomBytes(8).toString('base64url');
+        const csrf = getObjectFromEntries('InstagramSecurityConfig', html)?.csrf_token;
+
+        const anon_cookie = [
+            csrf && "csrftoken=" + csrf,
+            polarisSiteData?.device_id && "ig_did=" + polarisSiteData?.device_id,
+            "wd=1280x720",
+            "dpr=2",
+            polarisSiteData?.machine_id && "mid=" + polarisSiteData.machine_id,
+            "ig_nrcb=1"
+        ].filter(a => a).join('; ');
+
+        return {
+            headers: {
+                'x-ig-app-id': webConfig?.appId || '936619743392459',
+                'X-FB-LSD': lsd,
+                'X-CSRFToken': csrf,
+                'X-Bloks-Version-Id': getObjectFromEntries('WebBloksVersioningID', html)?.versioningID,
+                'x-asbd-id': 129477,
+                cookie: anon_cookie
+            },
+            body: {
+                __d: 'www',
+                __a: '1',
+                __s: '::' + Math.random().toString(36).substring(2).replace(/\d/g, '').slice(0, 6),
+                __hs: siteData?.haste_session || '20126.HYP:instagram_web_pkg.2.1...0',
+                __req: 'b',
+                __ccg: 'EXCELLENT',
+                __rev: pushInfo?.rollout_hash || '1019933358',
+                __hsi: siteData?.hsi || '7436540909012459023',
+                __dyn: randomBytes(154).toString('base64url'),
+                __csr: randomBytes(154).toString('base64url'),
+                __user: '0',
+                __comet_req: getNumberFromQuery('__comet_req', html) || '7',
+                av: '0',
+                dpr: '2',
+                lsd,
+                jazoest: getNumberFromQuery('jazoest', html) || Math.floor(Math.random() * 10000),
+                __spin_r: siteData?.__spin_r || '1019933358',
+                __spin_b: siteData?.__spin_b || 'trunk',
+                __spin_t: siteData?.__spin_t || Math.floor(new Date().getTime() / 1000),
+            }
+        };
+    }
+
     async function requestGQL(id, cookie) {
-        let dtsgId;
+        const { headers, body } = await getGQLParams(id, cookie);
 
-        if (cookie) {
-            dtsgId = await findDtsgId(cookie);
-        }
-        const url = new URL('https://www.instagram.com/api/graphql/');
+        const req = await fetch('https://www.instagram.com/graphql/query', {
+            method: 'POST',
+            dispatcher,
+            headers: {
+                ...embedHeaders,
+                ...headers,
+                cookie,
+                'content-type': 'application/x-www-form-urlencoded',
+                'X-FB-Friendly-Name': 'PolarisPostActionLoadPostQueryQuery',
+            },
+            body: new URLSearchParams({
+                ...body,
+                fb_api_caller_class: 'RelayModern',
+                fb_api_req_friendly_name: 'PolarisPostActionLoadPostQueryQuery',
+                variables: JSON.stringify({
+                    shortcode: id,
+                    fetch_tagged_user_count: null,
+                    hoisted_comment_id: null,
+                    hoisted_reply_id: null
+                }),
+                server_timestamps: true,
+                doc_id: '8845758582119845'
+            }).toString()
+        });
 
-        const requestData = {
-            jazoest: '26406',
-            variables: JSON.stringify({
-            shortcode: id,
-            __relay_internal__pv__PolarisShareMenurelayprovider: false
-            }),
-            doc_id: '7153618348081770'
+        return {
+            gql_data: await req.json()
+                        .then(r => r.data)
+                        .catch(() => null)
         };
-        if (dtsgId) {
-            requestData.fb_dtsg = dtsgId;
+    }
+
+    async function getErrorContext(id) {
+        try {
+            const { headers, body } = await getGQLParams(id);
+
+            const req = await fetch('https://www.instagram.com/ajax/bulk-route-definitions/', {
+                method: 'POST',
+                dispatcher,
+                headers: {
+                    ...embedHeaders,
+                    ...headers,
+                    'content-type': 'application/x-www-form-urlencoded',
+                    'X-Ig-D': 'www',
+                },
+                body: new URLSearchParams({
+                    'route_urls[0]': `/p/${id}/`,
+                    routing_namespace: 'igx_www',
+                    ...body
+                }).toString()
+            });
+
+            const response = await req.text();
+            if (response.includes('"tracePolicy":"polaris.privatePostPage"'))
+                return { error: 'content.post.private' };
+
+            const [, mediaId, mediaOwnerId] = response.match(
+                /"media_id":\s*?"(\d+)","media_owner_id":\s*?"(\d+)"/
+            ) || [];
+
+            if (mediaId && mediaOwnerId) {
+                const rulingURL = new URL('https://www.instagram.com/api/v1/web/get_ruling_for_media_content_logged_out');
+                rulingURL.searchParams.set('media_id', mediaId);
+                rulingURL.searchParams.set('owner_id', mediaOwnerId);
+
+                const rulingResponse = await fetch(rulingURL, {
+                    headers: {
+                        ...headers,
+                        ...commonHeaders
+                    },
+                    dispatcher,
+                }).then(a => a.json()).catch(() => ({}));
+
+                if (rulingResponse?.title?.includes('Restricted'))
+                    return { error: "content.post.age" };
+            }
+        } catch {
+            return { error: "fetch.fail" };
         }
 
-        return (await request(url, cookie, 'POST', requestData))
-                .data
-                ?.xdt_api__v1__media__shortcode__web_info
-                ?.items
-                ?.[0];
+        return { error: "fetch.empty" };
     }
 
     function extractOldPost(data, id, alwaysProxy) {
-        const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children;
+        const shortcodeMedia = data?.gql_data?.shortcode_media || data?.gql_data?.xdt_shortcode_media;
+        const sidecar = shortcodeMedia?.edge_sidecar_to_children;
+
         if (sidecar) {
             const picker = sidecar.edges.filter(e => e.node?.display_url)
                 .map((e, i) => {
-                    const type = e.node?.is_video ? "video" : "photo";
-                    const url = type === "video" ? e.node?.video_url : e.node?.display_url;
+                    const type = e.node?.is_video && e.node?.video_url ? "video" : "photo";
+
+                    let url;
+                    if (type === "video") {
+                        url = e.node?.video_url;
+                    } else if (type === "photo") {
+                        url = e.node?.display_url;
+                    }
 
                     let itemExt = type === "video" ? "mp4" : "jpg";
 
@@ -177,7 +320,7 @@ export default function(obj) {
                     if (alwaysProxy) proxyFile = createStream({
                         service: "instagram",
                         type: "proxy",
-                        u: url,
+                        url,
                         filename: `instagram_${id}_${i + 1}.${itemExt}`
                     });
 
@@ -189,23 +332,28 @@ export default function(obj) {
                         thumb: createStream({
                             service: "instagram",
                             type: "proxy",
-                            u: e.node?.display_url,
+                            url: e.node?.display_url,
                             filename: `instagram_${id}_${i + 1}.jpg`
                         })
                     }
                 });
 
             if (picker.length) return { picker }
-        } else if (data?.gql_data?.shortcode_media?.video_url) {
+        }
+
+        if (shortcodeMedia?.video_url) {
             return {
-                urls: data.gql_data.shortcode_media.video_url,
+                urls: shortcodeMedia.video_url,
                 filename: `instagram_${id}.mp4`,
                 audioFilename: `instagram_${id}_audio`
             }
-        } else if (data?.gql_data?.shortcode_media?.display_url) {
+        }
+
+        if (shortcodeMedia?.display_url) {
             return {
-                urls: data.gql_data?.shortcode_media.display_url,
-                isPhoto: true
+                urls: shortcodeMedia.display_url,
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
             }
         }
     }
@@ -230,7 +378,7 @@ export default function(obj) {
                     if (alwaysProxy) proxyFile = createStream({
                         service: "instagram",
                         type: "proxy",
-                        u: url,
+                        url,
                         filename: `instagram_${id}_${i + 1}.${itemExt}`
                     });
 
@@ -242,7 +390,7 @@ export default function(obj) {
                         thumb: createStream({
                             service: "instagram",
                             type: "proxy",
-                            u: imageUrl,
+                            url: imageUrl,
                             filename: `instagram_${id}_${i + 1}.jpg`
                         })
                     }
@@ -266,6 +414,9 @@ export default function(obj) {
     }
 
     async function getPost(id, alwaysProxy) {
+        const hasData = (data) => data
+                                    && data.gql_data !== null
+                                    && data?.gql_data?.xdt_shortcode_media !== null;
         let data, result;
         try {
             const cookie = getCookie('instagram');
@@ -282,19 +433,21 @@ export default function(obj) {
             if (media_id && token) data = await requestMobileApi(media_id, { token });
 
             // mobile api (no cookie, cookie)
-            if (media_id && !data) data = await requestMobileApi(media_id);
-            if (media_id && cookie && !data) data = await requestMobileApi(media_id, { cookie });
+            if (media_id && !hasData(data)) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie });
 
             // html embed (no cookie, cookie)
-            if (!data) data = await requestHTML(id);
-            if (!data && cookie) data = await requestHTML(id, cookie);
+            if (!hasData(data)) data = await requestHTML(id);
+            if (!hasData(data) && cookie) data = await requestHTML(id, cookie);
 
             // web app graphql api (no cookie, cookie)
-            if (!data) data = await requestGQL(id);
-            if (!data && cookie) data = await requestGQL(id, cookie);
+            if (!hasData(data)) data = await requestGQL(id);
+            if (!hasData(data) && cookie) data = await requestGQL(id, cookie);
         } catch {}
 
-        if (!data) return { error: "fetch.fail" };
+        if (!hasData(data)) {
+            return getErrorContext(id);
+        }
 
         if (data?.gql_data) {
             result = extractOldPost(data, id, alwaysProxy)
@@ -357,14 +510,30 @@ export default function(obj) {
         if (item.image_versions2?.candidates) {
             return {
                 urls: item.image_versions2.candidates[0].url,
-                isPhoto: true
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
             }
         }
 
         return { error: "link.unsupported" };
     }
 
-    const { postId, storyId, username, alwaysProxy } = obj;
+    const { postId, shareId, storyId, username, alwaysProxy } = obj;
+
+    if (shareId) {
+        return resolveRedirectingURL(
+            `https://www.instagram.com/share/${shareId}/`,
+            dispatcher,
+            // for some reason instagram decides to return HTML
+            // instead of a redirect when requesting with a normal
+            // browser user-agent
+            {'User-Agent': 'curl/7.88.1'}
+        ).then(match => instagram({
+            ...obj, ...match,
+            shareId: undefined
+        }));
+    }
+
     if (postId) return getPost(postId, alwaysProxy);
     if (username && storyId) return getStory(username, storyId);
 

api/src/processing/services/ok.js

@@ -1,5 +1,4 @@
 import { genericUserAgent, env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 const resolutions = {
     "ultra": "2160",
@@ -44,8 +43,8 @@ export default async function(o) {
     let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
 
     let fileMetadata = {
-        title: cleanString(videoData.movie.title.trim()),
-        author: cleanString((videoData.author?.name || videoData.compilationTitle).trim()),
+        title: videoData.movie.title.trim(),
+        author: (videoData.author?.name || videoData.compilationTitle)?.trim(),
     }
 
     if (bestVideo) return {

api/src/processing/services/pinterest.js

@@ -1,4 +1,5 @@
 import { genericUserAgent } from "../../config.js";
+import { resolveRedirectingURL } from "../url.js";
 
 const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
 const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
@@ -7,10 +8,10 @@ export default async function(o) {
     let id = o.id;
 
     if (!o.id && o.shortLink) {
-        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
-                   .then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
-                   .catch(() => {});
+        const patternMatch = await resolveRedirectingURL(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`);
+        id = patternMatch?.id;
     }
+
     if (id.includes("--")) id = id.split("--")[1];
     if (!id) return { error: "fetch.fail" };
 
@@ -22,12 +23,12 @@ export default async function(o) {
 
     const videoLink = [...html.matchAll(videoRegex)]
                     .map(([, link]) => link)
-                    .find(a => a.endsWith('.mp4') && a.includes('720p'));
+                    .find(a => a.endsWith('.mp4'));
 
     if (videoLink) return {
         urls: videoLink,
-        filename: `pinterest_${o.id}.mp4`,
-        audioFilename: `pinterest_${o.id}_audio`
+        filename: `pinterest_${id}.mp4`,
+        audioFilename: `pinterest_${id}_audio`
     }
 
     const imageLink = [...html.matchAll(imageRegex)]
@@ -39,7 +40,7 @@ export default async function(o) {
     if (imageLink) return {
         urls: imageLink,
         isPhoto: true,
-        filename: `pinterest_${o.id}.${imageType}`
+        filename: `pinterest_${id}.${imageType}`
     }
 
     return { error: "fetch.empty" };

api/src/processing/services/reddit.js

@@ -1,3 +1,4 @@
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent, env } from "../../config.js";
 import { getCookie, updateCookieValues } from "../cookie/manager.js";
 
@@ -48,23 +49,36 @@ async function getAccessToken() {
 }
 
 export default async function(obj) {
-    let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+    let params = obj;
+    const accessToken = await getAccessToken();
+    const headers = {
+        'user-agent': genericUserAgent,
+        authorization: accessToken && `Bearer ${accessToken}`,
+        accept: 'application/json'
+    };
+
+    if (params.shortId) {
+        params = await resolveRedirectingURL(
+            `https://www.reddit.com/video/${params.shortId}`,
+            obj.dispatcher, headers
+        );
+    }
 
-    if (obj.user) {
-        url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
+    if (!params.id && params.shareId) {
+        params = await resolveRedirectingURL(
+            `https://www.reddit.com/r/${params.sub}/s/${params.shareId}`,
+            obj.dispatcher, headers
+        );
     }
 
-    const accessToken = await getAccessToken();
+    if (!params?.id) return { error: "fetch.short_link" };
+
+    const url = new URL(`https://www.reddit.com/comments/${params.id}.json`);
+
     if (accessToken) url.hostname = 'oauth.reddit.com';
 
     let data = await fetch(
-        url, {
-            headers: {
-                'User-Agent': genericUserAgent,
-                accept: 'application/json',
-                authorization: accessToken && `Bearer ${accessToken}`
-            }
-        }
+        url, { headers }
     ).then(r => r.json()).catch(() => {});
 
     if (!data || !Array.isArray(data)) {
@@ -73,12 +87,17 @@ export default async function(obj) {
 
     data = data[0]?.data?.children[0]?.data;
 
-    const id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
+    let sourceId;
+    if (params.sub || params.user) {
+        sourceId = `${String(params.sub || params.user).toLowerCase()}_${params.id}`;
+    } else {
+        sourceId = params.id;
+    }
 
     if (data?.url?.endsWith('.gif')) return {
         typeId: "redirect",
         urls: data.url,
-        filename: `reddit_${id}.gif`,
+        filename: `reddit_${sourceId}.gif`,
     }
 
     if (!data.secure_media?.reddit_video)
@@ -87,8 +106,9 @@ export default async function(obj) {
     if (data.secure_media?.reddit_video?.duration > env.durationLimit)
         return { error: "content.too_long" };
 
+    const video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0];
+
     let audio = false,
-        video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
         audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
 
     if (video.match('.mp4')) {
@@ -121,7 +141,7 @@ export default async function(obj) {
         typeId: "tunnel",
         type: "merge",
         urls: [video, audioFileLink],
-        audioFilename: `reddit_${id}_audio`,
-        filename: `reddit_${id}.mp4`
+        audioFilename: `reddit_${sourceId}_audio`,
+        filename: `reddit_${sourceId}.mp4`
     }
 }

api/src/processing/services/rutube.js

@@ -1,7 +1,5 @@
 import HLS from "hls-parser";
-
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 async function requestJSON(url) {
     try {
@@ -35,6 +33,10 @@ export default async function(obj) {
     const play = await requestJSON(requestURL);
     if (!play) return { error: "fetch.fail" };
 
+    if (play.detail?.type === "blocking_rule") {
+        return { error: "content.video.region" };
+    }
+
     if (play.detail || !play.video_balancer) return { error: "fetch.empty" };
     if (play.live_streams?.hls) return { error: "content.video.live" };
 
@@ -59,13 +61,13 @@ export default async function(obj) {
     });
 
     const fileMetadata = {
-        title: cleanString(play.title.trim()),
-        artist: cleanString(play.author.name.trim()),
+        title: play.title.trim(),
+        artist: play.author.name.trim(),
     }
 
     return {
         urls: matchingQuality.uri,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             service: "rutube",
             id: obj.id,

api/src/processing/services/snapchat.js

@@ -1,7 +1,6 @@
-import { extract, normalizeURL } from "../url.js";
+import { resolveRedirectingURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { createStream } from "../../stream/manage.js";
-import { getRedirectingURL } from "../../misc/utils.js";
 
 const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
 const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
@@ -41,9 +40,9 @@ async function getStory(username, storyId, alwaysProxy) {
     const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
     if (nextDataString) {
         const data = JSON.parse(nextDataString);
-        const storyIdParam = data.query.profileParams[1];
+        const storyIdParam = data?.query?.profileParams?.[1];
 
-        if (storyIdParam && data.props.pageProps.story) {
+        if (storyIdParam && data?.props?.pageProps?.story) {
             const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
             if (story) {
                 if (story.snapMediaType === 0) {
@@ -62,7 +61,7 @@ async function getStory(username, storyId, alwaysProxy) {
             }
         }
 
-        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        const defaultStory = data?.props?.pageProps?.curatedHighlights?.[0];
         if (defaultStory) {
             return {
                 picker: defaultStory.snapList.map(snap => {
@@ -73,7 +72,7 @@ async function getStory(username, storyId, alwaysProxy) {
                     const proxy = createStream({
                         service: "snapchat",
                         type: "proxy",
-                        u: snapUrl,
+                        url: snapUrl,
                         filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`,
                     });
 
@@ -81,7 +80,7 @@ async function getStory(username, storyId, alwaysProxy) {
                     if (snapType === "video") thumbProxy = createStream({
                         service: "snapchat",
                         type: "proxy",
-                        u: snap.snapUrls.mediaPreviewUrl.value,
+                        url: snap.snapUrls.mediaPreviewUrl.value,
                     });
 
                     if (alwaysProxy) snapUrl = proxy;
@@ -100,18 +99,7 @@ async function getStory(username, storyId, alwaysProxy) {
 export default async function (obj) {
     let params = obj;
     if (obj.shortLink) {
-        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
-
-        if (!link?.startsWith('https://www.snapchat.com/')) {
-            return { error: "fetch.short_link" };
-        }
-
-        const extractResult = extract(normalizeURL(link));
-        if (extractResult?.host !== 'snapchat') {
-            return { error: "fetch.short_link" };
-        }
-
-        params = extractResult.patternMatch;
+        params = await resolveRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
     }
 
     if (params.spotlightId) {

api/src/processing/services/soundcloud.js

@@ -1,5 +1,4 @@
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
 
 const cachedID = {
     version: '',
@@ -63,7 +62,17 @@ export default async function(obj) {
 
     if (!json) return { error: "fetch.fail" };
 
-    if (!json.media.transcodings) return { error: "fetch.empty" };
+    if (json?.policy === "BLOCK") {
+        return { error: "content.region" };
+    }
+
+    if (json?.policy === "SNIP") {
+        return { error: "content.paid" };
+    }
+
+    if (!json?.media?.transcodings || !json?.media?.transcodings.length === 0) {
+        return { error: "fetch.empty" };
+    }
 
     let bestAudio = "opus",
         selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
@@ -75,6 +84,10 @@ export default async function(obj) {
         bestAudio = "mp3"
     }
 
+    if (!selectedStream) {
+        return { error: "fetch.empty" };
+    }
+
     let fileUrlBase = selectedStream.url;
     let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
 
@@ -91,8 +104,8 @@ export default async function(obj) {
     if (!file) return { error: "fetch.empty" };
 
     let fileMetadata = {
-        title: cleanString(json.title.trim()),
-        artist: cleanString(json.user.username.trim()),
+        title: json.title.trim(),
+        artist: json.user.username.trim(),
     }
 
     return {

api/src/processing/services/tiktok.js

@@ -1,6 +1,6 @@
 import Cookie from "../cookie/cookie.js";
 
-import { extract } from "../url.js";
+import { extract, normalizeURL } from "../url.js";
 import { genericUserAgent } from "../../config.js";
 import { updateCookie } from "../cookie/manager.js";
 import { createStream } from "../../stream/manage.js";
@@ -12,7 +12,7 @@ export default async function(obj) {
     let postId = obj.postId;
 
     if (!postId) {
-        let html = await fetch(`${shortDomain}${obj.id}`, {
+        let html = await fetch(`${shortDomain}${obj.shortLink}`, {
             redirect: "manual",
             headers: {
                 "user-agent": genericUserAgent.split(' Chrome/1')[0]
@@ -23,14 +23,14 @@ export default async function(obj) {
 
         if (html.startsWith('<a href="https://')) {
             const extractedURL = html.split('<a href="')[1].split('?')[0];
-            const { patternMatch } = extract(extractedURL);
-            postId = patternMatch.postId
+            const { patternMatch } = extract(normalizeURL(extractedURL));
+            postId = patternMatch?.postId;
         }
     }
     if (!postId) return { error: "fetch.short_link" };
 
     // should always be /video/, even for photos
-    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+    const res = await fetch(`https://www.tiktok.com/@i/video/${postId}`, {
         headers: {
             "user-agent": genericUserAgent,
             cookie,
@@ -44,20 +44,39 @@ export default async function(obj) {
     try {
         const json = html
             .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
-            .split('</script>')[0]
-        const data = JSON.parse(json)
-        detail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"]["itemInfo"]["itemStruct"]
+            .split('</script>')[0];
+
+        const data = JSON.parse(json);
+        const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"];
+
+        if (!videoDetail) throw "no video detail found";
+
+        // status_deleted or etc
+        if (videoDetail.statusMsg) {
+            return { error: "content.post.unavailable"};
+        }
+
+        detail = videoDetail?.itemInfo?.itemStruct;
     } catch {
         return { error: "fetch.fail" };
     }
 
+    if (detail.isContentClassified) {
+        return { error: "content.post.age" };
+    }
+
+    if (!detail.author) {
+        return { error: "fetch.empty" };
+    }
+
     let video, videoFilename, audioFilename, audio, images,
-        filenameBase = `tiktok_${detail.author.uniqueId}_${postId}`,
+        filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`,
         bestAudio; // will get defaulted to m4a later on in match-action
 
     images = detail.imagePost?.images;
 
-    let playAddr = detail.video.playAddr;
+    let playAddr = detail.video?.playAddr;
+
     if (obj.h265) {
         const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
         playAddr = h265PlayAddr || playAddr
@@ -102,7 +121,7 @@ export default async function(obj) {
                 if (obj.alwaysProxy) url = createStream({
                     service: "tiktok",
                     type: "proxy",
-                    u: url,
+                    url,
                     filename: `${filenameBase}_photo_${i + 1}.jpg`
                 })
 

api/src/processing/services/tumblr.js

@@ -1,4 +1,4 @@
-import psl from "psl";
+import psl from "@imput/psl";
 
 const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z';
 const API_BASE = 'https://api-http2.tumblr.com';

api/src/processing/services/twitch.js

@@ -1,5 +1,4 @@
 import { env } from "../../config.js";
-import { cleanString } from '../../misc/utils.js';
 
 const gqlURL = "https://gql.twitch.tv/gql";
 const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };
@@ -73,13 +72,13 @@ export default async function (obj) {
             token: req_token[0].data.clip.playbackAccessToken.value
         })}`,
         fileMetadata: {
-            title: cleanString(clipMetadata.title.trim()),
+            title: clipMetadata.title.trim(),
             artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`,
         },
         filenameAttributes: {
             service: "twitch",
             id: clipMetadata.id,
-            title: cleanString(clipMetadata.title.trim()),
+            title: clipMetadata.title.trim(),
             author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`,
             qualityLabel: `${format.quality}p`,
             extension: 'mp4'

api/src/processing/services/twitter.js

@@ -24,6 +24,11 @@ const badContainerEnd = new Date(1702605600000);
 
 function needsFixing(media) {
     const representativeId = media.source_status_id_str ?? media.id_str;
+
+    // syndication api doesn't have media ids in its response,
+    // so we just assume it's all good
+    if (!representativeId) return false;
+
     const mediaTimestamp = new Date(
         Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
     );
@@ -53,6 +58,25 @@ const getGuestToken = async (dispatcher, forceReload = false) => {
     }
 }
 
+const requestSyndication = async(dispatcher, tweetId) => {
+    // thank you
+    // https://github.com/yt-dlp/yt-dlp/blob/05c8023a27dd37c49163c0498bf98e3e3c1cb4b9/yt_dlp/extractor/twitter.py#L1334
+    const token = (id) => ((Number(id) / 1e15) * Math.PI).toString(36).replace(/(0+|\.)/g, '');
+    const syndicationUrl = new URL("https://cdn.syndication.twimg.com/tweet-result");
+
+    syndicationUrl.searchParams.set("id", tweetId);
+    syndicationUrl.searchParams.set("token", token(tweetId));
+
+    const result = await fetch(syndicationUrl, {
+        headers: {
+            "user-agent": genericUserAgent
+        },
+        dispatcher
+    });
+
+    return result;
+}
+
 const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     const graphqlTweetURL = new URL(graphqlURL);
 
@@ -87,36 +111,24 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {
     let result = await fetch(graphqlTweetURL, { headers, dispatcher });
     updateCookie(cookie, result.headers);
 
-    // we might have been missing the `ct0` cookie, retry
+    // we might have been missing the ct0 cookie, retry
     if (result.status === 403 && result.headers.get('set-cookie')) {
-        result = await fetch(graphqlTweetURL, {
-            headers: {
-                ...headers,
-                'x-csrf-token': cookie.values().ct0
-            },
-            dispatcher
-        });
+        const cookieValues = cookie?.values();
+        if (cookieValues?.ct0) {
+            result = await fetch(graphqlTweetURL, {
+                headers: {
+                    ...headers,
+                    'x-csrf-token': cookieValues.ct0
+                },
+                dispatcher
+            });
+        }
     }
 
     return result
 }
 
-export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
-    const cookie = await getCookie('twitter');
-
-    let guestToken = await getGuestToken(dispatcher);
-    if (!guestToken) return { error: "fetch.fail" };
-
-    let tweet = await requestTweet(dispatcher, id, guestToken);
-
-    // get new token & retry if old one expired
-    if ([403, 429].includes(tweet.status)) {
-        guestToken = await getGuestToken(dispatcher, true);
-        tweet = await requestTweet(dispatcher, id, guestToken)
-    }
-
-    tweet = await tweet.json();
-
+const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {
     let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
 
     if (!tweetTypename) {
@@ -127,13 +139,13 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         const reason = tweet?.data?.tweetResult?.result?.reason;
         switch(reason) {
             case "Protected":
-                return { error: "content.post.private" }
+                return { error: "content.post.private" };
             case "NsfwLoggedOut":
                 if (cookie) {
                     tweet = await requestTweet(dispatcher, id, guestToken, cookie);
                     tweet = await tweet.json();
                     tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
-                } else return { error: "content.post.age" }
+                } else return { error: "content.post.age" };
         }
     }
 
@@ -150,7 +162,69 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
         repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
     }
 
-    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+    return (repostedTweet?.media || baseTweet?.extended_entities?.media);
+}
+
+const testResponse = (result) => {
+    const contentLength = result.headers.get("content-length");
+
+    if (!contentLength || contentLength === '0') {
+        return false;
+    }
+
+    if (!result.headers.get("content-type").startsWith("application/json")) {
+        return false;
+    }
+
+    return true;
+}
+
+export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
+    const cookie = await getCookie('twitter');
+
+    let syndication = false;
+
+    let guestToken = await getGuestToken(dispatcher);
+    if (!guestToken) return { error: "fetch.fail" };
+
+    // for now we assume that graphql api will come back after some time,
+    // so we try it first
+
+    let tweet = await requestTweet(dispatcher, id, guestToken);
+
+    // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
+        guestToken = await getGuestToken(dispatcher, true);
+        if (cookie) {
+            tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+        } else {
+            tweet = await requestTweet(dispatcher, id, guestToken);
+        }
+    }
+
+    const testGraphql = testResponse(tweet);
+
+    // if graphql requests fail, then resort to tweet embed api
+    if (!testGraphql) {
+        syndication = true;
+        tweet = await requestSyndication(dispatcher, id);
+
+        const testSyndication = testResponse(tweet);
+
+        // if even syndication request failed, then cry out loud
+        if (!testSyndication) {
+            return { error: "fetch.fail" };
+        }
+    }
+
+    tweet = await tweet.json();
+
+    let media =
+        syndication
+            ? tweet.mediaDetails
+            : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);
+
+    if (!media) return { error: "fetch.empty" };
 
     // check if there's a video at given index (/video/<index>)
     if (index >= 0 && index < media?.length) {
@@ -159,11 +233,11 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
 
     const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1];
 
-    const proxyMedia = (u, filename) => createStream({
+    const proxyMedia = (url, filename) => createStream({
         service: "twitter",
         type: "proxy",
-        u, filename,
-    })
+        url, filename,
+    });
 
     switch (media?.length) {
         case undefined:
@@ -208,7 +282,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
 
                 let url = bestQuality(content.video_info.variants);
                 const shouldRenderGif = content.type === "animated_gif" && toGif;
-                const videoFilename = `twitter_${id}_${i + 1}.mp4`;
+                const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`;
 
                 let type = "video";
                 if (shouldRenderGif) type = "gif";
@@ -217,7 +291,7 @@ export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
                     url = createStream({
                         service: "twitter",
                         type: shouldRenderGif ? "gif" : "remux",
-                        u: url,
+                        url,
                         filename: videoFilename,
                     })
                 } else if (alwaysProxy) {

api/src/processing/services/vimeo.js

@@ -1,7 +1,6 @@
 import HLS from "hls-parser";
-
 import { env } from "../../config.js";
-import { cleanString, merge } from '../../misc/utils.js';
+import { merge } from '../../misc/utils.js';
 
 const resolutionMatch = {
     "3840": 2160,
@@ -122,7 +121,7 @@ const getHLS = async (configURL, obj) => {
 
     return {
         urls,
-        isM3U8: true,
+        isHLS: true,
         filenameAttributes: {
             resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
             qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
@@ -152,8 +151,8 @@ export default async function(obj) {
     }
 
     const fileMetadata = {
-        title: cleanString(info.name),
-        artist: cleanString(info.user.name),
+        title: info.name,
+        artist: info.user.name,
     };
 
     return merge(

api/src/processing/services/vk.js

@@ -1,63 +1,140 @@
-import { cleanString } from "../../misc/utils.js";
-import { genericUserAgent, env } from "../../config.js";
+import { env } from "../../config.js";
 
-const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240"];
+const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"];
 
-export default async function(o) {
-    let html, url, quality = o.quality === "max" ? 2160 : o.quality;
+const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token";
+const apiUrl = "https://api.vk.com/method";
 
-    html = await fetch(`https://vk.com/video${o.userId}_${o.videoId}`, {
+const clientId = "51552953";
+const clientSecret = "qgr0yWwXCrsxA1jnRtRX";
+
+// used in stream/shared.js for accessing media files
+export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119";
+
+const cachedToken = {
+    token: "",
+    expiry: 0,
+    device_id: "",
+};
+
+const getToken = async () => {
+    if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) {
+        return cachedToken.token;
+    }
+
+    const randomDeviceId = crypto.randomUUID().toUpperCase();
+
+    const anonymOauth = new URL(oauthUrl);
+    anonymOauth.searchParams.set("client_id", clientId);
+    anonymOauth.searchParams.set("client_secret", clientSecret);
+    anonymOauth.searchParams.set("device_id", randomDeviceId);
+
+    const oauthResponse = await fetch(anonymOauth.toString(), {
         headers: {
-            "user-agent": genericUserAgent
+            "user-agent": vkClientAgent,
         }
+    }).then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    if (!oauthResponse) return;
+
+    if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") {
+        cachedToken.token = oauthResponse.token;
+        cachedToken.expiry = oauthResponse.expired_at;
+        cachedToken.device_id = randomDeviceId;
+    }
+
+    if (!cachedToken.token) return;
+
+    return cachedToken.token;
+}
+
+const getVideo = async (ownerId, videoId, accessKey) => {
+    const video = await fetch(`${apiUrl}/video.get`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/x-www-form-urlencoded; charset=utf-8",
+            "user-agent": vkClientAgent,
+        },
+        body: new URLSearchParams({
+            anonymous_token: cachedToken.token,
+            device_id: cachedToken.device_id,
+            lang: "en",
+            v: "5.244",
+            videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`
+        }).toString()
     })
-    .then(r => r.arrayBuffer())
-    .catch(() => {});
+    .then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
 
-    if (!html) return { error: "fetch.fail" };
+    return video;
+}
+
+export default async function ({ ownerId, videoId, accessKey, quality }) {
+    const token = await getToken();
+    if (!token) return { error: "fetch.fail" };
 
-    // decode cyrillic from windows-1251 because vk still uses apis from prehistoric times
-    let decoder = new TextDecoder('windows-1251');
-    html = decoder.decode(html);
+    const videoGet = await getVideo(ownerId, videoId, accessKey);
 
-    if (!html.includes(`{"lang":`)) return { error: "fetch.empty" };
+    if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) {
+        return { error: "fetch.empty" };
+    }
 
-    let js = JSON.parse('{"lang":' + html.split(`{"lang":`)[1].split(']);')[0]);
+    const video = videoGet.response.items[0];
 
-    if (Number(js.mvData.is_active_live) !== 0) {
-        return { error: "content.video.live" };
+    if (video.restriction) {
+        const title = video.restriction.title;
+        if (title.endsWith("country") || title.endsWith("region.")) {
+            return { error: "content.video.region" };
+        }
+        if (title === "Processing video") {
+            return { error: "fetch.empty" };
+        }
+        return { error: "content.video.unavailable" };
+    }
+
+    if (!video.files || !video.duration) {
+        return { error: "fetch.fail" };
     }
 
-    if (js.mvData.duration > env.durationLimit) {
+    if (video.duration > env.durationLimit) {
         return { error: "content.too_long" };
     }
 
-    for (let i in resolutions) {
-        if (js.player.params[0][`url${resolutions[i]}`]) {
-            quality = resolutions[i];
+    const userQuality = quality === "max" ? resolutions[0] : quality;
+    let pickedQuality;
+
+    for (const resolution of resolutions) {
+        if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) {
+            pickedQuality = resolution;
             break
         }
     }
-    if (Number(quality) > Number(o.quality)) quality = o.quality;
 
-    url = js.player.params[0][`url${quality}`];
+    const url = video.files[`mp4_${pickedQuality}`];
+
+    if (!url) return { error: "fetch.fail" };
 
-    let fileMetadata = {
-        title: cleanString(js.player.params[0].md_title.trim()),
-        author: cleanString(js.player.params[0].md_author.trim()),
+    const fileMetadata = {
+        title: video.title.trim(),
     }
 
-    if (url) return {
+    return {
         urls: url,
+        fileMetadata,
         filenameAttributes: {
             service: "vk",
-            id: `${o.userId}_${o.videoId}`,
+            id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`,
             title: fileMetadata.title,
-            author: fileMetadata.author,
-            resolution: `${quality}p`,
-            qualityLabel: `${quality}p`,
+            resolution: `${pickedQuality}p`,
+            qualityLabel: `${pickedQuality}p`,
             extension: "mp4"
         }
     }
-    return { error: "fetch.empty" }
 }

api/src/processing/services/xiaohongshu.js

@@ -0,0 +1,109 @@
+import { resolveRedirectingURL } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+
+const https = (url) => {
+    return url.replace(/^http:/i, 'https:');
+}
+
+export default async function ({ id, token, shareId, h265, isAudioOnly, dispatcher }) {
+    let noteId = id;
+    let xsecToken = token;
+
+    if (!noteId) {
+        const patternMatch = await resolveRedirectingURL(
+            `https://xhslink.com/a/${shareId}`,
+            dispatcher
+        );
+
+        noteId = patternMatch?.id;
+        xsecToken = patternMatch?.token;
+    }
+
+    if (!noteId || !xsecToken) return { error: "fetch.short_link" };
+
+    const res = await fetch(`https://www.xiaohongshu.com/explore/${noteId}?xsec_token=${xsecToken}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+        },
+        dispatcher,
+    });
+
+    const html = await res.text();
+
+    let note;
+    try {
+        const initialState = html
+            .split('<script>window.__INITIAL_STATE__=')[1]
+            .split('</script>')[0]
+            .replace(/:\s*undefined/g, ":null");
+
+        const data = JSON.parse(initialState);
+
+        const noteInfo = data?.note?.noteDetailMap;
+        if (!noteInfo) throw "no note detail map";
+
+        const currentNote = noteInfo[noteId];
+        if (!currentNote) throw "no current note in detail map";
+
+        note = currentNote.note;
+    } catch {}
+
+    if (!note) return { error: "fetch.empty" };
+
+    const video = note.video;
+    const images = note.imageList;
+
+    const filenameBase = `xiaohongshu_${noteId}`;
+
+    if (video) {
+        const videoFilename = `${filenameBase}.mp4`;
+        const audioFilename = `${filenameBase}_audio`;
+
+        let videoURL;
+
+        if (h265 && !isAudioOnly && video.consumer?.originVideoKey) {
+            videoURL = `https://sns-video-bd.xhscdn.com/${video.consumer.originVideoKey}`;
+        } else {
+            const h264Streams = video.media?.stream?.h264;
+
+            if (h264Streams?.length) {
+                videoURL = h264Streams.reduce((a, b) => Number(a?.videoBitrate) > Number(b?.videoBitrate) ? a : b).masterUrl;
+            }
+        }
+
+        if (!videoURL) return { error: "fetch.empty" };
+
+        return {
+            urls: https(videoURL),
+            filename: videoFilename,
+            audioFilename: audioFilename,
+        }
+    }
+
+    if (!images || images.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    if (images.length === 1) {
+        return {
+            isPhoto: true,
+            urls: https(images[0].urlDefault),
+            filename: `${filenameBase}.jpg`,
+        }
+    }
+
+    const picker = images.map((image, i) => {
+        return {
+            type: "photo",
+            url: createStream({
+                service: "xiaohongshu",
+                type: "proxy",
+                url: https(image.urlDefault),
+                filename: `${filenameBase}_${i + 1}.jpg`,
+            })
+        }
+    });
+
+    return { picker };
+}

api/src/processing/services/youtube.js

@@ -1,16 +1,17 @@
-import { fetch } from "undici";
+import HLS from "hls-parser";
 
+import { fetch } from "undici";
 import { Innertube, Session } from "youtubei.js";
 
 import { env } from "../../config.js";
-import { cleanString } from "../../misc/utils.js";
-import { getCookie, updateCookieValues } from "../cookie/manager.js";
+import { getCookie } from "../cookie/manager.js";
+import { getYouTubeSession } from "../helpers/youtube-session.js";
 
 const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
 
 let innertube, lastRefreshedAt;
 
-const codecMatch = {
+const codecList = {
     h264: {
         videoCodec: "avc1",
         audioCodec: "mp4a",
@@ -28,32 +29,43 @@ const codecMatch = {
     }
 }
 
-const transformSessionData = (cookie) => {
-    if (!cookie)
-        return;
-
-    const values = { ...cookie.values() };
-    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
-
-    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
-        return;
+const hlsCodecList = {
+    h264: {
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    vp9: {
+        videoCodec: "vp09",
+        audioCodec: "mp4a",
+        container: "webm"
     }
+}
 
-    if (values.expires) {
-        values.expiry_date = values.expires;
-        delete values.expires;
-    } else if (!values.expiry_date) {
-        return;
-    }
+const clientsWithNoCipher = ['IOS', 'ANDROID', 'YTSTUDIO_ANDROID', 'YTMUSIC_ANDROID'];
 
-    return values;
-}
+const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
 
-const cloneInnertube = async (customFetch) => {
+const cloneInnertube = async (customFetch, useSession) => {
     const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
+
+    const rawCookie = getCookie('youtube');
+    const cookie = rawCookie?.toString();
+
+    const sessionTokens = getYouTubeSession();
+    const retrieve_player = Boolean(sessionTokens || cookie);
+
+    if (useSession && env.ytSessionServer && !sessionTokens?.potoken) {
+        throw "no_session_tokens";
+    }
+
     if (!innertube || shouldRefreshPlayer) {
         innertube = await Innertube.create({
-            fetch: customFetch
+            fetch: customFetch,
+            retrieve_player,
+            cookie,
+            po_token: useSession ? sessionTokens?.potoken : undefined,
+            visitor_data: useSession ? sessionTokens?.visitor_data : undefined,
         });
         lastRefreshedAt = +new Date();
     }
@@ -64,81 +76,88 @@ const cloneInnertube = async (customFetch) => {
         innertube.session.api_version,
         innertube.session.account_index,
         innertube.session.player,
-        undefined,
+        cookie,
         customFetch ?? innertube.session.http.fetch,
         innertube.session.cache
     );
 
-    const cookie = getCookie('youtube_oauth');
-    const oauthData = transformSessionData(cookie);
+    const yt = new Innertube(session);
+    return yt;
+}
 
-    if (!session.logged_in && oauthData) {
-        await session.oauth.init(oauthData);
-        session.logged_in = true;
-    }
+export default async function (o) {
+    const quality = o.quality === "max" ? 9000 : Number(o.quality);
 
-    if (session.logged_in) {
-        if (session.oauth.shouldRefreshToken()) {
-            await session.oauth.refreshAccessToken();
-        }
+    let useHLS = o.youtubeHLS;
+    let innertubeClient = o.innertubeClient || env.customInnertubeClient || "IOS";
 
-        const cookieValues = cookie.values();
-        const oldExpiry = new Date(cookieValues.expiry_date);
-        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
+    // HLS playlists from the iOS client don't contain the av1 video format.
+    if (useHLS && o.format === "av1") {
+        useHLS = false;
+    }
 
-        if (oldExpiry.getTime() !== newExpiry.getTime()) {
-            updateCookieValues(cookie, {
-                ...session.oauth.client_id,
-                ...session.oauth.oauth2_tokens,
-                expiry_date: newExpiry.toISOString()
-            });
-        }
+    if (useHLS) {
+        innertubeClient = "IOS";
     }
 
-    const yt = new Innertube(session);
-    return yt;
-}
+    // iOS client doesn't have adaptive formats of resolution >1080p,
+    // so we use the WEB_EMBEDDED client instead for those cases
+    const useSession =
+        env.ytSessionServer && (
+            (
+                !useHLS
+                && innertubeClient === "IOS"
+                && (
+                    (quality > 1080 && o.format !== "h264")
+                    || (quality > 1080 && o.format !== "vp9")
+                )
+            )
+        );
+
+    if (useSession) {
+        innertubeClient = env.ytSessionInnertubeClient || "WEB_EMBEDDED";
+    }
 
-export default async function(o) {
     let yt;
     try {
         yt = await cloneInnertube(
             (input, init) => fetch(input, {
                 ...init,
                 dispatcher: o.dispatcher
-            })
+            }),
+            useSession
         );
-    } catch(e) {
-        if (e.message?.endsWith("decipher algorithm")) {
+    } catch (e) {
+        if (e === "no_session_tokens") {
+            return { error: "youtube.no_session_tokens" };
+        } else if (e.message?.endsWith("decipher algorithm")) {
             return { error: "youtube.decipher" }
         } else if (e.message?.includes("refresh access token")) {
             return { error: "youtube.token_expired" }
         } else throw e;
     }
 
-    const quality = o.quality === "max" ? "9000" : o.quality;
-
-    let info, isDubbed,
-        format = o.format || "h264";
-
-    function qual(i) {
-        if (!i.quality_label) {
-            return;
+    let info;
+    try {
+        info = await yt.getBasicInfo(o.id, innertubeClient);
+    } catch (e) {
+        if (e?.info) {
+            let errorInfo;
+            try { errorInfo = JSON.parse(e?.info); } catch {}
+
+            if (errorInfo?.reason === "This video is private") {
+                return { error: "content.video.private" };
+            }
+            if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) {
+                return { error: "youtube.api_error" };
+            }
         }
 
-        return i.quality_label.split('p')[0].split('s')[0]
-    }
-
-    try {
-        info = await yt.getBasicInfo(o.id, yt.session.logged_in ? 'ANDROID' : 'IOS');
-    } catch(e) {
-        if (e?.info?.reason === "This video is private") {
-            return { error: "content.video.private" };
-        } else if (e?.message === "This video is unavailable") {
+        if (e?.message === "This video is unavailable") {
             return { error: "content.video.unavailable" };
-        } else {
-            return { error: "fetch.fail" };
         }
+
+        return { error: "fetch.fail" };
     }
 
     if (!info) return { error: "fetch.fail" };
@@ -146,37 +165,47 @@ export default async function(o) {
     const playability = info.playability_status;
     const basicInfo = info.basic_info;
 
-    if (playability.status === "LOGIN_REQUIRED") {
-        if (playability.reason.endsWith("bot")) {
-            return { error: "youtube.login" }
-        }
-        if (playability.reason.endsWith("age")) {
-            return { error: "content.video.age" }
-        }
-        if (playability?.error_screen?.reason?.text === "Private video") {
-            return { error: "content.video.private" }
-        }
-    }
+    switch (playability.status) {
+        case "LOGIN_REQUIRED":
+            if (playability.reason.endsWith("bot")) {
+                return { error: "youtube.login" }
+            }
+            if (playability.reason.endsWith("age") || playability.reason.endsWith("inappropriate for some users.")) {
+                return { error: "content.video.age" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
 
-    if (playability.status === "UNPLAYABLE") {
-        if (playability?.reason?.endsWith("request limit.")) {
-            return { error: "fetch.rate" }
-        }
-        if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
-            return { error: "content.video.region" }
-        }
-        if (playability?.error_screen?.reason?.text === "Private video") {
-            return { error: "content.video.private" }
-        }
+        case "UNPLAYABLE":
+            if (playability?.reason?.endsWith("request limit.")) {
+                return { error: "fetch.rate" }
+            }
+            if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
+                return { error: "content.video.region" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
+
+        case "AGE_VERIFICATION_REQUIRED":
+            return { error: "content.video.age" };
     }
 
     if (playability.status !== "OK") {
         return { error: "content.video.unavailable" };
     }
+
     if (basicInfo.is_live) {
         return { error: "content.video.live" };
     }
 
+    if (basicInfo.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
     // return a critical error if returned video is "Video Not Available"
     // or a similar stub by youtube
     if (basicInfo.id !== o.id) {
@@ -186,64 +215,206 @@ export default async function(o) {
         }
     }
 
-    const filterByCodec = (formats) =>
-        formats
-        .filter(e =>
-            e.mime_type.includes(codecMatch[format].videoCodec)
-            || e.mime_type.includes(codecMatch[format].audioCodec)
-        )
-        .sort((a, b) => Number(b.bitrate) - Number(a.bitrate));
+    const normalizeQuality = res => {
+        const shortestSide = Math.min(res.height, res.width);
+        return videoQualities.find(qual => qual >= shortestSide);
+    }
 
-    let adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats);
+    let video, audio, dubbedLanguage,
+        codec = o.format || "h264", itag = o.itag;
 
-    if (adaptive_formats.length === 0 && format === "vp9") {
-        format = "h264"
-        adaptive_formats = filterByCodec(info.streaming_data.adaptive_formats)
-    }
+    if (useHLS) {
+        const hlsManifest = info.streaming_data.hls_manifest_url;
 
-    let bestQuality;
+        if (!hlsManifest) {
+            return { error: "youtube.no_hls_streams" };
+        }
 
-    const bestVideo = adaptive_formats.find(i => i.has_video && i.content_length);
-    const hasAudio = adaptive_formats.find(i => i.has_audio && i.content_length);
+        const fetchedHlsManifest = await fetch(hlsManifest, {
+            dispatcher: o.dispatcher,
+        }).then(r => {
+            if (r.status === 200) {
+                return r.text();
+            } else {
+                throw new Error("couldn't fetch the HLS playlist");
+            }
+        }).catch(() => { });
 
-    if (bestVideo) bestQuality = qual(bestVideo);
+        if (!fetchedHlsManifest) {
+            return { error: "youtube.no_hls_streams" };
+        }
 
-    if ((!bestQuality && !o.isAudioOnly) || !hasAudio)
-        return { error: "youtube.codec" };
+        const variants = HLS.parse(fetchedHlsManifest).variants.sort(
+            (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+        );
 
-    if (basicInfo.duration > env.durationLimit)
-        return { error: "content.too_long" };
+        if (!variants || variants.length === 0) {
+            return { error: "youtube.no_hls_streams" };
+        }
 
-    const checkBestAudio = (i) => (i.has_audio && !i.has_video);
+        const matchHlsCodec = codecs => (
+            codecs.includes(hlsCodecList[codec].videoCodec)
+        );
 
-    let audio = adaptive_formats.find(i =>
-        checkBestAudio(i) && i.is_original
-    );
+        const best = variants.find(i => matchHlsCodec(i.codecs));
+
+        const preferred = variants.find(i =>
+            matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality
+        );
+
+        let selected = preferred || best;
+
+        if (!selected) {
+            codec = "h264";
+            selected = variants.find(i => matchHlsCodec(i.codecs));
+        }
+
+        if (!selected) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = selected.audio.find(i => i.isDefault);
+
+        // some videos (mainly those with AI dubs) don't have any tracks marked as default
+        // why? god knows, but we assume that a default track is marked as such in the title
+        if (!audio) {
+            audio = selected.audio.find(i => i.name.endsWith("- original"));
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = selected.audio.find(i =>
+                i.language?.startsWith(o.dubLang)
+            );
+
+            if (dubbedAudio && !dubbedAudio.isDefault) {
+                dubbedLanguage = dubbedAudio.language;
+                audio = dubbedAudio;
+            }
+        }
+
+        selected.audio = [];
+        selected.subtitles = [];
+        video = selected;
+    } else {
+        // i miss typescript so bad
+        const sorted_formats = {
+            h264: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            vp9: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            av1: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+        }
+
+        const checkFormat = (format, pCodec) => format.content_length &&
+            (format.mime_type.includes(codecList[pCodec].videoCodec)
+                || format.mime_type.includes(codecList[pCodec].audioCodec));
+
+        // sort formats & weed out bad ones
+        info.streaming_data.adaptive_formats.sort((a, b) =>
+            Number(b.bitrate) - Number(a.bitrate)
+        ).forEach(format => {
+            Object.keys(codecList).forEach(yCodec => {
+                const matchingItag = slot => !itag?.[slot] || itag[slot] === format.itag;
+                const sorted = sorted_formats[yCodec];
+                const goodFormat = checkFormat(format, yCodec);
+                if (!goodFormat) return;
+
+                if (format.has_video && matchingItag('video')) {
+                    sorted.video.push(format);
+                    if (!sorted.bestVideo)
+                        sorted.bestVideo = format;
+                }
+
+                if (format.has_audio && matchingItag('audio')) {
+                    sorted.audio.push(format);
+                    if (!sorted.bestAudio)
+                        sorted.bestAudio = format;
+                }
+            })
+        });
+
+        const noBestMedia = () => {
+            const vid = sorted_formats[codec]?.bestVideo;
+            const aud = sorted_formats[codec]?.bestAudio;
+            return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly)
+        };
+
+        if (noBestMedia()) {
+            if (codec === "av1") codec = "vp9";
+            else if (codec === "vp9") codec = "av1";
 
-    if (o.dubLang) {
-        let dubbedAudio = adaptive_formats.find(i =>
-            checkBestAudio(i)
-            && i.language === o.dubLang
-            && i.audio_track
-        )
+            // if there's no higher quality fallback, then use h264
+            if (noBestMedia()) codec = "h264";
+        }
+
+        // if there's no proper combo of av1, vp9, or h264, then give up
+        if (noBestMedia()) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = sorted_formats[codec].bestAudio;
+
+        if (audio?.audio_track && !audio?.audio_track?.audio_is_default) {
+            audio = sorted_formats[codec].audio.find(i =>
+                i?.audio_track?.audio_is_default
+            );
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = sorted_formats[codec].audio.find(i =>
+                i.language?.startsWith(o.dubLang) && i.audio_track
+            );
+
+            if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
+                audio = dubbedAudio;
+                dubbedLanguage = dubbedAudio.language;
+            }
+        }
+
+        if (!o.isAudioOnly) {
+            const qual = (i) => {
+                return normalizeQuality({
+                    width: i.width,
+                    height: i.height,
+                })
+            }
 
-        if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
-            audio = dubbedAudio;
-            isDubbed = true;
+            const bestQuality = qual(sorted_formats[codec].bestVideo);
+            const useBestQuality = quality >= bestQuality;
+
+            video = useBestQuality
+                ? sorted_formats[codec].bestVideo
+                : sorted_formats[codec].video.find(i => qual(i) === quality);
+
+            if (!video) video = sorted_formats[codec].bestVideo;
         }
     }
 
-    if (!audio) {
-        audio = adaptive_formats.find(i => checkBestAudio(i));
+    if (video?.drm_families || audio?.drm_families) {
+        return { error: "youtube.drm" };
     }
 
-    let fileMetadata = {
-        title: cleanString(basicInfo.title.trim()),
-        artist: cleanString(basicInfo.author.replace("- Topic", "").trim()),
+    const fileMetadata = {
+        title: basicInfo.title.trim(),
+        artist: basicInfo.author.replace("- Topic", "").trim()
     }
 
     if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
-        let descItems = basicInfo.short_description.split("\n\n", 5);
+        const descItems = basicInfo.short_description.split("\n\n", 5);
+
         if (descItems.length === 5) {
             fileMetadata.album = descItems[2];
             fileMetadata.copyright = descItems[3];
@@ -253,61 +424,94 @@ export default async function(o) {
         }
     }
 
-    let filenameAttributes = {
+    const filenameAttributes = {
         service: "youtube",
         id: o.id,
         title: fileMetadata.title,
         author: fileMetadata.artist,
-        youtubeDubName: isDubbed ? o.dubLang : false
+        youtubeDubName: dubbedLanguage || false,
     }
 
-    if (audio && o.isAudioOnly) return {
-        type: "audio",
-        isAudioOnly: true,
-        urls: audio.decipher(yt.session.player),
-        filenameAttributes: filenameAttributes,
-        fileMetadata: fileMetadata,
-        bestAudio: format === "h264" ? "m4a" : "opus"
+    itag = {
+        video: video?.itag,
+        audio: audio?.itag
+    };
+
+    const originalRequest = {
+        ...o,
+        dispatcher: undefined,
+        itag,
+        innertubeClient
+    };
+
+    if (audio && o.isAudioOnly) {
+        let bestAudio = codec === "h264" ? "m4a" : "opus";
+        let urls = audio.url;
+
+        if (useHLS) {
+            bestAudio = "mp3";
+            urls = audio.uri;
+        }
+
+        if (!clientsWithNoCipher.includes(innertubeClient) && innertube) {
+            urls = audio.decipher(innertube.session.player);
+        }
+
+        return {
+            type: "audio",
+            isAudioOnly: true,
+            urls,
+            filenameAttributes,
+            fileMetadata,
+            bestAudio,
+            isHLS: useHLS,
+            originalRequest
+        }
     }
 
-    const matchingQuality = Number(quality) > Number(bestQuality) ? bestQuality : quality,
-        checkSingle = i =>
-            qual(i) === matchingQuality && i.mime_type.includes(codecMatch[format].videoCodec),
-        checkRender = i =>
-            qual(i) === matchingQuality && i.has_video && !i.has_audio;
+    if (video && audio) {
+        let resolution;
 
-    let match, type, urls;
+        if (useHLS) {
+            resolution = normalizeQuality(video.resolution);
+            filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`;
+            filenameAttributes.extension = hlsCodecList[codec].container;
 
-    // prefer good premuxed videos if available
-    if (!o.isAudioOnly && !o.isAudioMuted && format === "h264" && bestVideo.fps <= 30) {
-        match = info.streaming_data.formats.find(checkSingle);
-        type = "proxy";
-        urls = match?.decipher(yt.session.player);
-    }
+            video = video.uri;
+            audio = audio.uri;
+        } else {
+            resolution = normalizeQuality({
+                width: video.width,
+                height: video.height,
+            });
 
-    const video = adaptive_formats.find(checkRender);
+            filenameAttributes.resolution = `${video.width}x${video.height}`;
+            filenameAttributes.extension = codecList[codec].container;
 
-    if (!match && video && audio) {
-        match = video;
-        type = "merge";
-        urls = [
-            video.decipher(yt.session.player),
-            audio.decipher(yt.session.player)
-        ]
-    }
+            if (!clientsWithNoCipher.includes(innertubeClient) && innertube) {
+                video = video.decipher(innertube.session.player);
+                audio = audio.decipher(innertube.session.player);
+            } else {
+                video = video.url;
+                audio = audio.url;
+            }
+        }
+
+        filenameAttributes.qualityLabel = `${resolution}p`;
+        filenameAttributes.youtubeFormat = codec;
 
-    if (match) {
-        filenameAttributes.qualityLabel = match.quality_label;
-        filenameAttributes.resolution = `${match.width}x${match.height}`;
-        filenameAttributes.extension = codecMatch[format].container;
-        filenameAttributes.youtubeFormat = format;
         return {
-            type,
-            urls,
+            type: "merge",
+            urls: [
+                video,
+                audio,
+            ],
             filenameAttributes,
-            fileMetadata
+            fileMetadata,
+            isHLS: useHLS,
+            originalRequest
         }
     }
 
-    return { error: "fetch.fail" }
+    return { error: "youtube.no_matching_format" };
 }

api/src/processing/url.js

@@ -1,8 +1,9 @@
-import psl from "psl";
+import psl from "@imput/psl";
 import { strict as assert } from "node:assert";
 
 import { env } from "../config.js";
 import { services } from "./service-config.js";
+import { getRedirectingURL } from "../misc/utils.js";
 import { friendlyServiceName } from "./service-alias.js";
 
 function aliasURL(url) {
@@ -42,7 +43,7 @@ function aliasURL(url) {
         case "fixvx":
         case "x":
             if (services.twitter.altDomains.includes(url.hostname)) {
-                url.hostname = 'twitter.com'
+                url.hostname = 'twitter.com';
             }
             break;
 
@@ -85,9 +86,37 @@ function aliasURL(url) {
                 url.hostname = 'instagram.com';
             }
             break;
+
+        case "vk":
+        case "vkvideo":
+            if (services.vk.altDomains.includes(url.hostname)) {
+                url.hostname = 'vk.com';
+            }
+            break;
+
+        case "xhslink":
+            if (url.hostname === 'xhslink.com' && parts.length === 3) {
+                url = new URL(`https://www.xiaohongshu.com/a/${parts[2]}`);
+            }
+            break;
+
+        case "loom":
+            const idPart = parts[parts.length - 1];
+            if (idPart.length > 32) {
+                url.pathname = `/share/${idPart.slice(-32)}`;
+            }
+            break;
+            
+        case "redd":
+            /* reddit short video links can be treated by changing https://v.redd.it/<id>
+            to https://reddit.com/video/<id>.*/
+            if (url.hostname === "v.redd.it" && parts.length === 2) {
+                url = new URL(`https://www.reddit.com/video/${parts[1]}`);
+            }
+            break;
     }
 
-    return url
+    return url;
 }
 
 function cleanURL(url) {
@@ -107,31 +136,41 @@ function cleanURL(url) {
             break;
         case "vk":
             if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
-                limitQuery('z')
+                limitQuery('z');
             }
             break;
         case "youtube":
             if (url.searchParams.get('v')) {
-                limitQuery('v')
+                limitQuery('v');
             }
             break;
         case "rutube":
             if (url.searchParams.get('p')) {
-                limitQuery('p')
+                limitQuery('p');
+            }
+            break;
+        case "twitter":
+            if (url.searchParams.get('post_id')) {
+                limitQuery('post_id');
+            }
+            break;
+        case "xiaohongshu":
+            if (url.searchParams.get('xsec_token')) {
+                limitQuery('xsec_token');
             }
             break;
     }
 
     if (stripQuery) {
-        url.search = ''
+        url.search = '';
     }
 
-    url.username = url.password = url.port = url.hash = ''
+    url.username = url.password = url.port = url.hash = '';
 
     if (url.pathname.endsWith('/'))
         url.pathname = url.pathname.slice(0, -1);
 
-    return url
+    return url;
 }
 
 function getHostIfValid(url) {
@@ -169,6 +208,11 @@ export function extract(url) {
     }
 
     if (!env.enabledServices.has(host)) {
+        // show a different message when youtube is disabled on official instances
+        // as it only happens when shit hits the fan
+        if (new URL(env.apiURL).hostname.endsWith(".imput.net") && host === "youtube") {
+            return { error: "youtube.temporary_disabled" };
+        }
         return { error: "service.disabled" };
     }
 
@@ -194,3 +238,17 @@ export function extract(url) {
 
     return { host, patternMatch };
 }
+
+export async function resolveRedirectingURL(url, dispatcher, headers) {
+    const originalService = getHostIfValid(normalizeURL(url));
+    if (!originalService) return;
+
+    const canonicalURL = await getRedirectingURL(url, dispatcher, headers);
+    if (!canonicalURL) return;
+
+    const { host, patternMatch } = extract(normalizeURL(canonicalURL));
+
+    if (host === originalService) {
+        return patternMatch;
+    }
+}

api/src/security/api-keys.js

@@ -0,0 +1,227 @@
+import { env } from "../config.js";
+import { readFile } from "node:fs/promises";
+import { Green, Yellow } from "../misc/console-text.js";
+import ip from "ipaddr.js";
+import * as cluster from "../misc/cluster.js";
+
+// this function is a modified variation of code
+// from https://stackoverflow.com/a/32402438/14855621
+const generateWildcardRegex = rule => {
+    var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$");
+}
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+let keys = {};
+
+const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']);
+
+/* Expected format pseudotype:
+** type KeyFileContents = Record<
+**    UUIDv4String,
+**    {
+**        name?: string,
+**        limit?: number | "unlimited",
+**        ips?: CIDRString[],
+**        userAgents?: string[]
+**    }
+** >;
+*/
+
+const validateKeys = (input) => {
+    if (typeof input !== 'object' || input === null) {
+        throw "input is not an object";
+    }
+
+    if (Object.keys(input).some(x => !UUID_REGEX.test(x))) {
+        throw "key file contains invalid key(s)";
+    }
+
+    Object.values(input).forEach(details => {
+        if (typeof details !== 'object' || details === null) {
+            throw "some key(s) are incorrectly configured";
+        }
+
+        const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k));
+        if (unexpected_key) {
+            throw "detail object contains unexpected key: " + unexpected_key;
+        }
+
+        if (details.limit && details.limit !== 'unlimited') {
+            if (typeof details.limit !== 'number')
+                throw "detail object contains invalid limit (not a number)";
+            else if (details.limit < 1)
+                throw "detail object contains invalid limit (not a positive number)";
+        }
+
+        if (details.ips) {
+            if (!Array.isArray(details.ips))
+                throw "details object contains value for `ips` which is not an array";
+
+            const invalid_ip = details.ips.find(
+                addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr))
+            );
+
+            if (invalid_ip) {
+                throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip;
+            }
+        }
+
+        if (details.userAgents) {
+            if (!Array.isArray(details.userAgents))
+                throw "details object contains value for `userAgents` which is not an array";
+
+            const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string');
+            if (invalid_ua) {
+                throw "`userAgents` in details contains an invalid user agent: " + invalid_ua;
+            }
+        }
+    });
+}
+
+const formatKeys = (keyData) => {
+    const formatted = {};
+
+    for (let key in keyData) {
+        const data = keyData[key];
+        key = key.toLowerCase();
+
+        formatted[key] = {};
+
+        if (data.limit) {
+            if (data.limit === "unlimited") {
+                data.limit = Infinity;
+            }
+
+            formatted[key].limit = data.limit;
+        }
+
+        if (data.ips) {
+            formatted[key].ips = data.ips.map(addr => {
+                if (ip.isValid(addr)) {
+                    const parsed = ip.parse(addr);
+                    const range = parsed.kind() === 'ipv6' ? 128 : 32;
+                    return [ parsed, range ];
+                }
+
+                return ip.parseCIDR(addr);
+            });
+        }
+
+        if (data.userAgents) {
+            formatted[key].userAgents = data.userAgents.map(generateWildcardRegex);
+        }
+    }
+
+    return formatted;
+}
+
+const updateKeys = (newKeys) => {
+    keys = formatKeys(newKeys);
+}
+
+const loadKeys = async (source) => {
+    let updated;
+    if (source.protocol === 'file:') {
+        const pathname = source.pathname === '/' ? '' : source.pathname;
+        updated = JSON.parse(
+            await readFile(
+                decodeURIComponent(source.host + pathname),
+                'utf8'
+            )
+        );
+    } else {
+        updated = await fetch(source).then(a => a.json());
+    }
+
+    validateKeys(updated);
+
+    cluster.broadcast({ api_keys: updated });
+
+    updateKeys(updated);
+}
+
+const wrapLoad = (url, initial = false) => {
+    loadKeys(url)
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} api keys loaded successfully!`)
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+const err = (reason) => ({ success: false, error: reason });
+
+export const validateAuthorization = (req) => {
+    const authHeader = req.get('Authorization');
+
+    if (typeof authHeader !== 'string') {
+        return err("missing");
+    }
+
+    const [ authType, keyString ] = authHeader.split(' ', 2);
+    if (authType.toLowerCase() !== 'api-key') {
+        return err("not_api_key");
+    }
+
+    if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) {
+        return err("invalid");
+    }
+
+    const matchingKey = keys[keyString.toLowerCase()];
+    if (!matchingKey) {
+        return err("not_found");
+    }
+
+    if (matchingKey.ips) {
+        let addr;
+        try {
+            addr = ip.parse(req.ip);
+        } catch {
+            return err("invalid_ip");
+        }
+
+        const ip_allowed = matchingKey.ips.some(
+            ([ allowed, size ]) => {
+                return addr.kind() === allowed.kind()
+                        && addr.match(allowed, size);
+            }
+        );
+
+        if (!ip_allowed) {
+            return err("ip_not_allowed");
+        }
+    }
+
+    if (matchingKey.userAgents) {
+        const userAgent = req.get('User-Agent');
+        if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) {
+            return err("ua_not_allowed");
+        }
+    }
+
+    req.rateLimitKey = keyString.toLowerCase();
+    req.rateLimitMax = matchingKey.limit;
+
+    return { success: true };
+}
+
+export const setup = (url) => {
+    if (cluster.isPrimary) {
+        wrapLoad(url, true);
+        if (env.keyReloadInterval > 0) {
+            setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('api_keys' in message) {
+                updateKeys(message.api_keys);
+            }
+        });
+    }
+}

api/src/security/jwt.js

@@ -6,12 +6,19 @@ import { env } from "../config.js";
 const toBase64URL = (b) => Buffer.from(b).toString("base64url");
 const fromBase64URL = (b) => Buffer.from(b, "base64url").toString();
 
-const makeHmac = (header, payload) =>
-    createHmac("sha256", env.jwtSecret)
-        .update(`${header}.${payload}`)
-        .digest("base64url");
+const makeHmac = (data) => {
+    return createHmac("sha256", env.jwtSecret)
+            .update(data)
+            .digest("base64url");
+}
+
+const sign = (header, payload) =>
+        makeHmac(`${header}.${payload}`);
+
+const getIPHash = (ip) =>
+        makeHmac(ip).slice(0, 8);
 
-const generate = () => {
+const generate = (ip) => {
     const exp = Math.floor(new Date().getTime() / 1000) + env.jwtLifetime;
 
     const header = toBase64URL(JSON.stringify({
@@ -21,10 +28,11 @@ const generate = () => {
 
     const payload = toBase64URL(JSON.stringify({
         jti: nanoid(8),
+        sub: getIPHash(ip),
         exp,
     }));
 
-    const signature = makeHmac(header, payload);
+    const signature = sign(header, payload);
 
     return {
         token: `${header}.${payload}.${signature}`,
@@ -32,7 +40,7 @@ const generate = () => {
     };
 }
 
-const verify = (jwt) => {
+const verify = (jwt, ip) => {
     const [header, payload, signature] = jwt.split(".", 3);
     const timestamp = Math.floor(new Date().getTime() / 1000);
 
@@ -40,17 +48,16 @@ const verify = (jwt) => {
         return false;
     }
 
-    const verifySignature = makeHmac(header, payload);
+    const verifySignature = sign(header, payload);
 
     if (verifySignature !== signature) {
         return false;
     }
 
-    if (timestamp >= JSON.parse(fromBase64URL(payload)).exp) {
-        return false;
-    }
+    const data = JSON.parse(fromBase64URL(payload));
 
-    return true;
+    return getIPHash(ip) === data.sub
+            && timestamp <= data.exp;
 }
 
 export default {

api/src/security/secrets.js

@@ -0,0 +1,62 @@
+import cluster from "node:cluster";
+import { createHmac, randomBytes } from "node:crypto";
+
+const generateSalt = () => {
+    if (cluster.isPrimary)
+        return randomBytes(64);
+
+    return null;
+}
+
+let rateSalt = generateSalt();
+let streamSalt = generateSalt();
+
+export const syncSecrets = () => {
+    return new Promise((resolve, reject) => {
+        if (cluster.isPrimary) {
+            let remaining = Object.values(cluster.workers).length;
+            const handleReady = (worker, m) => {
+                if (m.ready)
+                    worker.send({ rateSalt, streamSalt });
+
+                if (!--remaining)
+                    resolve();
+            }
+
+            for (const worker of Object.values(cluster.workers)) {
+                worker.once(
+                    'message',
+                    (m) => handleReady(worker, m)
+                );
+            }
+        } else if (cluster.isWorker) {
+            if (rateSalt || streamSalt)
+                return reject();
+
+            process.send({ ready: true });
+            process.once('message', (message) => {
+                if (rateSalt || streamSalt)
+                    return reject();
+
+                if (message.rateSalt && message.streamSalt) {
+                    streamSalt = Buffer.from(message.streamSalt);
+                    rateSalt = Buffer.from(message.rateSalt);
+                    resolve();
+                }
+            });
+        } else reject();
+    });
+}
+
+
+export const hashHmac = (value, type) => {
+    let salt;
+    if (type === 'rate')
+        salt = rateSalt;
+    else if (type === 'stream')
+        salt = streamSalt;
+    else
+        throw "unknown salt";
+
+    return createHmac("sha256", salt).update(value).digest();
+}

api/src/store/base-store.js

@@ -0,0 +1,48 @@
+const _stores = new Set();
+
+export class Store {
+    id;
+
+    constructor(name) {
+        name = name.toUpperCase();
+
+        if (_stores.has(name))
+            throw `${name} store already exists`;
+        _stores.add(name);
+
+        this.id = name;
+    }
+
+    async _has(_key) { await Promise.reject("needs implementation"); }
+    has(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        return this._has(key);
+    }
+
+    async _get(_key) { await Promise.reject("needs implementation"); }
+    async get(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        const val = await this._get(key);
+        if (val === null)
+            return null;
+
+        return val;
+    }
+
+    async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") }
+    set(key, val, exp_sec = -1) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        exp_sec = Math.round(exp_sec);
+
+        return this._set(key, val, exp_sec);
+    }
+};

api/src/store/memory-store.js

@@ -0,0 +1,77 @@
+import { MinPriorityQueue } from '@datastructures-js/priority-queue';
+import { Store } from './base-store.js';
+
+// minimum delay between sweeps to avoid repeatedly
+// sweeping entries close in proximity one by one.
+const MIN_THRESHOLD_MS = 2500;
+
+export default class MemoryStore extends Store {
+    #store = new Map();
+    #timeouts = new MinPriorityQueue/*<{ t: number, k: unknown }>*/((obj) => obj.t);
+    #nextSweep = { id: null, t: null };
+
+    constructor(name) {
+        super(name);
+    }
+
+    _has(key) {
+        return this.#store.has(key);
+    }
+
+    _get(key) {
+        const val = this.#store.get(key);
+
+        return val === undefined ? null : val;
+    }
+
+    _set(key, val, exp_sec = -1) {
+        if (this.#store.has(key)) {
+            this.#timeouts.remove(o => o.k === key);
+        }
+
+        if (exp_sec > 0) {
+            const exp = 1000 * exp_sec;
+            const timeout_at = +new Date() + exp;
+
+            this.#timeouts.enqueue({ k: key, t: timeout_at });
+        }
+
+        this.#store.set(key, val);
+        this.#reschedule();
+    }
+
+    #reschedule() {
+        const current_time = new Date().getTime();
+        const time = this.#timeouts.front()?.t;
+        if (!time) {
+            return;
+        } else if (time < current_time) {
+            return this.#sweepNow();
+        }
+
+        const sweep = this.#nextSweep;
+        if (sweep.id === null || sweep.t > time) {
+            if (sweep.id) {
+                clearTimeout(sweep.id);
+            }
+
+            sweep.t = time;
+            sweep.id = setTimeout(
+                () => this.#sweepNow(),
+                Math.max(MIN_THRESHOLD_MS, time - current_time)
+            );
+            sweep.id.unref();
+        }
+    }
+
+    #sweepNow() {
+        while (this.#timeouts.front()?.t < new Date().getTime()) {
+            const item = this.#timeouts.dequeue();
+            this.#store.delete(item.k);
+        }
+
+        this.#nextSweep.id = null;
+        this.#nextSweep.t = null;
+        this.#reschedule();
+    }
+}

api/src/store/redis-ratelimit.js

@@ -0,0 +1,19 @@
+import { env } from "../config.js";
+
+let client, redis, redisLimiter;
+
+export const createStore = async (name) => {
+    if (!env.redisURL) return;
+
+    if (!client) {
+        redis = await import('redis');
+        redisLimiter = await import('rate-limit-redis');
+        client = redis.createClient({ url: env.redisURL });
+        await client.connect();
+    }
+
+    return new redisLimiter.default({
+        prefix: `RL${name}_`,
+        sendCommand: (...args) => client.sendCommand(args),
+    });
+}

api/src/store/redis-store.js

@@ -0,0 +1,64 @@
+import { commandOptions, createClient } from "redis";
+import { env } from "../config.js";
+import { Store } from "./base-store.js";
+
+export default class RedisStore extends Store {
+    #client = createClient({
+        url: env.redisURL,
+    });
+    #connected;
+
+    constructor(name) {
+        super(name);
+        this.#connected = this.#client.connect();
+    }
+
+    #keyOf(key) {
+        return this.id + '_' + key;
+    }
+
+    async _has(key) {
+        await this.#connected;
+
+        return this.#client.hExists(key);
+    }
+
+    async _get(key) {
+        await this.#connected;
+
+        const valueType = await this.#client.get(this.#keyOf(key) + '_t');
+        const value = await this.#client.get(
+            commandOptions({ returnBuffers: true }),
+            this.#keyOf(key)
+        );
+
+        if (!value) {
+            return null;
+        }
+
+        if (valueType === 'b')
+            return value;
+        else
+            return JSON.parse(value);
+    }
+
+    async _set(key, val, exp_sec = -1) {
+        await this.#connected;
+
+        const options = exp_sec > 0 ? { EX: exp_sec } : undefined;
+
+        if (val instanceof Buffer) {
+            await this.#client.set(
+                this.#keyOf(key) + '_t',
+                'b',
+                options
+            );
+        }
+
+        await this.#client.set(
+            this.#keyOf(key),
+            val,
+            options
+        );
+    }
+}

api/src/store/store.js

@@ -0,0 +1,10 @@
+import { env } from '../config.js';
+
+let _export;
+if (env.redisURL) {
+    _export = await import('./redis-store.js');
+} else {
+    _export = await import('./memory-store.js');
+}
+
+export default _export.default;

api/src/stream/internal-hls.js

@@ -16,15 +16,17 @@ function transformObject(streamInfo, hlsObject) {
 
     let fullUrl;
     if (getURL(hlsObject.uri)) {
-        fullUrl = hlsObject.uri;
+        fullUrl = new URL(hlsObject.uri);
     } else {
         fullUrl = new URL(hlsObject.uri, streamInfo.url);
     }
 
-    hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
+    if (fullUrl.hostname !== '127.0.0.1') {
+        hlsObject.uri = createInternalStream(fullUrl.toString(), streamInfo);
 
-    if (hlsObject.map) {
-        hlsObject.map = transformObject(streamInfo, hlsObject.map);
+        if (hlsObject.map) {
+            hlsObject.map = transformObject(streamInfo, hlsObject.map);
+        }
     }
 
     return hlsObject;
@@ -53,7 +55,7 @@ function transformMediaPlaylist(streamInfo, hlsPlaylist) {
 
 const HLS_MIME_TYPES = ["application/vnd.apple.mpegurl", "audio/mpegurl", "application/x-mpegURL"];
 
-export function isHlsRequest (req) {
+export function isHlsResponse (req) {
     return HLS_MIME_TYPES.includes(req.headers['content-type']);
 }