ggg

.env

@@ -0,0 +1,17 @@
+# Port dan URL utama untuk API
+API_PORT=7860
+API_URL=https://closure-ri-test.hf.space/
+
+# Konfigurasi JWT (Authentication)
+JWT_SECRET=9f4b3a2c1d8e7f6a5c4b3a2e1f8e7d6c5b4a3c2e1f7d8c9b0a1e2f3c4d5b6a7
+JWT_EXPIRY=120
+
+# Konfigurasi Rate Limit
+RATELIMIT_WINDOW=60
+RATELIMIT_MAX=20
+
+# Durasi maksimum video
+DURATION_LIMIT=10800
+
+# Redis Cache untuk kinerja
+API_REDIS_URL=redis://localhost:6379

Dockerfile

@@ -1,23 +1,31 @@
-# Gunakan image Docker resmi
-FROM docker:latest
-
-# Install curl dan docker-compose
-RUN apk add --no-cache curl docker-compose
-
-# Buat user non-root
-RUN adduser -D -u 1000 user
-
-# Buat direktori kerja dan berikan izin
-WORKDIR /home/user/app
-RUN mkdir -p /home/user/app && chown user:user /home/user/app
-
-# Switch ke user non-root
-USER user
-
-# Ambil docker-compose.yml dari raw URL
-RUN curl -L -o docker-compose.yml https://raw.githubusercontent.com/Lingz-ui/data-myBot/refs/heads/main/docker-compose.yml
-
-EXPOSE 7860
-
-# Jalankan Docker Compose saat container dimulai
-CMD ["docker-compose", "up", "-d"]
+# Gunakan base image Node.js
+FROM node:20
+
+# Install ffmpeg dan dependency tambahan
+RUN apt-get update && apt-get install -y \
+    ffmpeg \
+    curl \
+    wget \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install pnpm secara global
+RUN corepack enable && corepack prepare pnpm@latest --activate
+
+# Atur direktori kerja di dalam container
+WORKDIR /app
+
+# Salin file konfigurasi proyek (package.json, pnpm-lock.yaml, dll.)
+COPY package.json pnpm-lock.yaml ./
+
+# Install dependencies menggunakan pnpm
+RUN pnpm install --frozen-lockfile
+
+# Salin semua file proyek ke dalam container
+COPY . .
+
+# Ekspose port 7860
+EXPOSE 7860
+
+# Jalankan aplikasi
+CMD ["pnpm", "start"]

package.json

@@ -0,0 +1,52 @@
+{
+    "name": "@imput/cobalt-api",
+    "description": "save what you love",
+    "version": "10.4.4",
+    "author": "imput",
+    "exports": "./src/cobalt.js",
+    "type": "module",
+    "engines": {
+        "node": ">=18"
+    },
+    "scripts": {
+        "start": "node src/cobalt",
+        "test": "node src/util/test",
+        "token:youtube": "node src/util/generate-youtube-tokens",
+        "token:jwt": "node src/util/generate-jwt-secret"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/imputnet/cobalt.git"
+    },
+    "license": "AGPL-3.0",
+    "bugs": {
+        "url": "https://github.com/imputnet/cobalt/issues"
+    },
+    "homepage": "https://github.com/imputnet/cobalt#readme",
+    "dependencies": {
+        "@datastructures-js/priority-queue": "^6.3.1",
+        "@imput/psl": "^2.0.4",
+        "@imput/version-info": "workspace:^",
+        "content-disposition-header": "0.6.0",
+        "cors": "^2.8.5",
+        "dotenv": "^16.0.1",
+        "esbuild": "^0.14.51",
+        "express": "^4.21.2",
+        "express-rate-limit": "^7.4.1",
+        "ffmpeg-static": "^5.1.0",
+        "hls-parser": "^0.10.7",
+        "ipaddr.js": "2.2.0",
+        "nanoid": "^5.0.9",
+        "node-cache": "^5.1.2",
+        "set-cookie-parser": "2.6.0",
+        "undici": "^5.19.1",
+        "url-pattern": "1.0.3",
+        "youtubei.js": "^12.2.0",
+        "zod": "^3.23.8"
+    },
+    "optionalDependencies": {
+        "freebind": "^0.2.2",
+        "rate-limit-redis": "^4.2.0",
+        "redis": "^4.7.0"
+    }
+}

src/cobalt.js

@@ -0,0 +1,32 @@
+import "dotenv/config";
+
+import express from "express";
+import cluster from "node:cluster";
+
+import path from "path";
+import { fileURLToPath } from "url";
+
+import { env, isCluster } from "./config.js"
+import { Red } from "./misc/console-text.js";
+import { initCluster } from "./misc/cluster.js";
+
+const app = express();
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename).slice(0, -4);
+
+app.disable("x-powered-by");
+
+if (env.apiURL) {
+    const { runAPI } = await import("./core/api.js");
+
+    if (isCluster) {
+       await initCluster();
+    }
+
+    runAPI(express, app, __dirname, cluster.isPrimary);
+} else {
+    console.log(
+        Red("API_URL env variable is missing, cobalt api can't start.")
+    )
+}

src/config.js

@@ -0,0 +1,81 @@
+import { getVersion } from "@imput/version-info";
+import { services } from "./processing/service-config.js";
+import { supportsReusePort } from "./misc/cluster.js";
+
+const version = await getVersion();
+
+const disabledServices = process.env.DISABLED_SERVICES?.split(',') || [];
+const enabledServices = new Set(Object.keys(services).filter(e => {
+    if (!disabledServices.includes(e)) {
+        return e;
+    }
+}));
+
+const env = {
+    apiURL: process.env.API_URL || '',
+    apiPort: process.env.API_PORT || 9000,
+    tunnelPort: process.env.API_PORT || 9000,
+
+    listenAddress: process.env.API_LISTEN_ADDRESS,
+    freebindCIDR: process.platform === 'linux' && process.env.FREEBIND_CIDR,
+
+    corsWildcard: process.env.CORS_WILDCARD !== '0',
+    corsURL: process.env.CORS_URL,
+
+    cookiePath: process.env.COOKIE_PATH,
+
+    rateLimitWindow: (process.env.RATELIMIT_WINDOW && parseInt(process.env.RATELIMIT_WINDOW)) || 60,
+    rateLimitMax: (process.env.RATELIMIT_MAX && parseInt(process.env.RATELIMIT_MAX)) || 20,
+
+    durationLimit: (process.env.DURATION_LIMIT && parseInt(process.env.DURATION_LIMIT)) || 10800,
+    streamLifespan: (process.env.TUNNEL_LIFESPAN && parseInt(process.env.TUNNEL_LIFESPAN)) || 90,
+
+    processingPriority: process.platform !== 'win32'
+        && process.env.PROCESSING_PRIORITY
+        && parseInt(process.env.PROCESSING_PRIORITY),
+
+    externalProxy: process.env.API_EXTERNAL_PROXY,
+
+    turnstileSitekey: process.env.TURNSTILE_SITEKEY,
+    turnstileSecret: process.env.TURNSTILE_SECRET,
+    jwtSecret: process.env.JWT_SECRET,
+    jwtLifetime: process.env.JWT_EXPIRY || 120,
+
+    sessionEnabled: process.env.TURNSTILE_SITEKEY
+                        && process.env.TURNSTILE_SECRET
+                        && process.env.JWT_SECRET,
+
+    apiKeyURL: process.env.API_KEY_URL && new URL(process.env.API_KEY_URL),
+    authRequired: process.env.API_AUTH_REQUIRED === '1',
+    redisURL: process.env.API_REDIS_URL,
+    instanceCount: (process.env.API_INSTANCE_COUNT && parseInt(process.env.API_INSTANCE_COUNT)) || 1,
+    keyReloadInterval: 900,
+
+    enabledServices,
+}
+
+const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
+const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
+
+export const setTunnelPort = (port) => env.tunnelPort = port;
+export const isCluster = env.instanceCount > 1;
+
+if (env.sessionEnabled && env.jwtSecret.length < 16) {
+    throw new Error("JWT_SECRET env is too short (must be at least 16 characters long)");
+}
+
+if (env.instanceCount > 1 && !env.redisURL) {
+    throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
+} else if (env.instanceCount > 1 && !await supportsReusePort()) {
+    console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
+    console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
+    console.error('(or other OS that supports it). for more info, see `reusePort` option on');
+    console.error('https://nodejs.org/api/net.html#serverlistenoptions-callback');
+    throw new Error('SO_REUSEPORT is not supported');
+}
+
+export {
+    env,
+    genericUserAgent,
+    cobaltUserAgent,
+}

src/core/api.js

@@ -0,0 +1,394 @@
+import cors from "cors";
+import http from "node:http";
+import rateLimit from "express-rate-limit";
+import { setGlobalDispatcher, ProxyAgent } from "undici";
+import { getCommit, getBranch, getRemote, getVersion } from "@imput/version-info";
+
+import jwt from "../security/jwt.js";
+import stream from "../stream/stream.js";
+import match from "../processing/match.js";
+
+import { env, isCluster, setTunnelPort } from "../config.js";
+import { extract } from "../processing/url.js";
+import { Green, Bright, Cyan } from "../misc/console-text.js";
+import { hashHmac } from "../security/secrets.js";
+import { createStore } from "../store/redis-ratelimit.js";
+import { randomizeCiphers } from "../misc/randomize-ciphers.js";
+import { verifyTurnstileToken } from "../security/turnstile.js";
+import { friendlyServiceName } from "../processing/service-alias.js";
+import { verifyStream, getInternalStream } from "../stream/manage.js";
+import { createResponse, normalizeRequest, getIP } from "../processing/request.js";
+import * as APIKeys from "../security/api-keys.js";
+import * as Cookies from "../processing/cookie/manager.js";
+
+const git = {
+    branch: await getBranch(),
+    commit: await getCommit(),
+    remote: await getRemote(),
+}
+
+const version = await getVersion();
+
+const acceptRegex = /^application\/json(; charset=utf-8)?$/;
+
+const corsConfig = env.corsWildcard ? {} : {
+    origin: env.corsURL,
+    optionsSuccessStatus: 200
+}
+
+const fail = (res, code, context) => {
+    const { status, body } = createResponse("error", { code, context });
+    res.status(status).json(body);
+}
+
+export const runAPI = async (express, app, __dirname, isPrimary = true) => {
+    const startTime = new Date();
+    const startTimestamp = startTime.getTime();
+
+    const serverInfo = JSON.stringify({
+        cobalt: {
+            version: version,
+            url: env.apiURL,
+            startTime: `${startTimestamp}`,
+            durationLimit: env.durationLimit,
+            turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
+            services: [...env.enabledServices].map(e => {
+                return friendlyServiceName(e);
+            }),
+        },
+        git,
+    })
+
+    const handleRateExceeded = (_, res) => {
+        const { status, body } = createResponse("error", {
+            code: "error.api.rate_exceeded",
+            context: {
+                limit: env.rateLimitWindow
+            }
+        });
+        return res.status(status).json(body);
+    };
+
+    const keyGenerator = (req) => hashHmac(getIP(req), 'rate').toString('base64url');
+
+    const sessionLimiter = rateLimit({
+        windowMs: 60000,
+        limit: 10,
+        standardHeaders: 'draft-6',
+        legacyHeaders: false,
+        keyGenerator,
+        store: await createStore('session'),
+        handler: handleRateExceeded
+    });
+
+    const apiLimiter = rateLimit({
+        windowMs: env.rateLimitWindow * 1000,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
+        legacyHeaders: false,
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('api'),
+        handler: handleRateExceeded
+    })
+
+    const apiTunnelLimiter = rateLimit({
+        windowMs: env.rateLimitWindow * 1000,
+        limit: (req) => req.rateLimitMax || env.rateLimitMax,
+        standardHeaders: 'draft-6',
+        legacyHeaders: false,
+        keyGenerator: req => req.rateLimitKey || keyGenerator(req),
+        store: await createStore('tunnel'),
+        handler: (_, res) => {
+            return res.sendStatus(429)
+        }
+    })
+
+    app.set('trust proxy', ['loopback', 'uniquelocal']);
+
+    app.use('/', cors({
+        methods: ['GET', 'POST'],
+        exposedHeaders: [
+            'Ratelimit-Limit',
+            'Ratelimit-Policy',
+            'Ratelimit-Remaining',
+            'Ratelimit-Reset'
+        ],
+        ...corsConfig,
+    }));
+
+    app.post('/', (req, res, next) => {
+        if (!acceptRegex.test(req.header('Accept'))) {
+            return fail(res, "error.api.header.accept");
+        }
+        if (!acceptRegex.test(req.header('Content-Type'))) {
+            return fail(res, "error.api.header.content_type");
+        }
+        next();
+    });
+
+    app.post('/', (req, res, next) => {
+        if (!env.apiKeyURL) {
+            return next();
+        }
+
+        const { success, error } = APIKeys.validateAuthorization(req);
+        if (!success) {
+            // We call next() here if either if:
+            // a) we have user sessions enabled, meaning the request
+            //    will still need a Bearer token to not be rejected, or
+            // b) we do not require the user to be authenticated, and
+            //    so they can just make the request with the regular
+            //    rate limit configuration;
+            // otherwise, we reject the request.
+            if (
+                (env.sessionEnabled || !env.authRequired)
+                && ['missing', 'not_api_key'].includes(error)
+            ) {
+                return next();
+            }
+
+            return fail(res, `error.api.auth.key.${error}`);
+        }
+
+        return next();
+    });
+
+    app.post('/', (req, res, next) => {
+        if (!env.sessionEnabled || req.rateLimitKey) {
+            return next();
+        }
+
+        try {
+            const authorization = req.header("Authorization");
+            if (!authorization) {
+                return fail(res, "error.api.auth.jwt.missing");
+            }
+
+            if (authorization.length >= 256) {
+                return fail(res, "error.api.auth.jwt.invalid");
+            }
+
+            const [ type, token, ...rest ] = authorization.split(" ");
+            if (!token || type.toLowerCase() !== 'bearer' || rest.length) {
+                return fail(res, "error.api.auth.jwt.invalid");
+            }
+
+            if (!jwt.verify(token)) {
+                return fail(res, "error.api.auth.jwt.invalid");
+            }
+
+            req.rateLimitKey = hashHmac(token, 'rate');
+        } catch {
+            return fail(res, "error.api.generic");
+        }
+        next();
+    });
+
+    app.post('/', apiLimiter);
+    app.use('/', express.json({ limit: 1024 }));
+
+    app.use('/', (err, _, res, next) => {
+        if (err) {
+            const { status, body } = createResponse("error", {
+                code: "error.api.invalid_body",
+            });
+            return res.status(status).json(body);
+        }
+
+        next();
+    });
+
+    app.post("/session", sessionLimiter, async (req, res) => {
+        if (!env.sessionEnabled) {
+            return fail(res, "error.api.auth.not_configured")
+        }
+
+        const turnstileResponse = req.header("cf-turnstile-response");
+
+        if (!turnstileResponse) {
+            return fail(res, "error.api.auth.turnstile.missing");
+        }
+
+        const turnstileResult = await verifyTurnstileToken(
+            turnstileResponse,
+            req.ip
+        );
+
+        if (!turnstileResult) {
+            return fail(res, "error.api.auth.turnstile.invalid");
+        }
+
+        try {
+            res.json(jwt.generate());
+        } catch {
+            return fail(res, "error.api.generic");
+        }
+    });
+
+    app.post('/', async (req, res) => {
+        const request = req.body;
+
+        if (!request.url) {
+            return fail(res, "error.api.link.missing");
+        }
+
+        const { success, data: normalizedRequest } = await normalizeRequest(request);
+        if (!success) {
+            return fail(res, "error.api.invalid_body");
+        }
+
+        const parsed = extract(normalizedRequest.url);
+
+        if (!parsed) {
+            return fail(res, "error.api.link.invalid");
+        }
+        if ("error" in parsed) {
+            let context;
+            if (parsed?.context) {
+                context = parsed.context;
+            }
+            return fail(res, `error.api.${parsed.error}`, context);
+        }
+
+        try {
+            const result = await match({
+                host: parsed.host,
+                patternMatch: parsed.patternMatch,
+                params: normalizedRequest,
+            });
+
+            res.status(result.status).json(result.body);
+        } catch {
+            fail(res, "error.api.generic");
+        }
+    })
+
+    app.get('/tunnel', apiTunnelLimiter, async (req, res) => {
+        const id = String(req.query.id);
+        const exp = String(req.query.exp);
+        const sig = String(req.query.sig);
+        const sec = String(req.query.sec);
+        const iv = String(req.query.iv);
+
+        const checkQueries = id && exp && sig && sec && iv;
+        const checkBaseLength = id.length === 21 && exp.length === 13;
+        const checkSafeLength = sig.length === 43 && sec.length === 43 && iv.length === 22;
+
+        if (!checkQueries || !checkBaseLength || !checkSafeLength) {
+            return res.status(400).end();
+        }
+
+        if (req.query.p) {
+            return res.status(200).end();
+        }
+
+        const streamInfo = await verifyStream(id, sig, exp, sec, iv);
+        if (!streamInfo?.service) {
+            return res.status(streamInfo.status).end();
+        }
+
+        if (streamInfo.type === 'proxy') {
+            streamInfo.range = req.headers['range'];
+        }
+
+        return stream(res, streamInfo);
+    })
+
+    const itunnelHandler = (req, res) => {
+        if (!req.ip.endsWith('127.0.0.1')) {
+            return res.sendStatus(403);
+        }
+
+        if (String(req.query.id).length !== 21) {
+            return res.sendStatus(400);
+        }
+
+        const streamInfo = getInternalStream(req.query.id);
+        if (!streamInfo) {
+            return res.sendStatus(404);
+        }
+
+        streamInfo.headers = new Map([
+            ...(streamInfo.headers || []),
+            ...Object.entries(req.headers)
+        ]);
+
+        return stream(res, { type: 'internal', ...streamInfo });
+    };
+
+    app.get('/itunnel', itunnelHandler);
+
+    app.get('/', (_, res) => {
+        res.type('json');
+        res.status(200).send(serverInfo);
+    })
+
+    app.get('/favicon.ico', (req, res) => {
+        res.status(404).end();
+    })
+
+    app.get('/*', (req, res) => {
+        res.redirect('/');
+    })
+
+    // handle all express errors
+    app.use((_, __, res, ___) => {
+        return fail(res, "error.api.generic");
+    })
+
+    randomizeCiphers();
+    setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
+
+    if (env.externalProxy) {
+        if (env.freebindCIDR) {
+            throw new Error('Freebind is not available when external proxy is enabled')
+        }
+
+        setGlobalDispatcher(new ProxyAgent(env.externalProxy))
+    }
+
+    http.createServer(app).listen({
+        port: env.apiPort,
+        host: env.listenAddress,
+        reusePort: env.instanceCount > 1 || undefined
+    }, () => {
+        if (isPrimary) {
+            console.log(`\n` +
+                Bright(Cyan("cobalt ")) + Bright("API ^ω⁠^") + "\n" +
+
+                "~~~~~~\n" +
+                Bright("version: ") + version + "\n" +
+                Bright("commit: ") + git.commit + "\n" +
+                Bright("branch: ") + git.branch + "\n" +
+                Bright("remote: ") + git.remote + "\n" +
+                Bright("start time: ") + startTime.toUTCString() + "\n" +
+                "~~~~~~\n" +
+
+                Bright("url: ") + Bright(Cyan(env.apiURL)) + "\n" +
+                Bright("port: ") + env.apiPort + "\n"
+            );
+        }
+
+        if (env.apiKeyURL) {
+            APIKeys.setup(env.apiKeyURL);
+        }
+
+        if (env.cookiePath) {
+            Cookies.setup(env.cookiePath);
+        }
+    });
+
+    if (isCluster) {
+        const istreamer = express();
+        istreamer.get('/itunnel', itunnelHandler);
+        const server = istreamer.listen({
+            port: 0,
+            host: '127.0.0.1',
+            exclusive: true
+        }, () => {
+            const { port } = server.address();
+            console.log(`${Green('[✓]')} cobalt sub-instance running on 127.0.0.1:${port}`);
+            setTunnelPort(port);
+        });
+    }
+}

src/misc/cluster.js

@@ -0,0 +1,71 @@
+import cluster from "node:cluster";
+import net from "node:net";
+import { syncSecrets } from "../security/secrets.js";
+import { env, isCluster } from "../config.js";
+
+export { isPrimary, isWorker } from "node:cluster";
+
+export const supportsReusePort = async () => {
+    try {
+        await new Promise((resolve, reject) => {
+            const server = net.createServer().listen({ port: 0, reusePort: true });
+            server.on('listening', () => server.close(resolve));
+            server.on('error', (err) => (server.close(), reject(err)));
+        });
+
+        return true;
+    } catch {
+        return false;
+    }
+}
+
+export const initCluster = async () => {
+    if (cluster.isPrimary) {
+        for (let i = 1; i < env.instanceCount; ++i) {
+            cluster.fork();
+        }
+    }
+
+    await syncSecrets();
+}
+
+export const broadcast = (message) => {
+    if (!isCluster || !cluster.isPrimary || !cluster.workers) {
+        return;
+    }
+
+    for (const worker of Object.values(cluster.workers)) {
+        worker.send(message);
+    }
+}
+
+export const send = (message) => {
+    if (!isCluster) {
+        return;
+    }
+
+    if (cluster.isPrimary) {
+        return broadcast(message);
+    } else {
+        return process.send(message);
+    }
+}
+
+export const waitFor = (key) => {
+    return new Promise(resolve => {
+        const listener = (message) => {
+            if (key in message) {
+                process.off('message', listener);
+                return resolve(message);
+            }
+        }
+
+        process.on('message', listener);
+    });
+}
+
+export const mainOnMessage = (cb) => {
+    for (const worker of Object.values(cluster.workers)) {
+        worker.on('message', cb);
+    }
+}

src/misc/console-text.js

@@ -0,0 +1,36 @@
+const ANSI = {
+    RESET: "\x1b[0m",
+    BRIGHT: "\x1b[1m",
+    RED: "\x1b[31m",
+    GREEN: "\x1b[32m",
+    CYAN: "\x1b[36m",
+    YELLOW: "\x1b[93m"
+}
+
+function wrap(color, text) {
+    if (!ANSI[color.toUpperCase()]) {
+        throw "invalid color";
+    }
+
+    return ANSI[color.toUpperCase()] + text + ANSI.RESET;
+}
+
+export function Bright(text) {
+    return wrap('bright', text);
+}
+
+export function Red(text) {
+    return wrap('red', text);
+}
+
+export function Green(text) {
+    return wrap('green', text);
+}
+
+export function Cyan(text) {
+    return wrap('cyan', text);
+}
+
+export function Yellow(text) {
+    return wrap('yellow', text);
+}

src/misc/crypto.js

@@ -0,0 +1,19 @@
+import { createCipheriv, createDecipheriv } from "crypto";
+
+const algorithm = "aes256";
+
+export function encryptStream(plaintext, iv, secret) {
+    const buff = Buffer.from(JSON.stringify(plaintext));
+    const key = Buffer.from(secret, "base64url");
+    const cipher = createCipheriv(algorithm, key, Buffer.from(iv, "base64url"));
+
+    return Buffer.concat([ cipher.update(buff), cipher.final() ])
+}
+
+export function decryptStream(ciphertext, iv, secret) {
+    const buff = Buffer.from(ciphertext);
+    const key = Buffer.from(secret, "base64url");
+    const decipher = createDecipheriv(algorithm, key, Buffer.from(iv, "base64url"));
+
+    return Buffer.concat([ decipher.update(buff), decipher.final() ])
+}

src/misc/load-from-fs.js

@@ -0,0 +1,20 @@
+import * as fs from "fs";
+import { join, dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = join(
+    dirname(fileURLToPath(import.meta.url)),
+    '../../'
+);
+
+export function loadFile(path) {
+    return fs.readFileSync(join(root, path), 'utf-8')
+}
+
+export function loadJSON(path) {
+    try {
+        return JSON.parse(loadFile(path))
+    } catch {
+        return false
+    }
+}

src/misc/randomize-ciphers.js

@@ -0,0 +1,28 @@
+import tls from 'node:tls';
+import { randomBytes } from 'node:crypto';
+
+const ORIGINAL_CIPHERS = tls.DEFAULT_CIPHERS;
+
+// How many ciphers from the top of the list to shuffle.
+// The remaining ciphers are left in the original order.
+const TOP_N_SHUFFLE = 8;
+
+// Modified variation of https://stackoverflow.com/a/12646864
+const shuffleArray = (array) => {
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = randomBytes(4).readUint32LE() % array.length;
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+
+    return array;
+}
+
+export const randomizeCiphers = () => {
+    do {
+        const cipherList = ORIGINAL_CIPHERS.split(':');
+        const shuffled = shuffleArray(cipherList.slice(0, TOP_N_SHUFFLE));
+        const retained = cipherList.slice(TOP_N_SHUFFLE);
+
+        tls.DEFAULT_CIPHERS = [ ...shuffled, ...retained ].join(':');
+    } while (tls.DEFAULT_CIPHERS === ORIGINAL_CIPHERS);
+}

src/misc/run-test.js

@@ -0,0 +1,44 @@
+import { normalizeRequest } from "../processing/request.js";
+import match from "../processing/match.js";
+import { extract } from "../processing/url.js";
+
+export async function runTest(url, params, expect) {
+    const { success, data: normalized } = await normalizeRequest({ url, ...params });
+    if (!success) {
+        throw "invalid request";
+    }
+
+    const parsed = extract(normalized.url);
+    if (parsed === null) {
+        throw `invalid url: ${normalized.url}`;
+    }
+
+    const result = await match({
+        host: parsed.host,
+        patternMatch: parsed.patternMatch,
+        params: normalized,
+    });
+
+    let error = [];
+    if (expect.status !== result.body.status) {
+        const detail = `${expect.status} (expected) != ${result.body.status} (actual)`;
+        error.push(`status mismatch: ${detail}`);
+    }
+
+    if (expect.code !== result.status) {
+        const detail = `${expect.code} (expected) != ${result.status} (actual)`;
+        error.push(`status code mismatch: ${detail}`);
+    }
+
+    if (error.length) {
+        if (result.body.text) {
+            error.push(`error message: ${result.body.text}`);
+        }
+
+        throw error.join('\n');
+    }
+
+    if (result.body.status === 'tunnel') {
+        // TODO: stream testing
+    }
+}

src/misc/utils.js

@@ -0,0 +1,31 @@
+export function getRedirectingURL(url) {
+    return fetch(url, { redirect: 'manual' }).then((r) => {
+        if ([301, 302, 303].includes(r.status) && r.headers.has('location'))
+            return r.headers.get('location');
+    }).catch(() => null);
+}
+
+export function merge(a, b) {
+    for (const k of Object.keys(b)) {
+        if (Array.isArray(b[k])) {
+            a[k] = [...(a[k] ?? []), ...b[k]];
+        } else if (typeof b[k] === 'object') {
+            a[k] = merge(a[k], b[k]);
+        } else {
+            a[k] = b[k];
+        }
+    }
+
+    return a;
+}
+
+export function splitFilenameExtension(filename) {
+    const parts = filename.split('.');
+    const ext = parts.pop();
+
+    if (!parts.length) {
+        return [ ext, "" ]
+    } else {
+        return [ parts.join('.'), ext ]
+    }
+}

src/processing/cookie/cookie.js

@@ -0,0 +1,48 @@
+import { strict as assert } from 'node:assert';
+
+export default class Cookie {
+    constructor(input) {
+        assert(typeof input === 'object');
+        this._values = {};
+
+        for (const [ k, v ] of Object.entries(input))
+            this.set(k, v);
+    }
+
+    set(key, value) {
+        const old = this._values[key];
+        if (old === value)
+            return false;
+
+        this._values[key] = value;
+        return true;
+    }
+
+    unset(keys) {
+        for (const key of keys) delete this._values[key]
+    }
+
+    static fromString(str) {
+        const obj = {};
+
+        str.split('; ').forEach(cookie => {
+            const key = cookie.split('=')[0];
+            const value = cookie.split('=').splice(1).join('=');
+            obj[key] = value
+        })
+
+        return new Cookie(obj)
+    }
+
+    toString() {
+        return Object.entries(this._values).map(([ name, value ]) => `${name}=${value}`).join('; ')
+    }
+
+    toJSON() {
+        return this.toString()
+    }
+
+    values() {
+        return Object.freeze({ ...this._values })
+    }
+}

src/processing/cookie/manager.js

@@ -0,0 +1,156 @@
+import Cookie from './cookie.js';
+
+import { readFile, writeFile } from 'fs/promises';
+import { Red, Green, Yellow } from '../../misc/console-text.js';
+import { parse as parseSetCookie, splitCookiesString } from 'set-cookie-parser';
+import * as cluster from '../../misc/cluster.js';
+import { isCluster } from '../../config.js';
+
+const WRITE_INTERVAL = 60000;
+const VALID_SERVICES = new Set([
+    'instagram',
+    'instagram_bearer',
+    'reddit',
+    'twitter',
+    'youtube_oauth'
+]);
+
+const invalidCookies = {};
+let cookies = {}, dirty = false, intervalId;
+
+function writeChanges(cookiePath) {
+    if (!dirty) return;
+    dirty = false;
+
+    const cookieData = JSON.stringify({ ...cookies, ...invalidCookies }, null, 4);
+    writeFile(cookiePath, cookieData).catch((e) => {
+        console.warn(`${Yellow('[!]')} failed writing updated cookies to storage`);
+        console.warn(e);
+        clearInterval(intervalId);
+        intervalId = null;
+    })
+}
+
+const setupMain = async (cookiePath) => {
+    try {
+        cookies = await readFile(cookiePath, 'utf8');
+        cookies = JSON.parse(cookies);
+        for (const serviceName in cookies) {
+            if (!VALID_SERVICES.has(serviceName)) {
+                console.warn(`${Yellow('[!]')} ignoring unknown service in cookie file: ${serviceName}`);
+            } else if (!Array.isArray(cookies[serviceName])) {
+                console.warn(`${Yellow('[!]')} ${serviceName} in cookies file is not an array, ignoring it`);
+            } else if (cookies[serviceName].some(c => typeof c !== 'string')) {
+                console.warn(`${Yellow('[!]')} some cookie for ${serviceName} contains non-string value in cookies file`);
+            } else continue;
+
+            invalidCookies[serviceName] = cookies[serviceName];
+            delete cookies[serviceName];
+        }
+
+        if (!intervalId) {
+            intervalId = setInterval(() => writeChanges(cookiePath), WRITE_INTERVAL);
+        }
+
+        cluster.broadcast({ cookies });
+
+        console.log(`${Green('[✓]')} cookies loaded successfully!`);
+    } catch (e) {
+        console.error(`${Yellow('[!]')} failed to load cookies.`);
+        console.error('error:', e);
+    }
+}
+
+const setupWorker = async () => {
+    cookies = (await cluster.waitFor('cookies')).cookies;
+}
+
+export const loadFromFile = async (path) => {
+    if (cluster.isPrimary) {
+        await setupMain(path);
+    } else if (cluster.isWorker) {
+        await setupWorker();
+    }
+
+    dirty = false;
+}
+
+export const setup = async (path) => {
+    await loadFromFile(path);
+
+    if (isCluster) {
+        const messageHandler = (message) => {
+            if ('cookieUpdate' in message) {
+                const { cookieUpdate } = message;
+
+                if (cluster.isPrimary) {
+                    dirty = true;
+                    cluster.broadcast({ cookieUpdate });
+                }
+
+                const { service, idx, cookie } = cookieUpdate;
+                cookies[service][idx] = cookie;
+            }
+        }
+
+        if (cluster.isPrimary) {
+            cluster.mainOnMessage(messageHandler);
+        } else {
+            process.on('message', messageHandler);
+        }
+    }
+}
+
+export function getCookie(service) {
+    if (!VALID_SERVICES.has(service)) {
+        console.error(
+            `${Red('[!]')} ${service} not in allowed services list for cookies.`
+            + ' if adding a new cookie type, include it there.'
+        );
+        return;
+    }
+
+    if (!cookies[service] || !cookies[service].length) return;
+
+    const idx = Math.floor(Math.random() * cookies[service].length);
+
+    const cookie = cookies[service][idx];
+    if (typeof cookie === 'string') {
+        cookies[service][idx] = Cookie.fromString(cookie);
+    }
+
+    cookies[service][idx].meta = { service, idx };
+    return cookies[service][idx];
+}
+
+export function updateCookieValues(cookie, values) {
+    let changed = false;
+
+    for (const [ key, value ] of Object.entries(values)) {
+        changed = cookie.set(key, value) || changed;
+    }
+
+    if (changed && cookie.meta) {
+        dirty = true;
+        if (isCluster) {
+            const message = { cookieUpdate: { ...cookie.meta, cookie } };
+            cluster.send(message);
+        }
+    }
+
+    return changed;
+}
+
+export function updateCookie(cookie, headers) {
+    if (!cookie) return;
+
+    const parsed = parseSetCookie(
+        splitCookiesString(headers.get('set-cookie')),
+        { decodeValues: false }
+    ), values = {}
+
+    cookie.unset(parsed.filter(c => c.expires < new Date()).map(c => c.name));
+    parsed.filter(c => !c.expires || c.expires > new Date()).forEach(c => values[c.name] = c.value);
+
+    updateCookieValues(cookie, values);
+}

src/processing/create-filename.js

@@ -0,0 +1,70 @@
+const illegalCharacters = ['}', '{', '%', '>', '<', '^', ';', ':', '`', '$', '"', "@", '=', '?', '|', '*'];
+
+const sanitizeString = (string) => {
+    for (const i in illegalCharacters) {
+        string = string.replaceAll("/", "_").replaceAll("\\", "_")
+                       .replaceAll(illegalCharacters[i], '')
+    }
+    return string;
+}
+
+export default (f, style, isAudioOnly, isAudioMuted) => {
+    let filename = '';
+
+    let infoBase = [f.service, f.id];
+    let classicTags = [...infoBase];
+    let basicTags = [];
+
+    let title = sanitizeString(f.title);
+
+    if (f.author) {
+        title += ` - ${sanitizeString(f.author)}`;
+    }
+
+    if (f.resolution) {
+        classicTags.push(f.resolution);
+    }
+
+    if (f.qualityLabel) {
+        basicTags.push(f.qualityLabel);
+    }
+
+    if (f.youtubeFormat) {
+        classicTags.push(f.youtubeFormat);
+        basicTags.push(f.youtubeFormat);
+    }
+
+    if (isAudioMuted) {
+        classicTags.push("mute");
+        basicTags.push("mute");
+    } else if (f.youtubeDubName) {
+        classicTags.push(f.youtubeDubName);
+        basicTags.push(f.youtubeDubName);
+    }
+
+    switch (style) {
+        default:
+        case "classic":
+            if (isAudioOnly) {
+                if (f.youtubeDubName) {
+                    infoBase.push(f.youtubeDubName);
+                }
+                return `${infoBase.join("_")}_audio`;
+            }
+            filename = classicTags.join("_");
+            break;
+        case "basic":
+            if (isAudioOnly) return title;
+            filename = `${title} (${basicTags.join(", ")})`;
+            break;
+        case "pretty":
+            if (isAudioOnly) return `${title} (${infoBase[0]})`;
+            filename = `${title} (${[...basicTags, infoBase[0]].join(", ")})`;
+            break;
+        case "nerdy":
+            if (isAudioOnly) return `${title} (${infoBase.join(", ")})`;
+            filename = `${title} (${basicTags.concat(infoBase).join(", ")})`;
+            break;
+    }
+    return `${filename}.${f.extension}`;
+}

src/processing/match-action.js

@@ -0,0 +1,215 @@
+import createFilename from "./create-filename.js";
+
+import { createResponse } from "./request.js";
+import { audioIgnore } from "./service-config.js";
+import { createStream } from "../stream/manage.js";
+import { splitFilenameExtension } from "../misc/utils.js";
+
+export default function({ r, host, audioFormat, isAudioOnly, isAudioMuted, disableMetadata, filenameStyle, twitterGif, requestIP, audioBitrate, alwaysProxy }) {
+    let action,
+        responseType = "tunnel",
+        defaultParams = {
+            url: r.urls,
+            headers: r.headers,
+            service: host,
+            filename: r.filenameAttributes ?
+                    createFilename(r.filenameAttributes, filenameStyle, isAudioOnly, isAudioMuted) : r.filename,
+            fileMetadata: !disableMetadata ? r.fileMetadata : false,
+            requestIP
+        },
+        params = {};
+
+    if (r.isPhoto) action = "photo";
+    else if (r.picker) action = "picker"
+    else if (r.isGif && twitterGif) action = "gif";
+    else if (isAudioOnly) action = "audio";
+    else if (isAudioMuted) action = "muteVideo";
+    else if (r.isHLS) action = "hls";
+    else action = "video";
+
+    if (action === "picker" || action === "audio") {
+        if (!r.filenameAttributes) defaultParams.filename = r.audioFilename;
+        defaultParams.audioFormat = audioFormat;
+    }
+
+    if (action === "muteVideo" && isAudioMuted && !r.filenameAttributes) {
+        const [ name, ext ] = splitFilenameExtension(r.filename);
+        defaultParams.filename = `${name}_mute.${ext}`;
+    } else if (action === "gif") {
+        const [ name ] = splitFilenameExtension(r.filename);
+        defaultParams.filename = `${name}.gif`;
+    }
+
+    switch (action) {
+        default:
+            return createResponse("error", {
+                code: "error.api.fetch.empty"
+            });
+
+        case "photo":
+            responseType = "redirect";
+            break;
+
+        case "gif":
+            params = { type: "gif" };
+            break;
+
+        case "hls":
+            params = {
+                type: Array.isArray(r.urls) ? "merge" : "remux",
+                isHLS: true,
+            }
+            break;
+
+        case "muteVideo":
+            let muteType = "mute";
+            if (Array.isArray(r.urls) && !r.isHLS) {
+                muteType = "proxy";
+            }
+            params = {
+                type: muteType,
+                url: Array.isArray(r.urls) ? r.urls[0] : r.urls
+            }
+            if (host === "reddit" && r.typeId === "redirect") {
+                responseType = "redirect";
+            }
+            break;
+
+        case "picker":
+            responseType = "picker";
+            switch (host) {
+                case "instagram":
+                case "twitter":
+                case "snapchat":
+                case "bsky":
+                    params = { picker: r.picker };
+                    break;
+
+                case "tiktok":
+                    let audioStreamType = "audio";
+                    if (r.bestAudio === "mp3" && audioFormat === "best") {
+                        audioFormat = "mp3";
+                        audioStreamType = "proxy"
+                    }
+                    params = {
+                        picker: r.picker,
+                        url: createStream({
+                            service: "tiktok",
+                            type: audioStreamType,
+                            url: r.urls,
+                            headers: r.headers,
+                            filename: `${r.audioFilename}.${audioFormat}`,
+                            isAudioOnly: true,
+                            audioFormat,
+                        })
+                    }
+                    break;
+            }
+            break;
+
+        case "video":
+            switch (host) {
+                case "bilibili":
+                    params = { type: "merge" };
+                    break;
+
+                case "youtube":
+                    params = { type: r.type };
+                    break;
+
+                case "reddit":
+                    responseType = r.typeId;
+                    params = { type: r.type };
+                    break;
+
+                case "vimeo":
+                    if (Array.isArray(r.urls)) {
+                        params = { type: "merge" }
+                    } else {
+                        responseType = "redirect";
+                    }
+                    break;
+
+                case "twitter":
+                    if (r.type === "remux") {
+                        params = { type: r.type };
+                    } else {
+                        responseType = "redirect";
+                    }
+                    break;
+
+                case "ok":
+                case "vk":
+                case "tiktok":
+                    params = { type: "proxy" };
+                    break;
+
+                case "facebook":
+                case "instagram":
+                case "tumblr":
+                case "pinterest":
+                case "streamable":
+                case "snapchat":
+                case "loom":
+                case "twitch":
+                    responseType = "redirect";
+                    break;
+            }
+            break;
+
+        case "audio":
+            if (audioIgnore.includes(host) || (host === "reddit" && r.typeId === "redirect")) {
+                return createResponse("error", {
+                    code: "error.api.service.audio_not_supported"
+                })
+            }
+
+            let processType = "audio";
+            let copy = false;
+
+            if (audioFormat === "best") {
+                const serviceBestAudio = r.bestAudio;
+
+                if (serviceBestAudio) {
+                    audioFormat = serviceBestAudio;
+                    processType = "proxy";
+
+                    if (host === "soundcloud") {
+                        processType = "audio";
+                        copy = true;
+                    }
+                } else {
+                    audioFormat = "m4a";
+                    copy = true;
+                }
+            }
+
+            if (r.isHLS || host === "vimeo") {
+                copy = false;
+                processType = "audio";
+            }
+
+            params = {
+                type: processType,
+                url: Array.isArray(r.urls) ? r.urls[1] : r.urls,
+
+                audioBitrate,
+                audioCopy: copy,
+                audioFormat,
+
+                isHLS: r.isHLS,
+            }
+            break;
+    }
+
+    if (defaultParams.filename && (action === "picker" || action === "audio")) {
+        defaultParams.filename += `.${audioFormat}`;
+    }
+
+    if (alwaysProxy && responseType === "redirect") {
+        responseType = "tunnel";
+        params.type = "proxy";
+    }
+
+    return createResponse(responseType, {...defaultParams, ...params})
+}

src/processing/match.js

@@ -0,0 +1,306 @@
+import { strict as assert } from "node:assert";
+
+import { env } from "../config.js";
+import { createResponse } from "../processing/request.js";
+
+import { testers } from "./service-patterns.js";
+import matchAction from "./match-action.js";
+
+import { friendlyServiceName } from "./service-alias.js";
+
+import bilibili from "./services/bilibili.js";
+import reddit from "./services/reddit.js";
+import twitter from "./services/twitter.js";
+import youtube from "./services/youtube.js";
+import vk from "./services/vk.js";
+import ok from "./services/ok.js";
+import tiktok from "./services/tiktok.js";
+import tumblr from "./services/tumblr.js";
+import vimeo from "./services/vimeo.js";
+import soundcloud from "./services/soundcloud.js";
+import instagram from "./services/instagram.js";
+import pinterest from "./services/pinterest.js";
+import streamable from "./services/streamable.js";
+import twitch from "./services/twitch.js";
+import rutube from "./services/rutube.js";
+import dailymotion from "./services/dailymotion.js";
+import snapchat from "./services/snapchat.js";
+import loom from "./services/loom.js";
+import facebook from "./services/facebook.js";
+import bluesky from "./services/bluesky.js";
+
+let freebind;
+
+export default async function({ host, patternMatch, params }) {
+    const { url } = params;
+    assert(url instanceof URL);
+    let dispatcher, requestIP;
+
+    if (env.freebindCIDR) {
+        if (!freebind) {
+            freebind = await import('freebind');
+        }
+
+        requestIP = freebind.ip.random(env.freebindCIDR);
+        dispatcher = freebind.dispatcherFromIP(requestIP, { strict: false });
+    }
+
+    try {
+        let r,
+            isAudioOnly = params.downloadMode === "audio",
+            isAudioMuted = params.downloadMode === "mute";
+
+        if (!testers[host]) {
+            return createResponse("error", {
+                code: "error.api.service.unsupported"
+            });
+        }
+        if (!(testers[host](patternMatch))) {
+            return createResponse("error", {
+                code: "error.api.link.unsupported",
+                context: {
+                    service: friendlyServiceName(host),
+                }
+            });
+        }
+
+        switch (host) {
+            case "twitter":
+                r = await twitter({
+                    id: patternMatch.id,
+                    index: patternMatch.index - 1,
+                    toGif: !!params.twitterGif,
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                });
+                break;
+
+            case "vk":
+                r = await vk({
+                    ownerId: patternMatch.ownerId,
+                    videoId: patternMatch.videoId,
+                    accessKey: patternMatch.accessKey,
+                    quality: params.videoQuality
+                });
+                break;
+
+            case "ok":
+                r = await ok({
+                    id: patternMatch.id,
+                    quality: params.videoQuality
+                });
+                break;
+
+            case "bilibili":
+                r = await bilibili(patternMatch);
+                break;
+
+            case "youtube":
+                let fetchInfo = {
+                    dispatcher,
+                    id: patternMatch.id.slice(0, 11),
+                    quality: params.videoQuality,
+                    format: params.youtubeVideoCodec,
+                    isAudioOnly,
+                    isAudioMuted,
+                    dubLang: params.youtubeDubLang,
+                    youtubeHLS: params.youtubeHLS,
+                }
+
+                if (url.hostname === "music.youtube.com" || isAudioOnly) {
+                    fetchInfo.quality = "max";
+                    fetchInfo.format = "vp9";
+                    fetchInfo.isAudioOnly = true;
+                    fetchInfo.isAudioMuted = false;
+                }
+
+                r = await youtube(fetchInfo);
+                break;
+
+            case "reddit":
+                r = await reddit({
+                    sub: patternMatch.sub,
+                    id: patternMatch.id,
+                    user: patternMatch.user
+                });
+                break;
+
+            case "tiktok":
+                r = await tiktok({
+                    postId: patternMatch.postId,
+                    shortLink: patternMatch.shortLink,
+                    fullAudio: params.tiktokFullAudio,
+                    isAudioOnly,
+                    h265: params.tiktokH265,
+                    alwaysProxy: params.alwaysProxy,
+                });
+                break;
+
+            case "tumblr":
+                r = await tumblr({
+                    id: patternMatch.id,
+                    user: patternMatch.user,
+                    url
+                });
+                break;
+
+            case "vimeo":
+                r = await vimeo({
+                    id: patternMatch.id.slice(0, 11),
+                    password: patternMatch.password,
+                    quality: params.videoQuality,
+                    isAudioOnly,
+                });
+                break;
+
+            case "soundcloud":
+                isAudioOnly = true;
+                isAudioMuted = false;
+                r = await soundcloud({
+                    url,
+                    author: patternMatch.author,
+                    song: patternMatch.song,
+                    format: params.audioFormat,
+                    shortLink: patternMatch.shortLink || false,
+                    accessKey: patternMatch.accessKey || false
+                });
+                break;
+
+            case "instagram":
+                r = await instagram({
+                    ...patternMatch,
+                    quality: params.videoQuality,
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                })
+                break;
+
+            case "pinterest":
+                r = await pinterest({
+                    id: patternMatch.id,
+                    shortLink: patternMatch.shortLink || false
+                });
+                break;
+
+            case "streamable":
+                r = await streamable({
+                    id: patternMatch.id,
+                    quality: params.videoQuality,
+                    isAudioOnly,
+                });
+                break;
+
+            case "twitch":
+                r = await twitch({
+                    clipId: patternMatch.clip || false,
+                    quality: params.videoQuality,
+                    isAudioOnly,
+                });
+                break;
+
+            case "rutube":
+                r = await rutube({
+                    id: patternMatch.id,
+                    yappyId: patternMatch.yappyId,
+                    key: patternMatch.key,
+                    quality: params.videoQuality,
+                    isAudioOnly,
+                });
+                break;
+
+            case "dailymotion":
+                r = await dailymotion(patternMatch);
+                break;
+
+            case "snapchat":
+                r = await snapchat({
+                    ...patternMatch,
+                    alwaysProxy: params.alwaysProxy,
+                });
+                break;
+
+            case "loom":
+                r = await loom({
+                    id: patternMatch.id
+                });
+                break;
+
+            case "facebook":
+                r = await facebook({
+                    ...patternMatch
+                });
+                break;
+
+            case "bsky":
+                r = await bluesky({
+                    ...patternMatch,
+                    alwaysProxy: params.alwaysProxy,
+                    dispatcher
+                });
+                break;
+
+            default:
+                return createResponse("error", {
+                    code: "error.api.service.unsupported"
+                });
+        }
+
+        if (r.isAudioOnly) {
+            isAudioOnly = true;
+            isAudioMuted = false;
+        }
+
+        if (r.error && r.critical) {
+            return createResponse("critical", {
+                code: `error.api.${r.error}`,
+            })
+        }
+
+        if (r.error) {
+            let context;
+            switch(r.error) {
+                case "content.too_long":
+                    context = {
+                        limit: env.durationLimit / 60,
+                    }
+                    break;
+
+                case "fetch.fail":
+                case "fetch.rate":
+                case "fetch.critical":
+                case "link.unsupported":
+                case "content.video.unavailable":
+                    context = {
+                        service: friendlyServiceName(host),
+                    }
+                    break;
+            }
+
+            return createResponse("error", {
+                code: `error.api.${r.error}`,
+                context,
+            })
+        }
+
+        return matchAction({
+            r,
+            host,
+            audioFormat: params.audioFormat,
+            isAudioOnly,
+            isAudioMuted,
+            disableMetadata: params.disableMetadata,
+            filenameStyle: params.filenameStyle,
+            twitterGif: params.twitterGif,
+            requestIP,
+            audioBitrate: params.audioBitrate,
+            alwaysProxy: params.alwaysProxy,
+        })
+    } catch {
+        return createResponse("error", {
+            code: "error.api.fetch.critical",
+            context: {
+                service: friendlyServiceName(host),
+            }
+        })
+    }
+}

src/processing/request.js

@@ -0,0 +1,97 @@
+import ipaddr from "ipaddr.js";
+
+import { createStream } from "../stream/manage.js";
+import { apiSchema } from "./schema.js";
+
+export function createResponse(responseType, responseData) {
+    const internalError = (code) => {
+        return {
+            status: 500,
+            body: {
+                status: "error",
+                error: {
+                    code: code || "error.api.fetch.critical",
+                },
+                critical: true
+            }
+        }
+    }
+
+    try {
+        let status = 200,
+            response = {};
+
+        if (responseType === "error") {
+            status = 400;
+        }
+
+        switch (responseType) {
+            case "error":
+                response = {
+                    error: {
+                        code: responseData?.code,
+                        context: responseData?.context,
+                    }
+                }
+                break;
+
+            case "redirect":
+                response = {
+                    url: responseData?.url,
+                    filename: responseData?.filename
+                }
+                break;
+
+            case "tunnel":
+                response = {
+                    url: createStream(responseData),
+                    filename: responseData?.filename
+                }
+                break;
+
+            case "picker":
+                response = {
+                    picker: responseData?.picker,
+                    audio: responseData?.url,
+                    audioFilename: responseData?.filename
+                }
+                break;
+
+            case "critical":
+                return internalError(responseData?.code);
+
+            default:
+                throw "unreachable"
+        }
+
+        return {
+            status,
+            body: {
+                status: responseType,
+                ...response
+            }
+        }
+    } catch {
+        return internalError()
+    }
+}
+
+export function normalizeRequest(request) {
+    return apiSchema.safeParseAsync(request).catch(() => (
+        { success: false }
+    ));
+}
+
+export function getIP(req) {
+    const strippedIP = req.ip.replace(/^::ffff:/, '');
+    const ip = ipaddr.parse(strippedIP);
+    if (ip.kind() === 'ipv4') {
+        return strippedIP;
+    }
+
+    const prefix = 56;
+    const v6Bytes = ip.toByteArray();
+          v6Bytes.fill(0, prefix / 8);
+
+    return ipaddr.fromByteArray(v6Bytes).toString();
+}

src/processing/schema.js

@@ -0,0 +1,51 @@
+import { z } from "zod";
+import { normalizeURL } from "./url.js";
+
+export const apiSchema = z.object({
+    url: z.string()
+          .min(1)
+          .transform(url => normalizeURL(url)),
+
+    audioBitrate: z.enum(
+        ["320", "256", "128", "96", "64", "8"]
+    ).default("128"),
+
+    audioFormat: z.enum(
+        ["best", "mp3", "ogg", "wav", "opus"]
+    ).default("mp3"),
+
+    downloadMode: z.enum(
+        ["auto", "audio", "mute"]
+    ).default("auto"),
+
+    filenameStyle: z.enum(
+        ["classic", "pretty", "basic", "nerdy"]
+    ).default("classic"),
+
+    youtubeVideoCodec: z.enum(
+        ["h264", "av1", "vp9"]
+    ).default("h264"),
+
+    videoQuality: z.enum(
+        ["max", "4320", "2160", "1440", "1080", "720", "480", "360", "240", "144"]
+    ).default("1080"),
+
+    youtubeDubLang: z.string()
+                     .min(2)
+                     .max(8)
+                     .regex(/^[0-9a-zA-Z\-]+$/)
+                     .optional(),
+
+    // TODO: remove this variable as it's no longer used
+    // and is kept for schema compatibility reasons
+    youtubeDubBrowserLang: z.boolean().default(false),
+
+    alwaysProxy: z.boolean().default(false),
+    disableMetadata: z.boolean().default(false),
+    tiktokFullAudio: z.boolean().default(false),
+    tiktokH265: z.boolean().default(false),
+    twitterGif: z.boolean().default(true),
+
+    youtubeHLS: z.boolean().default(false),
+})
+.strict();

src/processing/service-alias.js

@@ -0,0 +1,10 @@
+const friendlyNames = {
+    bsky: "bluesky",
+}
+
+export const friendlyServiceName = (service) => {
+    if (service in friendlyNames) {
+        return friendlyNames[service];
+    }
+    return service;
+}

src/processing/service-config.js

@@ -0,0 +1,185 @@
+import UrlPattern from "url-pattern";
+
+export const audioIgnore = ["vk", "ok", "loom"];
+export const hlsExceptions = ["dailymotion", "vimeo", "rutube", "bsky", "youtube"];
+
+export const services = {
+    bilibili: {
+        patterns: [
+            "video/:comId",
+            "_shortLink/:comShortLink",
+            "_tv/:lang/video/:tvId",
+            "_tv/video/:tvId"
+        ],
+        subdomains: ["m"],
+    },
+    bsky: {
+        patterns: [
+            "profile/:user/post/:post"
+        ],
+        tld: "app",
+    },
+    dailymotion: {
+        patterns: ["video/:id"],
+    },
+    facebook: {
+        patterns: [
+            "_shortLink/:shortLink",
+            ":username/videos/:caption/:id",
+            ":username/videos/:id",
+            "reel/:id",
+            "share/:shareType/:id"
+        ],
+        subdomains: ["web", "m"],
+        altDomains: ["fb.watch"],
+    },
+    instagram: {
+        patterns: [
+            "reels/:postId",
+            ":username/reel/:postId",
+            "reel/:postId",
+            "p/:postId",
+            ":username/p/:postId",
+            "tv/:postId",
+            "stories/:username/:storyId"
+        ],
+        altDomains: ["ddinstagram.com"],
+    },
+    loom: {
+        patterns: ["share/:id", "embed/:id"],
+    },
+    ok: {
+        patterns: [
+            "video/:id",
+            "videoembed/:id"
+        ],
+        tld: "ru",
+    },
+    pinterest: {
+        patterns: [
+            "pin/:id",
+            "pin/:id/:garbage",
+            "url_shortener/:shortLink"
+        ],
+    },
+    reddit: {
+        patterns: [
+            "r/:sub/comments/:id/:title",
+            "user/:user/comments/:id/:title"
+        ],
+        subdomains: "*",
+    },
+    rutube: {
+        patterns: [
+            "video/:id",
+            "play/embed/:id",
+            "shorts/:id",
+            "yappy/:yappyId",
+            "video/private/:id?p=:key",
+            "video/private/:id"
+        ],
+        tld: "ru",
+    },
+    snapchat: {
+        patterns: [
+            ":shortLink",
+            "spotlight/:spotlightId",
+            "add/:username/:storyId",
+            "u/:username/:storyId",
+            "add/:username",
+            "u/:username",
+            "t/:shortLink",
+        ],
+        subdomains: ["t", "story"],
+    },
+    soundcloud: {
+        patterns: [
+            ":author/:song/s-:accessKey",
+            ":author/:song",
+            ":shortLink"
+        ],
+        subdomains: ["on", "m"],
+    },
+    streamable: {
+        patterns: [
+            ":id",
+            "o/:id",
+            "e/:id",
+            "s/:id"
+        ],
+    },
+    tiktok: {
+        patterns: [
+            ":user/video/:postId",
+            ":shortLink",
+            "t/:shortLink",
+            ":user/photo/:postId",
+            "v/:postId.html"
+        ],
+        subdomains: ["vt", "vm", "m"],
+    },
+    tumblr: {
+        patterns: [
+            "post/:id",
+            "blog/view/:user/:id",
+            ":user/:id",
+            ":user/:id/:trackingId"
+        ],
+        subdomains: "*",
+    },
+    twitch: {
+        patterns: [":channel/clip/:clip"],
+        tld: "tv",
+    },
+    twitter: {
+        patterns: [
+            ":user/status/:id",
+            ":user/status/:id/video/:index",
+            ":user/status/:id/photo/:index",
+            ":user/status/:id/mediaviewer",
+            ":user/status/:id/mediaViewer",
+            "i/bookmarks?post_id=:id"
+        ],
+        subdomains: ["mobile"],
+        altDomains: ["x.com", "vxtwitter.com", "fixvx.com"],
+    },
+    vimeo: {
+        patterns: [
+            ":id",
+            "video/:id",
+            ":id/:password",
+            "/channels/:user/:id"
+        ],
+        subdomains: ["player"],
+    },
+    vk: {
+        patterns: [
+            "video:ownerId_:videoId",
+            "clip:ownerId_:videoId",
+            "clips:duplicate?z=clip:ownerId_:videoId",
+            "videos:duplicate?z=video:ownerId_:videoId",
+            "video:ownerId_:videoId_:accessKey",
+            "clip:ownerId_:videoId_:accessKey",
+            "clips:duplicate?z=clip:ownerId_:videoId_:accessKey",
+            "videos:duplicate?z=video:ownerId_:videoId_:accessKey"
+        ],
+        subdomains: ["m"],
+        altDomains: ["vkvideo.ru", "vk.ru"],
+    },
+    youtube: {
+        patterns: [
+            "watch?v=:id",
+            "embed/:id",
+            "watch/:id"
+        ],
+        subdomains: ["music", "m"],
+    }
+}
+
+Object.values(services).forEach(service => {
+    service.patterns = service.patterns.map(
+        pattern => new UrlPattern(pattern, {
+            segmentValueCharset: UrlPattern.defaultOptions.segmentValueCharset + '@\\.:'
+        })
+    )
+})

src/processing/service-patterns.js

@@ -0,0 +1,74 @@
+export const testers = {
+    "bilibili": pattern =>
+        pattern.comId?.length <= 12 || pattern.comShortLink?.length <= 16
+        || pattern.tvId?.length <= 24,
+
+    "dailymotion": pattern => pattern.id?.length <= 32,
+
+    "instagram": pattern =>
+        pattern.postId?.length <= 12
+        || (pattern.username?.length <= 30 && pattern.storyId?.length <= 24),
+
+    "loom": pattern =>
+        pattern.id?.length <= 32,
+
+    "ok": pattern =>
+        pattern.id?.length <= 16,
+
+    "pinterest": pattern =>
+        pattern.id?.length <= 128 || pattern.shortLink?.length <= 32,
+
+    "reddit": pattern =>
+        (pattern.sub?.length <= 22 && pattern.id?.length <= 10)
+        || (pattern.user?.length <= 22 && pattern.id?.length <= 10),
+
+    "rutube": pattern =>
+        (pattern.id?.length === 32 && pattern.key?.length <= 32) ||
+        pattern.id?.length === 32 || pattern.yappyId?.length === 32,
+
+    "soundcloud": pattern =>
+        (pattern.author?.length <= 255 && pattern.song?.length <= 255)
+        || pattern.shortLink?.length <= 32,
+
+    "snapchat": pattern =>
+        (pattern.username?.length <= 32 && (!pattern.storyId || pattern.storyId?.length <= 255))
+        || pattern.spotlightId?.length <= 255
+        || pattern.shortLink?.length <= 16,
+
+    "streamable": pattern =>
+        pattern.id?.length <= 6,
+
+    "tiktok": pattern =>
+        pattern.postId?.length <= 21 || pattern.shortLink?.length <= 13,
+
+    "tumblr": pattern =>
+        pattern.id?.length < 21
+        || (pattern.id?.length < 21 && pattern.user?.length <= 32),
+
+    "twitch": pattern =>
+        pattern.channel && pattern.clip?.length <= 100,
+
+    "twitter": pattern =>
+        pattern.id?.length < 20,
+
+    "vimeo": pattern =>
+        pattern.id?.length <= 11
+        && (!pattern.password || pattern.password.length < 16),
+
+    "vk": pattern =>
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10) ||
+        (pattern.ownerId?.length <= 10 && pattern.videoId?.length <= 10 && pattern.videoId?.accessKey <= 18),
+
+    "youtube": pattern =>
+        pattern.id?.length <= 11,
+
+    "facebook": pattern =>
+        pattern.shortLink?.length <= 11
+        || pattern.username?.length <= 30
+        || pattern.caption?.length <= 255
+        || pattern.id?.length <= 20 && !pattern.shareType
+        || pattern.id?.length <= 20 && pattern.shareType?.length === 1,
+
+    "bsky": pattern =>
+        pattern.user?.length <= 128 && pattern.post?.length <= 128,
+}

src/processing/services/bilibili.js

@@ -0,0 +1,112 @@
+import { genericUserAgent, env } from "../../config.js";
+
+// TO-DO: higher quality downloads (currently requires an account)
+
+function com_resolveShortlink(shortId) {
+    return fetch(`https://b23.tv/${shortId}`, { redirect: 'manual' })
+            .then(r => r.status > 300 && r.status < 400 && r.headers.get('location'))
+            .then(url => {
+                if (!url) return;
+                const path = new URL(url).pathname;
+                if (path.startsWith('/video/'))
+                    return path.split('/')[2];
+            })
+            .catch(() => {})
+}
+
+function getBest(content) {
+    return content?.filter(v => v.baseUrl || v.url)
+                .map(v => (v.baseUrl = v.baseUrl || v.url, v))
+                .reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b);
+}
+
+function extractBestQuality(dashData) {
+    const bestVideo = getBest(dashData.video),
+          bestAudio = getBest(dashData.audio);
+
+    if (!bestVideo || !bestAudio) return [];
+    return [ bestVideo, bestAudio ];
+}
+
+async function com_download(id) {
+    let html = await fetch(`https://bilibili.com/video/${id}`, {
+        headers: {
+            "user-agent": genericUserAgent
+        }
+    })
+    .then(r => r.text())
+    .catch(() => {});
+
+    if (!html) {
+        return { error: "fetch.fail" }
+    }
+
+    if (!(html.includes('<script>window.__playinfo__=') && html.includes('"video_codecid"'))) {
+        return { error: "fetch.empty" };
+    }
+
+    let streamData = JSON.parse(html.split('<script>window.__playinfo__=')[1].split('</script>')[0]);
+    if (streamData.data.timelength > env.durationLimit * 1000) {
+        return { error: "content.too_long" };
+    }
+
+    const [ video, audio ] = extractBestQuality(streamData.data.dash);
+    if (!video || !audio) {
+        return { error: "fetch.empty" };
+    }
+
+    return {
+        urls: [video.baseUrl, audio.baseUrl],
+        audioFilename: `bilibili_${id}_audio`,
+        filename: `bilibili_${id}_${video.width}x${video.height}.mp4`
+    };
+}
+
+async function tv_download(id) {
+    const url = new URL(
+        'https://api.bilibili.tv/intl/gateway/web/playurl'
+        + '?s_locale=en_US&platform=web&qn=64&type=0&device=wap'
+        + '&tf=0&spm_id=bstar-web.ugc-video-detail.0.0&from_spm_id='
+    );
+
+    url.searchParams.set('aid', id);
+
+    const { data } = await fetch(url).then(a => a.json());
+    if (!data?.playurl?.video) {
+        return { error: "fetch.empty" };
+    }
+
+    const [ video, audio ] = extractBestQuality({
+        video: data.playurl.video.map(s => s.video_resource)
+                                 .filter(s => s.codecs.includes('avc1')),
+        audio: data.playurl.audio_resource
+    });
+
+    if (!video || !audio) {
+        return { error: "fetch.empty" };
+    }
+
+    if (video.duration > env.durationLimit * 1000) {
+        return { error: "content.too_long" };
+    }
+
+    return {
+        urls: [video.url, audio.url],
+        audioFilename: `bilibili_tv_${id}_audio`,
+        filename: `bilibili_tv_${id}.mp4`
+    };
+}
+
+export default async function({ comId, tvId, comShortLink }) {
+    if (comShortLink) {
+        comId = await com_resolveShortlink(comShortLink);
+    }
+
+    if (comId) {
+        return com_download(comId);
+    } else if (tvId) {
+        return tv_download(tvId);
+    }
+
+    return { error: "fetch.fail" };
+}

src/processing/services/bluesky.js

@@ -0,0 +1,124 @@
+import HLS from "hls-parser";
+import { cobaltUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+
+const extractVideo = async ({ media, filename, dispatcher }) => {
+    let urlMasterHLS = media?.playlist;
+
+    if (!urlMasterHLS || !urlMasterHLS.startsWith("https://video.bsky.app/")) {
+        return { error: "fetch.empty" };
+    }
+
+    urlMasterHLS = urlMasterHLS.replace(
+        "video.bsky.app/watch/",
+        "video.cdn.bsky.app/hls/"
+    );
+
+    const masterHLS = await fetch(urlMasterHLS, { dispatcher })
+        .then(r => {
+            if (r.status !== 200) return;
+            return r.text();
+        })
+        .catch(() => {});
+
+    if (!masterHLS) return { error: "fetch.empty" };
+
+    const video = HLS.parse(masterHLS)
+            ?.variants
+            ?.reduce((a, b) => a?.bandwidth > b?.bandwidth ? a : b);
+
+    const videoURL = new URL(video.uri, urlMasterHLS).toString();
+
+    return {
+        urls: videoURL,
+        filename: `${filename}.mp4`,
+        audioFilename: `${filename}_audio`,
+        isHLS: true,
+    }
+}
+
+const extractImages = ({ getPost, filename, alwaysProxy }) => {
+    const images = getPost?.thread?.post?.embed?.images;
+
+    if (!images || images.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    if (images.length === 1) return {
+        urls: images[0].fullsize,
+        isPhoto: true,
+        filename: `${filename}.jpg`,
+    }
+
+    const picker = images.map((image, i) => {
+        let url = image.fullsize;
+        let proxiedImage = createStream({
+            service: "bluesky",
+            type: "proxy",
+            url,
+            filename: `${filename}_${i + 1}.jpg`,
+        });
+
+        if (alwaysProxy) url = proxiedImage;
+
+        return {
+            type: "photo",
+            url,
+            thumb: proxiedImage,
+        }
+    });
+
+    return { picker };
+}
+
+export default async function ({ user, post, alwaysProxy, dispatcher }) {
+    const apiEndpoint = new URL("https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread?depth=0&parentHeight=0");
+    apiEndpoint.searchParams.set(
+        "uri",
+        `at://${user}/app.bsky.feed.post/${post}`
+    );
+
+    const getPost = await fetch(apiEndpoint, {
+        headers: {
+            "user-agent": cobaltUserAgent,
+        },
+        dispatcher
+    }).then(r => r.json()).catch(() => {});
+
+    if (!getPost) return { error: "fetch.empty" };
+
+    if (getPost.error) {
+        switch (getPost.error) {
+            case "NotFound":
+            case "InternalServerError":
+                return { error: "content.post.unavailable" };
+            case "InvalidRequest":
+                return { error: "link.unsupported" };
+            default:
+                return { error: "content.post.unavailable" };
+        }
+    }
+
+    const embedType = getPost?.thread?.post?.embed?.$type;
+    const filename = `bluesky_${user}_${post}`;
+
+    if (embedType === "app.bsky.embed.video#view") {
+        return extractVideo({
+            media: getPost.thread?.post?.embed,
+            filename,
+        })
+    }
+
+    if (embedType === "app.bsky.embed.recordWithMedia#view") {
+        return extractVideo({
+            media: getPost.thread?.post?.embed?.media,
+            filename,
+        })
+    }
+
+    if (embedType === "app.bsky.embed.images#view") {
+        return extractImages({ getPost, filename, alwaysProxy });
+    }
+
+    return { error: "fetch.empty" };
+}

src/processing/services/dailymotion.js

@@ -0,0 +1,107 @@
+import HLSParser from "hls-parser";
+import { env } from "../../config.js";
+
+let _token;
+
+function getExp(token) {
+    return JSON.parse(
+        Buffer.from(token.split('.')[1], 'base64')
+    ).exp * 1000;
+}
+
+const getToken = async () => {
+    if (_token && getExp(_token) > new Date().getTime()) {
+        return _token;
+    }
+
+    const req = await fetch('https://graphql.api.dailymotion.com/oauth/token', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            'User-Agent': 'dailymotion/240213162706 CFNetwork/1492.0.1 Darwin/23.3.0',
+            'Authorization': 'Basic MGQyZDgyNjQwOWFmOWU3MmRiNWQ6ODcxNmJmYTVjYmEwMmUwMGJkYTVmYTg1NTliNDIwMzQ3NzIyYWMzYQ=='
+        },
+        body: 'traffic_segment=&grant_type=client_credentials'
+    }).then(r => r.json()).catch(() => {});
+
+    if (req.access_token) {
+        return _token = req.access_token;
+    }
+}
+
+export default async function({ id }) {
+    const token = await getToken();
+    if (!token) return { error: "fetch.fail" };
+
+    const req = await fetch('https://graphql.api.dailymotion.com/',
+        {
+            method: 'POST',
+            headers: {
+                'User-Agent': 'dailymotion/240213162706 CFNetwork/1492.0.1 Darwin/23.3.0',
+                Authorization: `Bearer ${token}`,
+                'Content-Type': 'application/json',
+                'X-DM-AppInfo-Version': '7.16.0_240213162706',
+                'X-DM-AppInfo-Type': 'iosapp',
+                'X-DM-AppInfo-Id': 'com.dailymotion.dailymotion'
+            },
+            body: JSON.stringify({
+                operationName: "Media",
+                query: `
+                    query Media($xid: String!, $password: String) {
+                        media(xid: $xid, password: $password) {
+                        __typename
+                            ... on Video {
+                                xid
+                                hlsURL
+                                duration
+                                title
+                                channel {
+                                    displayName
+                                }
+                            }
+                        }
+                    }
+                `,
+                variables: { xid: id }
+            })
+        }
+    ).then(r => r.status === 200 && r.json()).catch(() => {});
+
+    const media = req?.data?.media;
+
+    if (media?.__typename !== 'Video' || !media.hlsURL) {
+        return { error: "fetch.empty" }
+    }
+
+    if (media.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
+    const manifest = await fetch(media.hlsURL).then(r => r.text()).catch(() => {});
+    if (!manifest) return { error: "fetch.fail" };
+
+    const bestQuality = HLSParser.parse(manifest).variants
+                        .filter(v => v.codecs.includes('avc1'))
+                        .reduce((a, b) => a.bandwidth > b.bandwidth ? a : b);
+    if (!bestQuality) return { error: "fetch.empty" }
+
+    const fileMetadata = {
+        title: media.title,
+        artist: media.channel.displayName
+    }
+
+    return {
+        urls: bestQuality.uri,
+        isHLS: true,
+        filenameAttributes: {
+            service: 'dailymotion',
+            id: media.xid,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            qualityLabel: `${bestQuality.resolution.height}p`,
+            extension: 'mp4'
+        },
+        fileMetadata
+    }
+}

src/processing/services/facebook.js

@@ -0,0 +1,57 @@
+import { genericUserAgent } from "../../config.js";
+
+const headers = {
+    'User-Agent': genericUserAgent,
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.5',
+    'Sec-Fetch-Mode': 'navigate',
+    'Sec-Fetch-Site': 'none',
+}
+
+const resolveUrl = (url) => {
+    return fetch(url, { headers })
+        .then(r => {
+            if (r.headers.get('location')) {
+                return decodeURIComponent(r.headers.get('location'));
+            }
+            if (r.headers.get('link')) {
+                const linkMatch = r.headers.get('link').match(/<(.*?)\/>/);
+                return decodeURIComponent(linkMatch[1]);
+            }
+            return false;
+        })
+        .catch(() => false);
+}
+
+export default async function({ id, shareType, shortLink }) {
+    let url = `https://web.facebook.com/i/videos/${id}`;
+
+    if (shareType) url = `https://web.facebook.com/share/${shareType}/${id}`;
+    if (shortLink) url = await resolveUrl(`https://fb.watch/${shortLink}`);
+
+    const html = await fetch(url, { headers })
+        .then(r => r.text())
+        .catch(() => false);
+
+    if (!html && shortLink) return { error: "fetch.short_link" }
+    if (!html) return { error: "fetch.fail" };
+
+    const urls = [];
+    const hd = html.match('"browser_native_hd_url":(".*?")');
+    const sd = html.match('"browser_native_sd_url":(".*?")');
+
+    if (hd?.[1]) urls.push(JSON.parse(hd[1]));
+    if (sd?.[1]) urls.push(JSON.parse(sd[1]));
+
+    if (!urls.length) {
+        return { error: "fetch.empty" };
+    }
+
+    const baseFilename = `facebook_${id || shortLink}`;
+
+    return {
+        urls: urls[0],
+        filename: `${baseFilename}.mp4`,
+        audioFilename: `${baseFilename}_audio`,
+    };
+}

src/processing/services/instagram.js

@@ -0,0 +1,373 @@
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+import { getCookie, updateCookie } from "../cookie/manager.js";
+
+const commonHeaders = {
+    "user-agent": genericUserAgent,
+    "sec-gpc": "1",
+    "sec-fetch-site": "same-origin",
+    "x-ig-app-id": "936619743392459"
+}
+const mobileHeaders = {
+    "x-ig-app-locale": "en_US",
+    "x-ig-device-locale": "en_US",
+    "x-ig-mapped-locale": "en_US",
+    "user-agent": "Instagram 275.0.0.27.98 Android (33/13; 280dpi; 720x1423; Xiaomi; Redmi 7; onclite; qcom; en_US; 458229237)",
+    "accept-language": "en-US",
+    "x-fb-http-engine": "Liger",
+    "x-fb-client-ip": "True",
+    "x-fb-server-cluster": "True",
+    "content-length": "0",
+}
+const embedHeaders = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "Accept-Language": "en-GB,en;q=0.9",
+    "Cache-Control": "max-age=0",
+    "Dnt": "1",
+    "Priority": "u=0, i",
+    "Sec-Ch-Ua": 'Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99',
+    "Sec-Ch-Ua-Mobile": "?0",
+    "Sec-Ch-Ua-Platform": "macOS",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+}
+
+const cachedDtsg = {
+    value: '',
+    expiry: 0
+}
+
+export default function(obj) {
+    const dispatcher = obj.dispatcher;
+
+    async function findDtsgId(cookie) {
+        try {
+            if (cachedDtsg.expiry > Date.now()) return cachedDtsg.value;
+
+            const data = await fetch('https://www.instagram.com/', {
+                headers: {
+                    ...commonHeaders,
+                    cookie
+                },
+                dispatcher
+            }).then(r => r.text());
+
+            const token = data.match(/"dtsg":{"token":"(.*?)"/)[1];
+
+            cachedDtsg.value = token;
+            cachedDtsg.expiry = Date.now() + 86390000;
+
+            if (token) return token;
+            return false;
+        }
+        catch {}
+    }
+
+    async function request(url, cookie, method = 'GET', requestData) {
+        let headers = {
+            ...commonHeaders,
+            'x-ig-www-claim': cookie?._wwwClaim || '0',
+            'x-csrftoken': cookie?.values()?.csrftoken,
+            cookie
+        }
+        if (method === 'POST') {
+            headers['content-type'] = 'application/x-www-form-urlencoded';
+        }
+
+        const data = await fetch(url, {
+            method,
+            headers,
+            body: requestData && new URLSearchParams(requestData),
+            dispatcher
+        });
+
+        if (data.headers.get('X-Ig-Set-Www-Claim') && cookie)
+            cookie._wwwClaim = data.headers.get('X-Ig-Set-Www-Claim');
+
+        updateCookie(cookie, data.headers);
+        return data.json();
+    }
+    async function getMediaId(id, { cookie, token } = {}) {
+        const oembedURL = new URL('https://i.instagram.com/api/v1/oembed/');
+        oembedURL.searchParams.set('url', `https://www.instagram.com/p/${id}/`);
+
+        const oembed = await fetch(oembedURL, {
+            headers: {
+                ...mobileHeaders,
+                ...( token && { authorization: `Bearer ${token}` } ),
+                cookie
+            },
+            dispatcher
+        }).then(r => r.json()).catch(() => {});
+
+        return oembed?.media_id;
+    }
+
+    async function requestMobileApi(mediaId, { cookie, token } = {}) {
+        const mediaInfo = await fetch(`https://i.instagram.com/api/v1/media/${mediaId}/info/`, {
+            headers: {
+                ...mobileHeaders,
+                ...( token && { authorization: `Bearer ${token}` } ),
+                cookie
+            },
+            dispatcher
+        }).then(r => r.json()).catch(() => {});
+
+        return mediaInfo?.items?.[0];
+    }
+    async function requestHTML(id, cookie) {
+        const data = await fetch(`https://www.instagram.com/p/${id}/embed/captioned/`, {
+            headers: {
+                ...embedHeaders,
+                cookie
+            },
+            dispatcher
+        }).then(r => r.text()).catch(() => {});
+
+        let embedData = JSON.parse(data?.match(/"init",\[\],\[(.*?)\]\],/)[1]);
+
+        if (!embedData || !embedData?.contextJSON) return false;
+
+        embedData = JSON.parse(embedData.contextJSON);
+
+        return embedData;
+    }
+    async function requestGQL(id, cookie) {
+        let dtsgId;
+
+        if (cookie) {
+            dtsgId = await findDtsgId(cookie);
+        }
+        const url = new URL('https://www.instagram.com/api/graphql/');
+
+        const requestData = {
+            jazoest: '26406',
+            variables: JSON.stringify({
+            shortcode: id,
+            __relay_internal__pv__PolarisShareMenurelayprovider: false
+            }),
+            doc_id: '7153618348081770'
+        };
+        if (dtsgId) {
+            requestData.fb_dtsg = dtsgId;
+        }
+
+        return (await request(url, cookie, 'POST', requestData))
+                .data
+                ?.xdt_api__v1__media__shortcode__web_info
+                ?.items
+                ?.[0];
+    }
+
+    function extractOldPost(data, id, alwaysProxy) {
+        const sidecar = data?.gql_data?.shortcode_media?.edge_sidecar_to_children;
+        if (sidecar) {
+            const picker = sidecar.edges.filter(e => e.node?.display_url)
+                .map((e, i) => {
+                    const type = e.node?.is_video ? "video" : "photo";
+                    const url = type === "video" ? e.node?.video_url : e.node?.display_url;
+
+                    let itemExt = type === "video" ? "mp4" : "jpg";
+
+                    let proxyFile;
+                    if (alwaysProxy) proxyFile = createStream({
+                        service: "instagram",
+                        type: "proxy",
+                        url,
+                        filename: `instagram_${id}_${i + 1}.${itemExt}`
+                    });
+
+                    return {
+                        type,
+                        url: proxyFile || url,
+                        /* thumbnails have `Cross-Origin-Resource-Policy`
+                        ** set to `same-origin`, so we need to proxy them */
+                        thumb: createStream({
+                            service: "instagram",
+                            type: "proxy",
+                            url: e.node?.display_url,
+                            filename: `instagram_${id}_${i + 1}.jpg`
+                        })
+                    }
+                });
+
+            if (picker.length) return { picker }
+        } else if (data?.gql_data?.shortcode_media?.video_url) {
+            return {
+                urls: data.gql_data.shortcode_media.video_url,
+                filename: `instagram_${id}.mp4`,
+                audioFilename: `instagram_${id}_audio`
+            }
+        } else if (data?.gql_data?.shortcode_media?.display_url) {
+            return {
+                urls: data.gql_data?.shortcode_media.display_url,
+                isPhoto: true
+            }
+        }
+    }
+
+    function extractNewPost(data, id, alwaysProxy) {
+        const carousel = data.carousel_media;
+        if (carousel) {
+            const picker = carousel.filter(e => e?.image_versions2)
+                .map((e, i) => {
+                    const type = e.video_versions ? "video" : "photo";
+                    const imageUrl = e.image_versions2.candidates[0].url;
+
+                    let url = imageUrl;
+                    let itemExt = type === "video" ? "mp4" : "jpg";
+
+                    if (type === "video") {
+                        const video = e.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a);
+                        url = video.url;
+                    }
+
+                    let proxyFile;
+                    if (alwaysProxy) proxyFile = createStream({
+                        service: "instagram",
+                        type: "proxy",
+                        url,
+                        filename: `instagram_${id}_${i + 1}.${itemExt}`
+                    });
+
+                    return {
+                        type,
+                        url: proxyFile || url,
+                        /* thumbnails have `Cross-Origin-Resource-Policy`
+                        ** set to `same-origin`, so we need to always proxy them */
+                        thumb: createStream({
+                            service: "instagram",
+                            type: "proxy",
+                            url: imageUrl,
+                            filename: `instagram_${id}_${i + 1}.jpg`
+                        })
+                    }
+                });
+
+            if (picker.length) return { picker }
+        } else if (data.video_versions) {
+            const video = data.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a)
+            return {
+                urls: video.url,
+                filename: `instagram_${id}.mp4`,
+                audioFilename: `instagram_${id}_audio`
+            }
+        } else if (data.image_versions2?.candidates) {
+            return {
+                urls: data.image_versions2.candidates[0].url,
+                isPhoto: true,
+                filename: `instagram_${id}.jpg`,
+            }
+        }
+    }
+
+    async function getPost(id, alwaysProxy) {
+        const hasData = (data) => data && data.gql_data !== null;
+        let data, result;
+        try {
+            const cookie = getCookie('instagram');
+
+            const bearer = getCookie('instagram_bearer');
+            const token = bearer?.values()?.token;
+
+            // get media_id for mobile api, three methods
+            let media_id = await getMediaId(id);
+            if (!media_id && token) media_id = await getMediaId(id, { token });
+            if (!media_id && cookie) media_id = await getMediaId(id, { cookie });
+
+            // mobile api (bearer)
+            if (media_id && token) data = await requestMobileApi(media_id, { token });
+
+            // mobile api (no cookie, cookie)
+            if (media_id && !hasData(data)) data = await requestMobileApi(media_id);
+            if (media_id && cookie && !hasData(data)) data = await requestMobileApi(media_id, { cookie });
+
+            // html embed (no cookie, cookie)
+            if (!hasData(data)) data = await requestHTML(id);
+            if (!hasData(data) && cookie) data = await requestHTML(id, cookie);
+
+            // web app graphql api (no cookie, cookie)
+            if (!hasData(data)) data = await requestGQL(id);
+            if (!hasData(data) && cookie) data = await requestGQL(id, cookie);
+        } catch {}
+
+        if (!data) return { error: "fetch.fail" };
+
+        if (data?.gql_data) {
+            result = extractOldPost(data, id, alwaysProxy)
+        } else {
+            result = extractNewPost(data, id, alwaysProxy)
+        }
+
+        if (result) return result;
+        return { error: "fetch.empty" }
+    }
+
+    async function usernameToId(username, cookie) {
+        const url = new URL('https://www.instagram.com/api/v1/users/web_profile_info/');
+            url.searchParams.set('username', username);
+
+        try {
+            const data = await request(url, cookie);
+            return data?.data?.user?.id;
+        } catch {}
+    }
+
+    async function getStory(username, id) {
+        const cookie = getCookie('instagram');
+        if (!cookie) return { error: "link.unsupported" };
+
+        const userId = await usernameToId(username, cookie);
+        if (!userId) return { error: "fetch.empty" };
+
+        const dtsgId = await findDtsgId(cookie);
+
+        const url = new URL('https://www.instagram.com/api/graphql/');
+        const requestData = {
+            fb_dtsg: dtsgId,
+            jazoest: '26438',
+            variables: JSON.stringify({
+                reel_ids_arr : [ userId ],
+            }),
+            server_timestamps: true,
+            doc_id: '25317500907894419'
+        };
+
+        let media;
+        try {
+            const data = (await request(url, cookie, 'POST', requestData));
+            media = data?.data?.xdt_api__v1__feed__reels_media?.reels_media?.find(m => m.id === userId);
+        } catch {}
+
+        const item = media.items.find(m => m.pk === id);
+        if (!item) return { error: "fetch.empty" };
+
+        if (item.video_versions) {
+            const video = item.video_versions.reduce((a, b) => a.width * a.height < b.width * b.height ? b : a)
+            return {
+                urls: video.url,
+                filename: `instagram_${id}.mp4`,
+                audioFilename: `instagram_${id}_audio`
+            }
+        }
+
+        if (item.image_versions2?.candidates) {
+            return {
+                urls: item.image_versions2.candidates[0].url,
+                isPhoto: true
+            }
+        }
+
+        return { error: "link.unsupported" };
+    }
+
+    const { postId, storyId, username, alwaysProxy } = obj;
+    if (postId) return getPost(postId, alwaysProxy);
+    if (username && storyId) return getStory(username, storyId);
+
+    return { error: "fetch.empty" }
+}

src/processing/services/loom.js

@@ -0,0 +1,39 @@
+import { genericUserAgent } from "../../config.js";
+
+export default async function({ id }) {
+    const gql = await fetch(`https://www.loom.com/api/campaigns/sessions/${id}/transcoded-url`, {
+        method: "POST",
+        headers: {
+            "user-agent": genericUserAgent,
+            origin: "https://www.loom.com",
+            referer: `https://www.loom.com/share/${id}`,
+            cookie: `loom_referral_video=${id};`,
+
+            "apollographql-client-name": "web",
+            "apollographql-client-version": "14c0b42",
+            "x-loom-request-source": "loom_web_14c0b42",
+        },
+        body: JSON.stringify({
+            force_original: false,
+            password: null,
+            anonID: null,
+            deviceID: null
+        })
+    })
+    .then(r => r.status === 200 ? r.json() : false)
+    .catch(() => {});
+
+    if (!gql) return { error: "fetch.empty" };
+
+    const videoUrl = gql?.url;
+
+    if (videoUrl?.includes('.mp4?')) {
+        return {
+            urls: videoUrl,
+            filename: `loom_${id}.mp4`,
+            audioFilename: `loom_${id}_audio`
+        }
+    }
+
+    return { error: "fetch.empty" }
+}

src/processing/services/ok.js

@@ -0,0 +1,64 @@
+import { genericUserAgent, env } from "../../config.js";
+
+const resolutions = {
+    "ultra": "2160",
+    "quad": "1440",
+    "full": "1080",
+    "hd": "720",
+    "sd": "480",
+    "low": "360",
+    "lowest": "240",
+    "mobile": "144"
+}
+
+export default async function(o) {
+    let quality = o.quality === "max" ? "2160" : o.quality;
+
+    let html = await fetch(`https://ok.ru/video/${o.id}`, {
+        headers: { "user-agent": genericUserAgent }
+    }).then(r => r.text()).catch(() => {});
+
+    if (!html) return { error: "fetch.fail" };
+
+    let videoData = html.match(/<div data-module="OKVideo" .*? data-options="({.*?})"( .*?)>/)
+                        ?.[1]
+                        ?.replaceAll("&quot;", '"');
+
+    if (!videoData) {
+        return { error: "fetch.empty" };
+    }
+
+    videoData = JSON.parse(JSON.parse(videoData).flashvars.metadata);
+
+    if (videoData.provider !== "UPLOADED_ODKL")
+        return { error: "link.unsupported" };
+
+    if (videoData.movie.is_live)
+        return { error: "content.video.live" };
+
+    if (videoData.movie.duration > env.durationLimit)
+        return { error: "content.too_long" };
+
+    let videos = videoData.videos.filter(v => !v.disallowed);
+    let bestVideo = videos.find(v => resolutions[v.name] === quality) || videos[videos.length - 1];
+
+    let fileMetadata = {
+        title: videoData.movie.title.trim(),
+        author: (videoData.author?.name || videoData.compilationTitle).trim(),
+    }
+
+    if (bestVideo) return {
+        urls: bestVideo.url,
+        filenameAttributes: {
+            service: "ok",
+            id: o.id,
+            title: fileMetadata.title,
+            author: fileMetadata.author,
+            resolution: `${resolutions[bestVideo.name]}p`,
+            qualityLabel: `${resolutions[bestVideo.name]}p`,
+            extension: "mp4"
+        }
+    }
+
+    return { error: "fetch.empty" }
+}

src/processing/services/pinterest.js

@@ -0,0 +1,46 @@
+import { genericUserAgent } from "../../config.js";
+
+const videoRegex = /"url":"(https:\/\/v1\.pinimg\.com\/videos\/.*?)"/g;
+const imageRegex = /src="(https:\/\/i\.pinimg\.com\/.*\.(jpg|gif))"/g;
+
+export default async function(o) {
+    let id = o.id;
+
+    if (!o.id && o.shortLink) {
+        id = await fetch(`https://api.pinterest.com/url_shortener/${o.shortLink}/redirect/`, { redirect: "manual" })
+                   .then(r => r.headers.get("location").split('pin/')[1].split('/')[0])
+                   .catch(() => {});
+    }
+    if (id.includes("--")) id = id.split("--")[1];
+    if (!id) return { error: "fetch.fail" };
+
+    const html = await fetch(`https://www.pinterest.com/pin/${id}/`, {
+        headers: { "user-agent": genericUserAgent }
+    }).then(r => r.text()).catch(() => {});
+
+    if (!html) return { error: "fetch.fail" };
+
+    const videoLink = [...html.matchAll(videoRegex)]
+                    .map(([, link]) => link)
+                    .find(a => a.endsWith('.mp4') && a.includes('720p'));
+
+    if (videoLink) return {
+        urls: videoLink,
+        filename: `pinterest_${o.id}.mp4`,
+        audioFilename: `pinterest_${o.id}_audio`
+    }
+
+    const imageLink = [...html.matchAll(imageRegex)]
+                    .map(([, link]) => link)
+                    .find(a => a.endsWith('.jpg') || a.endsWith('.gif'));
+
+    const imageType = imageLink.endsWith(".gif") ? "gif" : "jpg"
+
+    if (imageLink) return {
+        urls: imageLink,
+        isPhoto: true,
+        filename: `pinterest_${o.id}.${imageType}`
+    }
+
+    return { error: "fetch.empty" };
+}

src/processing/services/reddit.js

@@ -0,0 +1,127 @@
+import { genericUserAgent, env } from "../../config.js";
+import { getCookie, updateCookieValues } from "../cookie/manager.js";
+
+async function getAccessToken() {
+    /* "cookie" in cookiefile needs to contain:
+     * client_id, client_secret, refresh_token
+     * e.g. client_id=bla; client_secret=bla; refresh_token=bla
+     *
+     * you can get these by making a reddit app and
+     * authenticating an account against reddit's oauth2 api
+     * see: https://github.com/reddit-archive/reddit/wiki/OAuth2
+     *
+     * any additional cookie fields are managed by this code and you
+     * should not touch them unless you know what you're doing. **/
+    const cookie = await getCookie('reddit');
+    if (!cookie) return;
+
+    const values = cookie.values(),
+          needRefresh = !values.access_token
+                        || !values.expiry
+                        || Number(values.expiry) < new Date().getTime();
+    if (!needRefresh) return values.access_token;
+
+    const data = await fetch('https://www.reddit.com/api/v1/access_token', {
+        method: 'POST',
+        headers: {
+            'authorization': `Basic ${Buffer.from(
+                [values.client_id, values.client_secret].join(':')
+            ).toString('base64')}`,
+            'content-type': 'application/x-www-form-urlencoded',
+            'user-agent': genericUserAgent,
+            'accept': 'application/json'
+        },
+        body: `grant_type=refresh_token&refresh_token=${encodeURIComponent(values.refresh_token)}`
+    }).then(r => r.json()).catch(() => {});
+    if (!data) return;
+
+    const { access_token, refresh_token, expires_in } = data;
+    if (!access_token) return;
+
+    updateCookieValues(cookie, {
+        ...cookie.values(),
+        access_token, refresh_token,
+        expiry: new Date().getTime() + (expires_in * 1000),
+    });
+
+    return access_token;
+}
+
+export default async function(obj) {
+    let url = new URL(`https://www.reddit.com/r/${obj.sub}/comments/${obj.id}.json`);
+
+    if (obj.user) {
+        url.pathname = `/user/${obj.user}/comments/${obj.id}.json`;
+    }
+
+    const accessToken = await getAccessToken();
+    if (accessToken) url.hostname = 'oauth.reddit.com';
+
+    let data = await fetch(
+        url, {
+            headers: {
+                'User-Agent': genericUserAgent,
+                accept: 'application/json',
+                authorization: accessToken && `Bearer ${accessToken}`
+            }
+        }
+    ).then(r => r.json()).catch(() => {});
+
+    if (!data || !Array.isArray(data)) {
+        return { error: "fetch.fail" }
+    }
+
+    data = data[0]?.data?.children[0]?.data;
+
+    const id = `${String(obj.sub).toLowerCase()}_${obj.id}`;
+
+    if (data?.url?.endsWith('.gif')) return {
+        typeId: "redirect",
+        urls: data.url,
+        filename: `reddit_${id}.gif`,
+    }
+
+    if (!data.secure_media?.reddit_video)
+        return { error: "fetch.empty" };
+
+    if (data.secure_media?.reddit_video?.duration > env.durationLimit)
+        return { error: "content.too_long" };
+
+    let audio = false,
+        video = data.secure_media?.reddit_video?.fallback_url?.split('?')[0],
+        audioFileLink = `${data.secure_media?.reddit_video?.fallback_url?.split('DASH')[0]}audio`;
+
+    if (video.match('.mp4')) {
+        audioFileLink = `${video.split('_')[0]}_audio.mp4`
+    }
+
+    // test the existence of audio
+    await fetch(audioFileLink, { method: "HEAD" }).then(r => {
+        if (Number(r.status) === 200) {
+            audio = true
+        }
+    }).catch(() => {})
+
+    // fallback for videos with variable audio quality
+    if (!audio) {
+        audioFileLink = `${video.split('_')[0]}_AUDIO_128.mp4`
+        await fetch(audioFileLink, { method: "HEAD" }).then(r => {
+            if (Number(r.status) === 200) {
+                audio = true
+            }
+        }).catch(() => {})
+    }
+
+    if (!audio) return {
+        typeId: "redirect",
+        urls: video
+    }
+
+    return {
+        typeId: "tunnel",
+        type: "merge",
+        urls: [video, audioFileLink],
+        audioFilename: `reddit_${id}_audio`,
+        filename: `reddit_${id}.mp4`
+    }
+}

src/processing/services/rutube.js

@@ -0,0 +1,82 @@
+import HLS from "hls-parser";
+import { env } from "../../config.js";
+
+async function requestJSON(url) {
+    try {
+        const r = await fetch(url);
+        return await r.json();
+    } catch {}
+}
+
+const delta = (a, b) => Math.abs(a - b);
+
+export default async function(obj) {
+    if (obj.yappyId) {
+        const yappy = await requestJSON(
+            `https://rutube.ru/pangolin/api/web/yappy/yappypage/?client=wdp&videoId=${obj.yappyId}&page=1&page_size=15`
+        )
+        const yappyURL = yappy?.results?.find(r => r.id === obj.yappyId)?.link;
+        if (!yappyURL) return { error: "fetch.empty" };
+
+        return {
+            urls: yappyURL,
+            filename: `rutube_yappy_${obj.yappyId}.mp4`,
+            audioFilename: `rutube_yappy_${obj.yappyId}_audio`
+        }
+    }
+
+    const quality = Number(obj.quality) || 9000;
+
+    const requestURL = new URL(`https://rutube.ru/api/play/options/${obj.id}/?no_404=true&referer&pver=v2`);
+    if (obj.key) requestURL.searchParams.set('p', obj.key);
+
+    const play = await requestJSON(requestURL);
+    if (!play) return { error: "fetch.fail" };
+
+    if (play.detail?.type === "blocking_rule") {
+        return { error: "content.video.region" };
+    }
+
+    if (play.detail || !play.video_balancer) return { error: "fetch.empty" };
+    if (play.live_streams?.hls) return { error: "content.video.live" };
+
+    if (play.duration > env.durationLimit * 1000)
+        return { error: "content.too_long" };
+
+    let m3u8 = await fetch(play.video_balancer.m3u8)
+                     .then(r => r.text())
+                     .catch(() => {});
+
+    if (!m3u8) return { error: "fetch.fail" };
+
+    m3u8 = HLS.parse(m3u8).variants;
+
+    const matchingQuality = m3u8.reduce((prev, next) => {
+        const diff = {
+            prev: delta(quality, prev.resolution.height),
+            next: delta(quality, next.resolution.height)
+        };
+
+        return diff.prev < diff.next ? prev : next;
+    });
+
+    const fileMetadata = {
+        title: play.title.trim(),
+        artist: play.author.name.trim(),
+    }
+
+    return {
+        urls: matchingQuality.uri,
+        isHLS: true,
+        filenameAttributes: {
+            service: "rutube",
+            id: obj.id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist,
+            resolution: `${matchingQuality.resolution.width}x${matchingQuality.resolution.height}`,
+            qualityLabel: `${matchingQuality.resolution.height}p`,
+            extension: "mp4"
+        },
+        fileMetadata: fileMetadata
+    }
+}

src/processing/services/snapchat.js

@@ -0,0 +1,126 @@
+import { extract, normalizeURL } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+import { getRedirectingURL } from "../../misc/utils.js";
+
+const SPOTLIGHT_VIDEO_REGEX = /<link data-react-helmet="true" rel="preload" href="([^"]+)" as="video"\/>/;
+const NEXT_DATA_REGEX = /<script id="__NEXT_DATA__" type="application\/json">({.+})<\/script><\/body><\/html>$/;
+
+async function getSpotlight(id) {
+    const html = await fetch(`https://www.snapchat.com/spotlight/${id}`, {
+        headers: { 'user-agent': genericUserAgent }
+    }).then((r) => r.text()).catch(() => null);
+
+    if (!html) {
+        return { error: "fetch.fail" };
+    }
+
+    const videoURL = html.match(SPOTLIGHT_VIDEO_REGEX)?.[1];
+
+    if (videoURL && new URL(videoURL).hostname.endsWith(".sc-cdn.net")) {
+        return {
+            urls: videoURL,
+            filename: `snapchat_${id}.mp4`,
+            audioFilename: `snapchat_${id}_audio`
+        }
+    }
+}
+
+async function getStory(username, storyId, alwaysProxy) {
+    const html = await fetch(
+        `https://www.snapchat.com/add/${username}${storyId ? `/${storyId}` : ''}`,
+        { headers: { 'user-agent': genericUserAgent } }
+    )
+    .then((r) => r.text())
+    .catch(() => null);
+
+    if (!html) {
+        return { error: "fetch.fail" };
+    }
+
+    const nextDataString = html.match(NEXT_DATA_REGEX)?.[1];
+    if (nextDataString) {
+        const data = JSON.parse(nextDataString);
+        const storyIdParam = data.query.profileParams[1];
+
+        if (storyIdParam && data.props.pageProps.story) {
+            const story = data.props.pageProps.story.snapList.find((snap) => snap.snapId.value === storyIdParam);
+            if (story) {
+                if (story.snapMediaType === 0) {
+                    return {
+                        urls: story.snapUrls.mediaUrl,
+                        filename: `snapchat_${storyId}.jpg`,
+                        isPhoto: true
+                    }
+                }
+
+                return {
+                    urls: story.snapUrls.mediaUrl,
+                    filename: `snapchat_${storyId}.mp4`,
+                    audioFilename: `snapchat_${storyId}_audio`
+                }
+            }
+        }
+
+        const defaultStory = data.props.pageProps.curatedHighlights[0];
+        if (defaultStory) {
+            return {
+                picker: defaultStory.snapList.map(snap => {
+                    const snapType = snap.snapMediaType === 0 ? "photo" : "video";
+                    const snapExt = snapType === "video" ? "mp4" : "jpg";
+                    let snapUrl = snap.snapUrls.mediaUrl;
+
+                    const proxy = createStream({
+                        service: "snapchat",
+                        type: "proxy",
+                        url: snapUrl,
+                        filename: `snapchat_${username}_${snap.timestampInSec.value}.${snapExt}`,
+                    });
+
+                    let thumbProxy;
+                    if (snapType === "video") thumbProxy = createStream({
+                        service: "snapchat",
+                        type: "proxy",
+                        url: snap.snapUrls.mediaPreviewUrl.value,
+                    });
+
+                    if (alwaysProxy) snapUrl = proxy;
+
+                    return {
+                        type: snapType,
+                        url: snapUrl,
+                        thumb: thumbProxy || proxy,
+                    }
+                })
+            }
+        }
+    }
+}
+
+export default async function (obj) {
+    let params = obj;
+    if (obj.shortLink) {
+        const link = await getRedirectingURL(`https://t.snapchat.com/${obj.shortLink}`);
+
+        if (!link?.startsWith('https://www.snapchat.com/')) {
+            return { error: "fetch.short_link" };
+        }
+
+        const extractResult = extract(normalizeURL(link));
+        if (extractResult?.host !== 'snapchat') {
+            return { error: "fetch.short_link" };
+        }
+
+        params = extractResult.patternMatch;
+    }
+
+    if (params.spotlightId) {
+        const result = await getSpotlight(params.spotlightId);
+        if (result) return result;
+    } else if (params.username) {
+        const result = await getStory(params.username, params.storyId, obj.alwaysProxy);
+        if (result) return result;
+    }
+
+    return { error: "fetch.fail" };
+}

src/processing/services/soundcloud.js

@@ -0,0 +1,122 @@
+import { env } from "../../config.js";
+
+const cachedID = {
+    version: '',
+    id: ''
+}
+
+async function findClientID() {
+    try {
+        let sc = await fetch('https://soundcloud.com/').then(r => r.text()).catch(() => {});
+        let scVersion = String(sc.match(/<script>window\.__sc_version="[0-9]{10}"<\/script>/)[0].match(/[0-9]{10}/));
+
+        if (cachedID.version === scVersion) return cachedID.id;
+
+        let scripts = sc.matchAll(/<script.+src="(.+)">/g);
+        let clientid;
+        for (let script of scripts) {
+            let url = script[1];
+
+            if (!url?.startsWith('https://a-v2.sndcdn.com/')) {
+                return;
+            }
+
+            let scrf = await fetch(url).then(r => r.text()).catch(() => {});
+            let id = scrf.match(/\("client_id=[A-Za-z0-9]{32}"\)/);
+
+            if (id && typeof id[0] === 'string') {
+                clientid = id[0].match(/[A-Za-z0-9]{32}/)[0];
+                break;
+            }
+        }
+        cachedID.version = scVersion;
+        cachedID.id = clientid;
+
+        return clientid;
+    } catch {}
+}
+
+export default async function(obj) {
+    let clientId = await findClientID();
+    if (!clientId) return { error: "fetch.fail" };
+
+    let link;
+    if (obj.url.hostname === 'on.soundcloud.com' && obj.shortLink) {
+        link = await fetch(`https://on.soundcloud.com/${obj.shortLink}/`, { redirect: "manual" }).then(r => {
+            if (r.status === 302 && r.headers.get("location").startsWith("https://soundcloud.com/")) {
+                return r.headers.get("location").split('?', 1)[0]
+            }
+        }).catch(() => {});
+    }
+
+    if (!link && obj.author && obj.song) {
+        link = `https://soundcloud.com/${obj.author}/${obj.song}${obj.accessKey ? `/s-${obj.accessKey}` : ''}`
+    }
+
+    if (!link && obj.shortLink) return { error: "fetch.short_link" };
+    if (!link) return { error: "link.unsupported" };
+
+    let json = await fetch(`https://api-v2.soundcloud.com/resolve?url=${link}&client_id=${clientId}`)
+                     .then(r => r.status === 200 ? r.json() : false)
+                     .catch(() => {});
+
+    if (!json) return { error: "fetch.fail" };
+
+    if (json?.policy === "BLOCK") {
+        return { error: "content.region" };
+    }
+
+    if (json?.policy === "SNIP") {
+        return { error: "content.paid" };
+    }
+
+    if (!json?.media?.transcodings || !json?.media?.transcodings.length === 0) {
+        return { error: "fetch.empty" };
+    }
+
+    let bestAudio = "opus",
+        selectedStream = json.media.transcodings.find(v => v.preset === "opus_0_0"),
+        mp3Media = json.media.transcodings.find(v => v.preset === "mp3_0_0");
+
+    // use mp3 if present if user prefers it or if opus isn't available
+    if (mp3Media && (obj.format === "mp3" || !selectedStream)) {
+        selectedStream = mp3Media;
+        bestAudio = "mp3"
+    }
+
+    if (!selectedStream) {
+        return { error: "fetch.empty" };
+    }
+
+    let fileUrlBase = selectedStream.url;
+    let fileUrl = `${fileUrlBase}${fileUrlBase.includes("?") ? "&" : "?"}client_id=${clientId}&track_authorization=${json.track_authorization}`;
+
+    if (!fileUrl.startsWith("https://api-v2.soundcloud.com/media/soundcloud:tracks:"))
+        return { error: "fetch.empty" };
+
+    if (json.duration > env.durationLimit * 1000) {
+        return { error: "content.too_long" };
+    }
+
+    let file = await fetch(fileUrl)
+                     .then(async r => (await r.json()).url)
+                     .catch(() => {});
+    if (!file) return { error: "fetch.empty" };
+
+    let fileMetadata = {
+        title: json.title.trim(),
+        artist: json.user.username.trim(),
+    }
+
+    return {
+        urls: file,
+        filenameAttributes: {
+            service: "soundcloud",
+            id: json.id,
+            title: fileMetadata.title,
+            author: fileMetadata.artist
+        },
+        bestAudio,
+        fileMetadata
+    }
+}

src/processing/services/streamable.js

@@ -0,0 +1,22 @@
+export default async function(obj) {
+    let video = await fetch(`https://api.streamable.com/videos/${obj.id}`)
+                      .then(r => r.status === 200 ? r.json() : false)
+                      .catch(() => {});
+
+    if (!video) return { error: "fetch.empty" };
+
+    let best = video.files["mp4-mobile"];
+    if (video.files.mp4 && (obj.isAudioOnly || obj.quality === "max" || obj.quality >= 720)) {
+        best = video.files.mp4;
+    }
+
+    if (best) return {
+        urls: best.url,
+        filename: `streamable_${obj.id}_${best.width}x${best.height}.mp4`,
+        audioFilename: `streamable_${obj.id}_audio`,
+        fileMetadata: {
+            title: video.title
+        }
+    }
+    return { error: "fetch.fail" }
+}

src/processing/services/tiktok.js

@@ -0,0 +1,153 @@
+import Cookie from "../cookie/cookie.js";
+
+import { extract } from "../url.js";
+import { genericUserAgent } from "../../config.js";
+import { updateCookie } from "../cookie/manager.js";
+import { createStream } from "../../stream/manage.js";
+
+const shortDomain = "https://vt.tiktok.com/";
+
+export default async function(obj) {
+    const cookie = new Cookie({});
+    let postId = obj.postId;
+
+    if (!postId) {
+        let html = await fetch(`${shortDomain}${obj.shortLink}`, {
+            redirect: "manual",
+            headers: {
+                "user-agent": genericUserAgent.split(' Chrome/1')[0]
+            }
+        }).then(r => r.text()).catch(() => {});
+
+        if (!html) return { error: "fetch.fail" };
+
+        if (html.startsWith('<a href="https://')) {
+            const extractedURL = html.split('<a href="')[1].split('?')[0];
+            const { patternMatch } = extract(extractedURL);
+            postId = patternMatch.postId;
+        }
+    }
+    if (!postId) return { error: "fetch.short_link" };
+
+    // should always be /video/, even for photos
+    const res = await fetch(`https://tiktok.com/@i/video/${postId}`, {
+        headers: {
+            "user-agent": genericUserAgent,
+            cookie,
+        }
+    })
+    updateCookie(cookie, res.headers);
+
+    const html = await res.text();
+
+    let detail;
+    try {
+        const json = html
+            .split('<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">')[1]
+            .split('</script>')[0];
+
+        const data = JSON.parse(json);
+        const videoDetail = data["__DEFAULT_SCOPE__"]["webapp.video-detail"];
+
+        if (!videoDetail) throw "no video detail found";
+
+        // status_deleted or etc
+        if (videoDetail.statusMsg) {
+            return { error: "content.post.unavailable"};
+        }
+
+        detail = videoDetail?.itemInfo?.itemStruct;
+    } catch {
+        return { error: "fetch.fail" };
+    }
+
+    if (detail.isContentClassified) {
+        return { error: "content.post.age" };
+    }
+
+    if (!detail.author) {
+        return { error: "fetch.empty" };
+    }
+
+    let video, videoFilename, audioFilename, audio, images,
+        filenameBase = `tiktok_${detail.author?.uniqueId}_${postId}`,
+        bestAudio; // will get defaulted to m4a later on in match-action
+
+    images = detail.imagePost?.images;
+
+    let playAddr = detail.video?.playAddr;
+
+    if (obj.h265) {
+        const h265PlayAddr = detail?.video?.bitrateInfo?.find(b => b.CodecType.includes("h265"))?.PlayAddr.UrlList[0]
+        playAddr = h265PlayAddr || playAddr
+    }
+
+    if (!obj.isAudioOnly && !images) {
+        video = playAddr;
+        videoFilename = `${filenameBase}.mp4`;
+    } else {
+        audio = playAddr;
+        audioFilename = `${filenameBase}_audio`;
+
+        if (obj.fullAudio || !audio) {
+            audio = detail.music.playUrl;
+            audioFilename += `_original`
+        }
+        if (audio.includes("mime_type=audio_mpeg")) bestAudio = 'mp3';
+    }
+
+    if (video) {
+        return {
+            urls: video,
+            filename: videoFilename,
+            headers: { cookie }
+        }
+    }
+
+    if (images && obj.isAudioOnly) {
+        return {
+            urls: audio,
+            audioFilename: audioFilename,
+            isAudioOnly: true,
+            bestAudio,
+            headers: { cookie }
+        }
+    }
+
+    if (images) {
+        let imageLinks = images
+            .map(i => i.imageURL.urlList.find(p => p.includes(".jpeg?")))
+            .map((url, i) => {
+                if (obj.alwaysProxy) url = createStream({
+                    service: "tiktok",
+                    type: "proxy",
+                    url,
+                    filename: `${filenameBase}_photo_${i + 1}.jpg`
+                })
+
+                return {
+                    type: "photo",
+                    url
+                }
+            });
+
+        return {
+            picker: imageLinks,
+            urls: audio,
+            audioFilename: audioFilename,
+            isAudioOnly: true,
+            bestAudio,
+            headers: { cookie }
+        }
+    }
+
+    if (audio) {
+        return {
+            urls: audio,
+            audioFilename: audioFilename,
+            isAudioOnly: true,
+            bestAudio,
+            headers: { cookie }
+        }
+    }
+}

src/processing/services/tumblr.js

@@ -0,0 +1,71 @@
+import psl from "@imput/psl";
+
+const API_KEY = 'jrsCWX1XDuVxAFO4GkK147syAoN8BJZ5voz8tS80bPcj26Vc5Z';
+const API_BASE = 'https://api-http2.tumblr.com';
+
+function request(domain, id) {
+    const url = new URL(`/v2/blog/${domain}/posts/${id}/permalink`, API_BASE);
+    url.searchParams.set('api_key', API_KEY);
+    url.searchParams.set('fields[blogs]', 'uuid,name,avatar,?description,?can_message,?can_be_followed,?is_adult,?reply_conditions,'
+                                            + '?theme,?title,?url,?is_blocked_from_primary,?placement_id,?primary,?updated,?followed,'
+                                            + '?ask,?can_subscribe,?paywall_access,?subscription_plan,?is_blogless_advertiser,?tumblrmart_accessories');
+
+    return fetch(url, {
+        headers: {
+            'User-Agent': 'Tumblr/iPhone/33.3/333010/17.3.1/tumblr',
+            'X-Version': 'iPhone/33.3/333010/17.3.1/tumblr'
+        }
+    }).then(a => a.json()).catch(() => {});
+}
+
+export default async function(input) {
+    let { subdomain } = psl.parse(input.url.hostname);
+
+    if (subdomain?.includes('.')) {
+        return { error: "link.unsupported" };
+    } else if (subdomain === 'www' || subdomain === 'at') {
+        subdomain = undefined
+    }
+
+    const domain = `${subdomain ?? input.user}.tumblr.com`;
+    const data = await request(domain, input.id);
+
+    const element = data?.response?.timeline?.elements?.[0];
+    if (!element) return { error: "fetch.empty" };
+
+    const contents = [
+        ...element.content,
+        ...element?.trail?.map(t => t.content).flat()
+    ]
+
+    const audio = contents.find(c => c.type === 'audio');
+    if (audio && audio.provider === 'tumblr') {
+        const fileMetadata = {
+            title: audio?.title,
+            artist: audio?.artist
+        };
+
+        return {
+            urls: audio.media.url,
+            filenameAttributes: {
+                service: 'tumblr',
+                id: input.id,
+                title: fileMetadata.title,
+                author: fileMetadata.artist
+            },
+            isAudioOnly: true,
+            bestAudio: "mp3",
+        }
+    }
+
+    const video = contents.find(c => c.type === 'video');
+    if (video && video.provider === 'tumblr') {
+        return {
+            urls: video.media.url,
+            filename: `tumblr_${input.id}.mp4`,
+            audioFilename: `tumblr_${input.id}_audio`
+        }
+    }
+
+    return { error: "link.unsupported" }
+}

src/processing/services/twitch.js

@@ -0,0 +1,89 @@
+import { env } from "../../config.js";
+
+const gqlURL = "https://gql.twitch.tv/gql";
+const clientIdHead = { "client-id": "kimne78kx3ncx6brgo4mv6wki5h1ko" };
+
+export default async function (obj) {
+    const req_metadata = await fetch(gqlURL, {
+        method: "POST",
+        headers: clientIdHead,
+        body: JSON.stringify({
+            query: `{
+            clip(slug: "${obj.clipId}") {
+                broadcaster {
+                    login
+                }
+                createdAt
+                curator {
+                    login
+                }
+                durationSeconds
+                id
+                medium: thumbnailURL(width: 480, height: 272)
+                title
+                videoQualities {
+                    quality
+                    sourceURL
+                }
+            }
+        }`
+        })
+    }).then(r => r.status === 200 ? r.json() : false).catch(() => {});
+
+    if (!req_metadata) return { error: "fetch.fail" };
+
+    const clipMetadata = req_metadata.data.clip;
+
+    if (clipMetadata.durationSeconds > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+    if (!clipMetadata.videoQualities || !clipMetadata.broadcaster) {
+        return { error: "fetch.empty" };
+    }
+
+    const req_token = await fetch(gqlURL, {
+        method: "POST",
+        headers: clientIdHead,
+        body: JSON.stringify([
+            {
+                "operationName": "VideoAccessToken_Clip",
+                "variables": {
+                    "slug": obj.clipId
+                },
+                "extensions": {
+                    "persistedQuery": {
+                        "version": 1,
+                        "sha256Hash": "36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11"
+                    }
+                }
+            }
+        ])
+    }).then(r => r.status === 200 ? r.json() : false).catch(() => {});
+
+    if (!req_token) return { error: "fetch.fail" };
+
+    const formats = clipMetadata.videoQualities;
+    const format = formats.find(f => f.quality === obj.quality) || formats[0];
+
+    return {
+        type: "proxy",
+        urls: `${format.sourceURL}?${new URLSearchParams({
+            sig: req_token[0].data.clip.playbackAccessToken.signature,
+            token: req_token[0].data.clip.playbackAccessToken.value
+        })}`,
+        fileMetadata: {
+            title: clipMetadata.title.trim(),
+            artist: `Twitch Clip by @${clipMetadata.broadcaster.login}, clipped by @${clipMetadata.curator.login}`,
+        },
+        filenameAttributes: {
+            service: "twitch",
+            id: clipMetadata.id,
+            title: clipMetadata.title.trim(),
+            author: `${clipMetadata.broadcaster.login}, clipped by ${clipMetadata.curator.login}`,
+            qualityLabel: `${format.quality}p`,
+            extension: 'mp4'
+        },
+        filename: `twitchclip_${clipMetadata.id}_${format.quality}p.mp4`,
+        audioFilename: `twitchclip_${clipMetadata.id}_audio`
+    }
+}

src/processing/services/twitter.js

@@ -0,0 +1,235 @@
+import { genericUserAgent } from "../../config.js";
+import { createStream } from "../../stream/manage.js";
+import { getCookie, updateCookie } from "../cookie/manager.js";
+
+const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';
+const tokenURL = 'https://api.x.com/1.1/guest/activate.json';
+
+const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});
+
+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});
+
+const commonHeaders = {
+    "user-agent": genericUserAgent,
+    "authorization": "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA",
+    "x-twitter-client-language": "en",
+    "x-twitter-active-user": "yes",
+    "accept-language": "en"
+}
+
+// fix all videos affected by the container bug in twitter muxer (took them over two weeks to fix it????)
+const TWITTER_EPOCH = 1288834974657n;
+const badContainerStart = new Date(1701446400000);
+const badContainerEnd = new Date(1702605600000);
+
+function needsFixing(media) {
+    const representativeId = media.source_status_id_str ?? media.id_str;
+    const mediaTimestamp = new Date(
+        Number((BigInt(representativeId) >> 22n) + TWITTER_EPOCH)
+    );
+    return mediaTimestamp > badContainerStart && mediaTimestamp < badContainerEnd
+}
+
+function bestQuality(arr) {
+    return arr.filter(v => v.content_type === "video/mp4")
+        .reduce((a, b) => Number(a?.bitrate) > Number(b?.bitrate) ? a : b)
+        .url
+}
+
+let _cachedToken;
+const getGuestToken = async (dispatcher, forceReload = false) => {
+    if (_cachedToken && !forceReload) {
+        return _cachedToken;
+    }
+
+    const tokenResponse = await fetch(tokenURL, {
+        method: 'POST',
+        headers: commonHeaders,
+        dispatcher
+    }).then(r => r.status === 200 && r.json()).catch(() => {})
+
+    if (tokenResponse?.guest_token) {
+        return _cachedToken = tokenResponse.guest_token
+    }
+}
+
+const requestTweet = async(dispatcher, tweetId, token, cookie) => {
+    const graphqlTweetURL = new URL(graphqlURL);
+
+    let headers = {
+        ...commonHeaders,
+        'content-type': 'application/json',
+        'x-guest-token': token,
+        cookie: `guest_id=${encodeURIComponent(`v1:${token}`)}`
+    }
+
+    if (cookie) {
+        headers = {
+            ...commonHeaders,
+            'content-type': 'application/json',
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'x-csrf-token': cookie.values().ct0,
+            cookie
+        }
+    }
+
+    graphqlTweetURL.searchParams.set('variables',
+        JSON.stringify({
+            tweetId,
+            withCommunity: false,
+            includePromotedContent: false,
+            withVoice: false
+        })
+    );
+    graphqlTweetURL.searchParams.set('features', tweetFeatures);
+    graphqlTweetURL.searchParams.set('fieldToggles', tweetFieldToggles);
+
+    let result = await fetch(graphqlTweetURL, { headers, dispatcher });
+    updateCookie(cookie, result.headers);
+
+    // we might have been missing the `ct0` cookie, retry
+    if (result.status === 403 && result.headers.get('set-cookie')) {
+        result = await fetch(graphqlTweetURL, {
+            headers: {
+                ...headers,
+                'x-csrf-token': cookie.values().ct0
+            },
+            dispatcher
+        });
+    }
+
+    return result
+}
+
+export default async function({ id, index, toGif, dispatcher, alwaysProxy }) {
+    const cookie = await getCookie('twitter');
+
+    let guestToken = await getGuestToken(dispatcher);
+    if (!guestToken) return { error: "fetch.fail" };
+
+    let tweet = await requestTweet(dispatcher, id, guestToken);
+
+    // get new token & retry if old one expired
+    if ([403, 429].includes(tweet.status)) {
+        guestToken = await getGuestToken(dispatcher, true);
+        tweet = await requestTweet(dispatcher, id, guestToken)
+    }
+
+    tweet = await tweet.json();
+
+    let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+
+    if (!tweetTypename) {
+        return { error: "fetch.empty" }
+    }
+
+    if (tweetTypename === "TweetUnavailable") {
+        const reason = tweet?.data?.tweetResult?.result?.reason;
+        switch(reason) {
+            case "Protected":
+                return { error: "content.post.private" }
+            case "NsfwLoggedOut":
+                if (cookie) {
+                    tweet = await requestTweet(dispatcher, id, guestToken, cookie);
+                    tweet = await tweet.json();
+                    tweetTypename = tweet?.data?.tweetResult?.result?.__typename;
+                } else return { error: "content.post.age" }
+        }
+    }
+
+    if (!["Tweet", "TweetWithVisibilityResults"].includes(tweetTypename)) {
+        return { error: "content.post.unavailable" }
+    }
+
+    let tweetResult = tweet.data.tweetResult.result,
+        baseTweet = tweetResult.legacy,
+        repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;
+
+    if (tweetTypename === "TweetWithVisibilityResults") {
+        baseTweet = tweetResult.tweet.legacy;
+        repostedTweet = baseTweet?.retweeted_status_result?.result.tweet.legacy.extended_entities;
+    }
+
+    let media = (repostedTweet?.media || baseTweet?.extended_entities?.media);
+
+    // check if there's a video at given index (/video/<index>)
+    if (index >= 0 && index < media?.length) {
+        media = [media[index]]
+    }
+
+    const getFileExt = (url) => new URL(url).pathname.split(".", 2)[1];
+
+    const proxyMedia = (url, filename) => createStream({
+        service: "twitter",
+        type: "proxy",
+        url, filename,
+    })
+
+    switch (media?.length) {
+        case undefined:
+        case 0:
+            return {
+                error: "fetch.empty"
+            }
+        case 1:
+            if (media[0].type === "photo") {
+                return {
+                    type: "proxy",
+                    isPhoto: true,
+                    filename: `twitter_${id}.${getFileExt(media[0].media_url_https)}`,
+                    urls: `${media[0].media_url_https}?name=4096x4096`
+                }
+            }
+
+            return {
+                type: needsFixing(media[0]) ? "remux" : "proxy",
+                urls: bestQuality(media[0].video_info.variants),
+                filename: `twitter_${id}.mp4`,
+                audioFilename: `twitter_${id}_audio`,
+                isGif: media[0].type === "animated_gif"
+            }
+        default:
+            const proxyThumb = (url, i) =>
+                proxyMedia(url, `twitter_${id}_${i + 1}.${getFileExt(url)}`);
+
+            const picker = media.map((content, i) => {
+                if (content.type === "photo") {
+                    let url = `${content.media_url_https}?name=4096x4096`;
+                    let proxiedImage = proxyThumb(url, i);
+
+                    if (alwaysProxy) url = proxiedImage;
+
+                    return {
+                        type: "photo",
+                        url,
+                        thumb: proxiedImage,
+                    }
+                }
+
+                let url = bestQuality(content.video_info.variants);
+                const shouldRenderGif = content.type === "animated_gif" && toGif;
+                const videoFilename = `twitter_${id}_${i + 1}.${shouldRenderGif ? "gif" : "mp4"}`;
+
+                let type = "video";
+                if (shouldRenderGif) type = "gif";
+
+                if (needsFixing(content) || shouldRenderGif) {
+                    url = createStream({
+                        service: "twitter",
+                        type: shouldRenderGif ? "gif" : "remux",
+                        url,
+                        filename: videoFilename,
+                    })
+                } else if (alwaysProxy) {
+                    url = proxyMedia(url, videoFilename);
+                }
+
+                return {
+                    type,
+                    url,
+                    thumb: proxyThumb(content.media_url_https, i),
+                }
+            });
+            return { picker };
+    }
+}

src/processing/services/vimeo.js

@@ -0,0 +1,170 @@
+import HLS from "hls-parser";
+import { env } from "../../config.js";
+import { merge } from '../../misc/utils.js';
+
+const resolutionMatch = {
+    "3840": 2160,
+    "2732": 1440,
+    "2560": 1440,
+    "2048": 1080,
+    "1920": 1080,
+    "1366": 720,
+    "1280": 720,
+    "960": 480,
+    "640": 360,
+    "426": 240
+}
+
+const requestApiInfo = (videoId, password) => {
+    if (password) {
+        videoId += `:${password}`
+    }
+
+    return fetch(
+        `https://api.vimeo.com/videos/${videoId}`,
+        {
+            headers: {
+                Accept: 'application/vnd.vimeo.*+json; version=3.4.2',
+                'User-Agent': 'Vimeo/10.19.0 (com.vimeo; build:101900.57.0; iOS 17.5.1) Alamofire/5.9.0 VimeoNetworking/5.0.0',
+                Authorization: 'Basic MTMxNzViY2Y0NDE0YTQ5YzhjZTc0YmU0NjVjNDQxYzNkYWVjOWRlOTpHKzRvMmgzVUh4UkxjdU5FRW80cDNDbDhDWGR5dVJLNUJZZ055dHBHTTB4V1VzaG41bEx1a2hiN0NWYWNUcldSSW53dzRUdFRYZlJEZmFoTTArOTBUZkJHS3R4V2llYU04Qnl1bERSWWxUdXRidjNqR2J4SHFpVmtFSUcyRktuQw==',
+                'Accept-Language': 'en'
+            }
+        }
+    )
+    .then(a => a.json())
+    .catch(() => {});
+}
+
+const compareQuality = (rendition, requestedQuality) => {
+    const quality = parseInt(rendition);
+    return Math.abs(quality - requestedQuality);
+}
+
+const getDirectLink = (data, quality) => {
+    if (!data.files) return;
+
+    const match = data.files
+        .filter(f => f.rendition?.endsWith('p'))
+        .reduce((prev, next) => {
+            const delta = {
+                prev: compareQuality(prev.rendition, quality),
+                next: compareQuality(next.rendition, quality)
+            };
+
+            return delta.prev < delta.next ? prev : next;
+        });
+
+    if (!match) return;
+
+    return {
+        urls: match.link,
+        filenameAttributes: {
+            resolution: `${match.width}x${match.height}`,
+            qualityLabel: match.rendition,
+            extension: "mp4"
+        },
+        bestAudio: "mp3",
+    }
+}
+
+const getHLS = async (configURL, obj) => {
+    if (!configURL) return;
+
+    const api = await fetch(configURL)
+                    .then(r => r.json())
+                    .catch(() => {});
+    if (!api) return { error: "fetch.fail" };
+
+    if (api.video?.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
+    const urlMasterHLS = api.request?.files?.hls?.cdns?.akfire_interconnect_quic?.url;
+    if (!urlMasterHLS) return { error: "fetch.fail" };
+
+    const masterHLS = await fetch(urlMasterHLS)
+                            .then(r => r.text())
+                            .catch(() => {});
+
+    if (!masterHLS) return { error: "fetch.fail" };
+
+    const variants = HLS.parse(masterHLS)?.variants?.sort(
+        (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+    );
+    if (!variants || variants.length === 0) return { error: "fetch.empty" };
+
+    let bestQuality;
+
+    if (obj.quality < resolutionMatch[variants[0]?.resolution?.width]) {
+        bestQuality = variants.find(v =>
+            (obj.quality === resolutionMatch[v.resolution.width])
+        );
+    }
+
+    if (!bestQuality) bestQuality = variants[0];
+
+    const expandLink = (path) => {
+        return new URL(path, urlMasterHLS).toString();
+    };
+
+    let urls = expandLink(bestQuality.uri);
+
+    const audioPath = bestQuality?.audio[0]?.uri;
+    if (audioPath) {
+        urls = [
+            urls,
+            expandLink(audioPath)
+        ]
+    } else if (obj.isAudioOnly) {
+        return { error: "fetch.empty" };
+    }
+
+    return {
+        urls,
+        isHLS: true,
+        filenameAttributes: {
+            resolution: `${bestQuality.resolution.width}x${bestQuality.resolution.height}`,
+            qualityLabel: `${resolutionMatch[bestQuality.resolution.width]}p`,
+            extension: "mp4"
+        },
+        bestAudio: "mp3",
+    }
+}
+
+export default async function(obj) {
+    let quality = obj.quality === "max" ? 9000 : Number(obj.quality);
+    if (quality < 240) quality = 240;
+    if (!quality || obj.isAudioOnly) quality = 9000;
+
+    const info = await requestApiInfo(obj.id, obj.password);
+    let response;
+
+    if (obj.isAudioOnly) {
+        response = await getHLS(info.config_url, { ...obj, quality });
+    }
+
+    if (!response) response = getDirectLink(info, quality);
+    if (!response) response = { error: "fetch.empty" };
+
+    if (response.error) {
+        return response;
+    }
+
+    const fileMetadata = {
+        title: info.name,
+        artist: info.user.name,
+    };
+
+    return merge(
+        {
+            fileMetadata,
+            filenameAttributes: {
+                service: "vimeo",
+                id: obj.id,
+                title: fileMetadata.title,
+                author: fileMetadata.artist,
+            }
+        },
+        response
+    );
+}

src/processing/services/vk.js

@@ -0,0 +1,140 @@
+import { env } from "../../config.js";
+
+const resolutions = ["2160", "1440", "1080", "720", "480", "360", "240", "144"];
+
+const oauthUrl = "https://oauth.vk.com/oauth/get_anonym_token";
+const apiUrl = "https://api.vk.com/method";
+
+const clientId = "51552953";
+const clientSecret = "qgr0yWwXCrsxA1jnRtRX";
+
+// used in stream/shared.js for accessing media files
+export const vkClientAgent = "com.vk.vkvideo.prod/822 (iPhone, iOS 16.7.7, iPhone10,4, Scale/2.0) SAK/1.119";
+
+const cachedToken = {
+    token: "",
+    expiry: 0,
+    device_id: "",
+};
+
+const getToken = async () => {
+    if (cachedToken.expiry - 10 > Math.floor(new Date().getTime() / 1000)) {
+        return cachedToken.token;
+    }
+
+    const randomDeviceId = crypto.randomUUID().toUpperCase();
+
+    const anonymOauth = new URL(oauthUrl);
+    anonymOauth.searchParams.set("client_id", clientId);
+    anonymOauth.searchParams.set("client_secret", clientSecret);
+    anonymOauth.searchParams.set("device_id", randomDeviceId);
+
+    const oauthResponse = await fetch(anonymOauth.toString(), {
+        headers: {
+            "user-agent": vkClientAgent,
+        }
+    }).then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    if (!oauthResponse) return;
+
+    if (oauthResponse?.token && oauthResponse?.expired_at && typeof oauthResponse?.expired_at === "number") {
+        cachedToken.token = oauthResponse.token;
+        cachedToken.expiry = oauthResponse.expired_at;
+        cachedToken.device_id = randomDeviceId;
+    }
+
+    if (!cachedToken.token) return;
+
+    return cachedToken.token;
+}
+
+const getVideo = async (ownerId, videoId, accessKey) => {
+    const video = await fetch(`${apiUrl}/video.get`, {
+        method: "POST",
+        headers: {
+            "content-type": "application/x-www-form-urlencoded; charset=utf-8",
+            "user-agent": vkClientAgent,
+        },
+        body: new URLSearchParams({
+            anonymous_token: cachedToken.token,
+            device_id: cachedToken.device_id,
+            lang: "en",
+            v: "5.244",
+            videos: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`
+        }).toString()
+    })
+    .then(r => {
+        if (r.status === 200) {
+            return r.json();
+        }
+    });
+
+    return video;
+}
+
+export default async function ({ ownerId, videoId, accessKey, quality }) {
+    const token = await getToken();
+    if (!token) return { error: "fetch.fail" };
+
+    const videoGet = await getVideo(ownerId, videoId, accessKey);
+
+    if (!videoGet || !videoGet.response || videoGet.response.items.length !== 1) {
+        return { error: "fetch.empty" };
+    }
+
+    const video = videoGet.response.items[0];
+
+    if (video.restriction) {
+        const title = video.restriction.title;
+        if (title.endsWith("country") || title.endsWith("region.")) {
+            return { error: "content.video.region" };
+        }
+        if (title === "Processing video") {
+            return { error: "fetch.empty" };
+        }
+        return { error: "content.video.unavailable" };
+    }
+
+    if (!video.files || !video.duration) {
+        return { error: "fetch.fail" };
+    }
+
+    if (video.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
+    const userQuality = quality === "max" ? resolutions[0] : quality;
+    let pickedQuality;
+
+    for (const resolution of resolutions) {
+        if (video.files[`mp4_${resolution}`] && +resolution <= +userQuality) {
+            pickedQuality = resolution;
+            break
+        }
+    }
+
+    const url = video.files[`mp4_${pickedQuality}`];
+
+    if (!url) return { error: "fetch.fail" };
+
+    const fileMetadata = {
+        title: video.title.trim(),
+    }
+
+    return {
+        urls: url,
+        fileMetadata,
+        filenameAttributes: {
+            service: "vk",
+            id: `${ownerId}_${videoId}${accessKey ? `_${accessKey}` : ''}`,
+            title: fileMetadata.title,
+            resolution: `${pickedQuality}p`,
+            qualityLabel: `${pickedQuality}p`,
+            extension: "mp4"
+        }
+    }
+}

src/processing/services/youtube.js

@@ -0,0 +1,488 @@
+import HLS from "hls-parser";
+
+import { fetch } from "undici";
+import { Innertube, Session } from "youtubei.js";
+
+import { env } from "../../config.js";
+import { getCookie, updateCookieValues } from "../cookie/manager.js";
+
+const PLAYER_REFRESH_PERIOD = 1000 * 60 * 15; // ms
+
+let innertube, lastRefreshedAt;
+
+const codecList = {
+    h264: {
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    av1: {
+        videoCodec: "av01",
+        audioCodec: "opus",
+        container: "webm"
+    },
+    vp9: {
+        videoCodec: "vp9",
+        audioCodec: "opus",
+        container: "webm"
+    }
+}
+
+const hlsCodecList = {
+    h264: {
+        videoCodec: "avc1",
+        audioCodec: "mp4a",
+        container: "mp4"
+    },
+    vp9: {
+        videoCodec: "vp09",
+        audioCodec: "mp4a",
+        container: "webm"
+    }
+}
+
+const videoQualities = [144, 240, 360, 480, 720, 1080, 1440, 2160, 4320];
+
+const transformSessionData = (cookie) => {
+    if (!cookie)
+        return;
+
+    const values = { ...cookie.values() };
+    const REQUIRED_VALUES = [ 'access_token', 'refresh_token' ];
+
+    if (REQUIRED_VALUES.some(x => typeof values[x] !== 'string')) {
+        return;
+    }
+
+    if (values.expires) {
+        values.expiry_date = values.expires;
+        delete values.expires;
+    } else if (!values.expiry_date) {
+        return;
+    }
+
+    return values;
+}
+
+const cloneInnertube = async (customFetch) => {
+    const shouldRefreshPlayer = lastRefreshedAt + PLAYER_REFRESH_PERIOD < new Date();
+    if (!innertube || shouldRefreshPlayer) {
+        innertube = await Innertube.create({
+            fetch: customFetch,
+            retrieve_player: false,
+        });
+        lastRefreshedAt = +new Date();
+    }
+
+    const session = new Session(
+        innertube.session.context,
+        innertube.session.key,
+        innertube.session.api_version,
+        innertube.session.account_index,
+        innertube.session.player,
+        undefined,
+        customFetch ?? innertube.session.http.fetch,
+        innertube.session.cache
+    );
+
+    const cookie = getCookie('youtube_oauth');
+    const oauthData = transformSessionData(cookie);
+
+    if (!session.logged_in && oauthData) {
+        await session.oauth.init(oauthData);
+        session.logged_in = true;
+    }
+
+    if (session.logged_in) {
+        if (session.oauth.shouldRefreshToken()) {
+            await session.oauth.refreshAccessToken();
+        }
+
+        const cookieValues = cookie.values();
+        const oldExpiry = new Date(cookieValues.expiry_date);
+        const newExpiry = new Date(session.oauth.oauth2_tokens.expiry_date);
+
+        if (oldExpiry.getTime() !== newExpiry.getTime()) {
+            updateCookieValues(cookie, {
+                ...session.oauth.client_id,
+                ...session.oauth.oauth2_tokens,
+                expiry_date: newExpiry.toISOString()
+            });
+        }
+    }
+
+    const yt = new Innertube(session);
+    return yt;
+}
+
+export default async function(o) {
+    let yt;
+    try {
+        yt = await cloneInnertube(
+            (input, init) => fetch(input, {
+                ...init,
+                dispatcher: o.dispatcher
+            })
+        );
+    } catch (e) {
+        if (e.message?.endsWith("decipher algorithm")) {
+            return { error: "youtube.decipher" }
+        } else if (e.message?.includes("refresh access token")) {
+            return { error: "youtube.token_expired" }
+        } else throw e;
+    }
+
+    let useHLS = o.youtubeHLS;
+
+    // HLS playlists don't contain the av1 video format, at least with the iOS client
+    if (useHLS && o.format === "av1") {
+        useHLS = false;
+    }
+
+    let info;
+    try {
+        info = await yt.getBasicInfo(o.id, useHLS ? 'IOS' : 'ANDROID');
+    } catch (e) {
+        if (e?.info) {
+            const errorInfo = JSON.parse(e?.info);
+
+            if (errorInfo?.reason === "This video is private") {
+                return { error: "content.video.private" };
+            }
+            if (["INVALID_ARGUMENT", "UNAUTHENTICATED"].includes(errorInfo?.error?.status)) {
+                return { error: "youtube.api_error" };
+            }
+        }
+
+        if (e?.message === "This video is unavailable") {
+            return { error: "content.video.unavailable" };
+        }
+
+        return { error: "fetch.fail" };
+    }
+
+    if (!info) return { error: "fetch.fail" };
+
+    const playability = info.playability_status;
+    const basicInfo = info.basic_info;
+
+    switch(playability.status) {
+        case "LOGIN_REQUIRED":
+            if (playability.reason.endsWith("bot")) {
+                return { error: "youtube.login" }
+            }
+            if (playability.reason.endsWith("age")) {
+                return { error: "content.video.age" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
+
+        case "UNPLAYABLE":
+            if (playability?.reason?.endsWith("request limit.")) {
+                return { error: "fetch.rate" }
+            }
+            if (playability?.error_screen?.subreason?.text?.endsWith("in your country")) {
+                return { error: "content.video.region" }
+            }
+            if (playability?.error_screen?.reason?.text === "Private video") {
+                return { error: "content.video.private" }
+            }
+            break;
+
+        case "AGE_VERIFICATION_REQUIRED":
+            return { error: "content.video.age" };
+    }
+
+    if (playability.status !== "OK") {
+        return { error: "content.video.unavailable" };
+    }
+
+    if (basicInfo.is_live) {
+        return { error: "content.video.live" };
+    }
+
+    if (basicInfo.duration > env.durationLimit) {
+        return { error: "content.too_long" };
+    }
+
+    // return a critical error if returned video is "Video Not Available"
+    // or a similar stub by youtube
+    if (basicInfo.id !== o.id) {
+        return {
+            error: "fetch.fail",
+            critical: true
+        }
+    }
+
+    const quality = o.quality === "max" ? 9000 : Number(o.quality);
+
+    const normalizeQuality = res => {
+        const shortestSide = res.height > res.width ? res.width : res.height;
+        return videoQualities.find(qual => qual >= shortestSide);
+    }
+
+    let video, audio, dubbedLanguage,
+        codec = o.format || "h264";
+
+    if (useHLS) {
+        const hlsManifest = info.streaming_data.hls_manifest_url;
+
+        if (!hlsManifest) {
+            return { error: "youtube.no_hls_streams" };
+        }
+
+        const fetchedHlsManifest = await fetch(hlsManifest, {
+            dispatcher: o.dispatcher,
+        }).then(r => {
+            if (r.status === 200) {
+                return r.text();
+            } else {
+                throw new Error("couldn't fetch the HLS playlist");
+            }
+        }).catch(() => {});
+
+        if (!fetchedHlsManifest) {
+            return { error: "youtube.no_hls_streams" };
+        }
+
+        const variants = HLS.parse(fetchedHlsManifest).variants.sort(
+            (a, b) => Number(b.bandwidth) - Number(a.bandwidth)
+        );
+
+        if (!variants || variants.length === 0) {
+            return { error: "youtube.no_hls_streams" };
+        }
+
+        const matchHlsCodec = codecs => (
+            codecs.includes(hlsCodecList[codec].videoCodec)
+        );
+
+        const best = variants.find(i => matchHlsCodec(i.codecs));
+
+        const preferred = variants.find(i =>
+            matchHlsCodec(i.codecs) && normalizeQuality(i.resolution) === quality
+        );
+
+        let selected = preferred || best;
+
+        if (!selected) {
+            codec = "h264";
+            selected = variants.find(i => matchHlsCodec(i.codecs));
+        }
+
+        if (!selected) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = selected.audio.find(i => i.isDefault);
+
+        // some videos (mainly those with AI dubs) don't have any tracks marked as default
+        // why? god knows, but we assume that a default track is marked as such in the title
+        if (!audio) {
+            audio = selected.audio.find(i => i.name.endsWith("- original"));
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = selected.audio.find(i =>
+                i.language?.startsWith(o.dubLang)
+            );
+
+            if (dubbedAudio && !dubbedAudio.isDefault) {
+                dubbedLanguage = dubbedAudio.language;
+                audio = dubbedAudio;
+            }
+        }
+
+        selected.audio = [];
+        selected.subtitles = [];
+        video = selected;
+    } else {
+        // i miss typescript so bad
+        const sorted_formats = {
+            h264: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            vp9: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+            av1: {
+                video: [],
+                audio: [],
+                bestVideo: undefined,
+                bestAudio: undefined,
+            },
+        }
+
+        const checkFormat = (format, pCodec) => format.content_length &&
+                (format.mime_type.includes(codecList[pCodec].videoCodec)
+                || format.mime_type.includes(codecList[pCodec].audioCodec));
+
+        // sort formats & weed out bad ones
+        info.streaming_data.adaptive_formats.sort((a, b) =>
+            Number(b.bitrate) - Number(a.bitrate)
+        ).forEach(format => {
+            Object.keys(codecList).forEach(yCodec => {
+                const sorted = sorted_formats[yCodec];
+                const goodFormat = checkFormat(format, yCodec);
+                if (!goodFormat) return;
+
+                if (format.has_video) {
+                    sorted.video.push(format);
+                    if (!sorted.bestVideo) sorted.bestVideo = format;
+                }
+                if (format.has_audio) {
+                    sorted.audio.push(format);
+                    if (!sorted.bestAudio) sorted.bestAudio = format;
+                }
+            })
+        });
+
+        const noBestMedia = () => {
+            const vid = sorted_formats[codec]?.bestVideo;
+            const aud = sorted_formats[codec]?.bestAudio;
+            return (!vid && !o.isAudioOnly) || (!aud && o.isAudioOnly)
+        };
+
+        if (noBestMedia()) {
+            if (codec === "av1") codec = "vp9";
+            else if (codec === "vp9") codec = "av1";
+
+            // if there's no higher quality fallback, then use h264
+            if (noBestMedia()) codec = "h264";
+        }
+
+        // if there's no proper combo of av1, vp9, or h264, then give up
+        if (noBestMedia()) {
+            return { error: "youtube.no_matching_format" };
+        }
+
+        audio = sorted_formats[codec].bestAudio;
+
+        if (audio?.audio_track && !audio?.audio_track?.audio_is_default) {
+            audio = sorted_formats[codec].audio.find(i =>
+                i?.audio_track?.audio_is_default
+            );
+        }
+
+        if (o.dubLang) {
+            const dubbedAudio = sorted_formats[codec].audio.find(i =>
+                i.language?.startsWith(o.dubLang) && i.audio_track
+            );
+
+            if (dubbedAudio && !dubbedAudio?.audio_track?.audio_is_default) {
+                audio = dubbedAudio;
+                dubbedLanguage = dubbedAudio.language;
+            }
+        }
+
+        if (!o.isAudioOnly) {
+            const qual = (i) => {
+                return normalizeQuality({
+                    width: i.width,
+                    height: i.height,
+                })
+            }
+
+            const bestQuality = qual(sorted_formats[codec].bestVideo);
+            const useBestQuality = quality >= bestQuality;
+
+            video = useBestQuality
+                ? sorted_formats[codec].bestVideo
+                : sorted_formats[codec].video.find(i => qual(i) === quality);
+
+            if (!video) video = sorted_formats[codec].bestVideo;
+        }
+    }
+
+    const fileMetadata = {
+        title: basicInfo.title.trim(),
+        artist: basicInfo.author.replace("- Topic", "").trim()
+    }
+
+    if (basicInfo?.short_description?.startsWith("Provided to YouTube by")) {
+        const descItems = basicInfo.short_description.split("\n\n", 5);
+
+        if (descItems.length === 5) {
+            fileMetadata.album = descItems[2];
+            fileMetadata.copyright = descItems[3];
+            if (descItems[4].startsWith("Released on:")) {
+                fileMetadata.date = descItems[4].replace("Released on: ", '').trim();
+            }
+        }
+    }
+
+    const filenameAttributes = {
+        service: "youtube",
+        id: o.id,
+        title: fileMetadata.title,
+        author: fileMetadata.artist,
+        youtubeDubName: dubbedLanguage || false,
+    }
+
+    if (audio && o.isAudioOnly) {
+        let bestAudio = codec === "h264" ? "m4a" : "opus";
+        let urls = audio.url;
+
+        if (useHLS) {
+            bestAudio = "mp3";
+            urls = audio.uri;
+        }
+
+        return {
+            type: "audio",
+            isAudioOnly: true,
+            urls,
+            filenameAttributes,
+            fileMetadata,
+            bestAudio,
+            isHLS: useHLS,
+        }
+    }
+
+    if (video && audio) {
+        let resolution;
+
+        if (useHLS) {
+            resolution = normalizeQuality(video.resolution);
+            filenameAttributes.resolution = `${video.resolution.width}x${video.resolution.height}`;
+            filenameAttributes.extension = hlsCodecList[codec].container;
+
+            video = video.uri;
+            audio = audio.uri;
+        } else {
+            resolution = normalizeQuality({
+                width: video.width,
+                height: video.height,
+            });
+            filenameAttributes.resolution = `${video.width}x${video.height}`;
+            filenameAttributes.extension = codecList[codec].container;
+
+            video = video.url;
+            audio = audio.url;
+        }
+
+        filenameAttributes.qualityLabel = `${resolution}p`;
+        filenameAttributes.youtubeFormat = codec;
+
+        return {
+            type: "merge",
+            urls: [
+                video,
+                audio,
+            ],
+            filenameAttributes,
+            fileMetadata,
+            isHLS: useHLS,
+        }
+    }
+
+    return { error: "youtube.no_matching_format" };
+}

src/processing/url.js

@@ -0,0 +1,213 @@
+import psl from "@imput/psl";
+import { strict as assert } from "node:assert";
+
+import { env } from "../config.js";
+import { services } from "./service-config.js";
+import { friendlyServiceName } from "./service-alias.js";
+
+function aliasURL(url) {
+    assert(url instanceof URL);
+
+    const host = psl.parse(url.hostname);
+    const parts = url.pathname.split('/');
+
+    switch (host.sld) {
+        case "youtube":
+            if (url.pathname.startsWith('/live/') || url.pathname.startsWith('/shorts/')) {
+                url.pathname = '/watch';
+                // parts := ['', 'live' || 'shorts', id, ...rest]
+                url.search = `?v=${encodeURIComponent(parts[2])}`
+            }
+            break;
+
+        case "youtu":
+            if (url.hostname === 'youtu.be' && parts.length >= 2) {
+                /* youtu.be urls can be weird, e.g. https://youtu.be/<id>//asdasd// still works
+                ** but we only care about the 1st segment of the path */
+                url = new URL(`https://youtube.com/watch?v=${
+                    encodeURIComponent(parts[1])
+                }`)
+            }
+            break;
+
+        case "pin":
+            if (url.hostname === 'pin.it' && parts.length === 2) {
+                url = new URL(`https://pinterest.com/url_shortener/${
+                    encodeURIComponent(parts[1])
+                }`)
+            }
+            break;
+
+        case "vxtwitter":
+        case "fixvx":
+        case "x":
+            if (services.twitter.altDomains.includes(url.hostname)) {
+                url.hostname = 'twitter.com';
+            }
+            break;
+
+        case "twitch":
+            if (url.hostname === 'clips.twitch.tv' && parts.length >= 2) {
+                url = new URL(`https://twitch.tv/_/clip/${parts[1]}`);
+            }
+            break;
+
+        case "bilibili":
+            if (host.tld === 'tv') {
+                url = new URL(`https://bilibili.com/_tv${url.pathname}`);
+            }
+            break;
+
+        case "b23":
+            if (url.hostname === 'b23.tv' && parts.length === 2) {
+                url = new URL(`https://bilibili.com/_shortLink/${parts[1]}`)
+            }
+            break;
+
+        case "dai":
+            if (url.hostname === 'dai.ly' && parts.length === 2) {
+                url = new URL(`https://dailymotion.com/video/${parts[1]}`)
+            }
+            break;
+
+        case "facebook":
+        case "fb":
+            if (url.searchParams.get('v')) {
+                url = new URL(`https://web.facebook.com/user/videos/${url.searchParams.get('v')}`)
+            }
+            if (url.hostname === 'fb.watch') {
+                url = new URL(`https://web.facebook.com/_shortLink/${parts[1]}`)
+            }
+            break;
+
+        case "ddinstagram":
+            if (services.instagram.altDomains.includes(host.domain) && [null, 'd', 'g'].includes(host.subdomain)) {
+                url.hostname = 'instagram.com';
+            }
+            break;
+
+        case "vk":
+        case "vkvideo":
+            if (services.vk.altDomains.includes(url.hostname)) {
+                url.hostname = 'vk.com';
+            }
+            break;
+    }
+
+    return url
+}
+
+function cleanURL(url) {
+    assert(url instanceof URL);
+    const host = psl.parse(url.hostname).sld;
+
+    let stripQuery = true;
+
+    const limitQuery = (param) => {
+        url.search = `?${param}=` + encodeURIComponent(url.searchParams.get(param));
+        stripQuery = false;
+    }
+
+    switch (host) {
+        case "pinterest":
+            url.hostname = 'pinterest.com';
+            break;
+        case "vk":
+            if (url.pathname.includes('/clip') && url.searchParams.get('z')) {
+                limitQuery('z')
+            }
+            break;
+        case "youtube":
+            if (url.searchParams.get('v')) {
+                limitQuery('v')
+            }
+            break;
+        case "rutube":
+            if (url.searchParams.get('p')) {
+                limitQuery('p')
+            }
+            break;
+        case "twitter":
+            if (url.searchParams.get('post_id')) {
+                limitQuery('post_id')
+            }
+            break;
+    }
+
+    if (stripQuery) {
+        url.search = ''
+    }
+
+    url.username = url.password = url.port = url.hash = ''
+
+    if (url.pathname.endsWith('/'))
+        url.pathname = url.pathname.slice(0, -1);
+
+    return url
+}
+
+function getHostIfValid(url) {
+    const host = psl.parse(url.hostname);
+    if (host.error) return;
+
+    const service = services[host.sld];
+    if (!service) return;
+    if ((service.tld ?? 'com') !== host.tld) return;
+
+    const anySubdomainAllowed = service.subdomains === '*';
+    const validSubdomain = [null, 'www', ...(service.subdomains ?? [])].includes(host.subdomain);
+    if (!validSubdomain && !anySubdomainAllowed) return;
+
+    return host.sld;
+}
+
+export function normalizeURL(url) {
+    return cleanURL(
+        aliasURL(
+            new URL(url.replace(/^https\/\//, 'https://'))
+        )
+    );
+}
+
+export function extract(url) {
+    if (!(url instanceof URL)) {
+        url = new URL(url);
+    }
+
+    const host = getHostIfValid(url);
+
+    if (!host) {
+        return { error: "link.invalid" };
+    }
+
+    if (!env.enabledServices.has(host)) {
+        // show a different message when youtube is disabled on official instances
+        // as it only happens when shit hits the fan
+        if (new URL(env.apiURL).hostname.endsWith(".imput.net") && host === "youtube") {
+            return { error: "youtube.temporary_disabled" };
+        }
+        return { error: "service.disabled" };
+    }
+
+    let patternMatch;
+    for (const pattern of services[host].patterns) {
+        patternMatch = pattern.match(
+            url.pathname.substring(1) + url.search
+        );
+
+        if (patternMatch) {
+            break;
+        }
+    }
+
+    if (!patternMatch) {
+        return {
+            error: "link.unsupported",
+            context: {
+                service: friendlyServiceName(host),
+            }
+        };
+    }
+
+    return { host, patternMatch };
+}

src/security/api-keys.js

@@ -0,0 +1,227 @@
+import { env } from "../config.js";
+import { readFile } from "node:fs/promises";
+import { Green, Yellow } from "../misc/console-text.js";
+import ip from "ipaddr.js";
+import * as cluster from "../misc/cluster.js";
+
+// this function is a modified variation of code
+// from https://stackoverflow.com/a/32402438/14855621
+const generateWildcardRegex = rule => {
+    var escapeRegex = (str) => str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1");
+    return new RegExp("^" + rule.split("*").map(escapeRegex).join(".*") + "$");
+}
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+let keys = {};
+
+const ALLOWED_KEYS = new Set(['name', 'ips', 'userAgents', 'limit']);
+
+/* Expected format pseudotype:
+** type KeyFileContents = Record<
+**    UUIDv4String,
+**    {
+**        name?: string,
+**        limit?: number | "unlimited",
+**        ips?: CIDRString[],
+**        userAgents?: string[]
+**    }
+** >;
+*/
+
+const validateKeys = (input) => {
+    if (typeof input !== 'object' || input === null) {
+        throw "input is not an object";
+    }
+
+    if (Object.keys(input).some(x => !UUID_REGEX.test(x))) {
+        throw "key file contains invalid key(s)";
+    }
+
+    Object.values(input).forEach(details => {
+        if (typeof details !== 'object' || details === null) {
+            throw "some key(s) are incorrectly configured";
+        }
+
+        const unexpected_key = Object.keys(details).find(k => !ALLOWED_KEYS.has(k));
+        if (unexpected_key) {
+            throw "detail object contains unexpected key: " + unexpected_key;
+        }
+
+        if (details.limit && details.limit !== 'unlimited') {
+            if (typeof details.limit !== 'number')
+                throw "detail object contains invalid limit (not a number)";
+            else if (details.limit < 1)
+                throw "detail object contains invalid limit (not a positive number)";
+        }
+
+        if (details.ips) {
+            if (!Array.isArray(details.ips))
+                throw "details object contains value for `ips` which is not an array";
+
+            const invalid_ip = details.ips.find(
+                addr => typeof addr !== 'string' || (!ip.isValidCIDR(addr) && !ip.isValid(addr))
+            );
+
+            if (invalid_ip) {
+                throw "`ips` in details contains an invalid IP or CIDR range: " + invalid_ip;
+            }
+        }
+
+        if (details.userAgents) {
+            if (!Array.isArray(details.userAgents))
+                throw "details object contains value for `userAgents` which is not an array";
+
+            const invalid_ua = details.userAgents.find(ua => typeof ua !== 'string');
+            if (invalid_ua) {
+                throw "`userAgents` in details contains an invalid user agent: " + invalid_ua;
+            }
+        }
+    });
+}
+
+const formatKeys = (keyData) => {
+    const formatted = {};
+
+    for (let key in keyData) {
+        const data = keyData[key];
+        key = key.toLowerCase();
+
+        formatted[key] = {};
+
+        if (data.limit) {
+            if (data.limit === "unlimited") {
+                data.limit = Infinity;
+            }
+
+            formatted[key].limit = data.limit;
+        }
+
+        if (data.ips) {
+            formatted[key].ips = data.ips.map(addr => {
+                if (ip.isValid(addr)) {
+                    const parsed = ip.parse(addr);
+                    const range = parsed.kind() === 'ipv6' ? 128 : 32;
+                    return [ parsed, range ];
+                }
+
+                return ip.parseCIDR(addr);
+            });
+        }
+
+        if (data.userAgents) {
+            formatted[key].userAgents = data.userAgents.map(generateWildcardRegex);
+        }
+    }
+
+    return formatted;
+}
+
+const updateKeys = (newKeys) => {
+    keys = formatKeys(newKeys);
+}
+
+const loadKeys = async (source) => {
+    let updated;
+    if (source.protocol === 'file:') {
+        const pathname = source.pathname === '/' ? '' : source.pathname;
+        updated = JSON.parse(
+            await readFile(
+                decodeURIComponent(source.host + pathname),
+                'utf8'
+            )
+        );
+    } else {
+        updated = await fetch(source).then(a => a.json());
+    }
+
+    validateKeys(updated);
+
+    cluster.broadcast({ api_keys: updated });
+
+    updateKeys(updated);
+}
+
+const wrapLoad = (url, initial = false) => {
+    loadKeys(url)
+    .then(() => {
+        if (initial) {
+            console.log(`${Green('[✓]')} api keys loaded successfully!`)
+        }
+    })
+    .catch((e) => {
+        console.error(`${Yellow('[!]')} Failed loading API keys at ${new Date().toISOString()}.`);
+        console.error('Error:', e);
+    })
+}
+
+const err = (reason) => ({ success: false, error: reason });
+
+export const validateAuthorization = (req) => {
+    const authHeader = req.get('Authorization');
+
+    if (typeof authHeader !== 'string') {
+        return err("missing");
+    }
+
+    const [ authType, keyString ] = authHeader.split(' ', 2);
+    if (authType.toLowerCase() !== 'api-key') {
+        return err("not_api_key");
+    }
+
+    if (!UUID_REGEX.test(keyString) || `${authType} ${keyString}` !== authHeader) {
+        return err("invalid");
+    }
+
+    const matchingKey = keys[keyString.toLowerCase()];
+    if (!matchingKey) {
+        return err("not_found");
+    }
+
+    if (matchingKey.ips) {
+        let addr;
+        try {
+            addr = ip.parse(req.ip);
+        } catch {
+            return err("invalid_ip");
+        }
+
+        const ip_allowed = matchingKey.ips.some(
+            ([ allowed, size ]) => {
+                return addr.kind() === allowed.kind()
+                        && addr.match(allowed, size);
+            }
+        );
+
+        if (!ip_allowed) {
+            return err("ip_not_allowed");
+        }
+    }
+
+    if (matchingKey.userAgents) {
+        const userAgent = req.get('User-Agent');
+        if (!matchingKey.userAgents.some(regex => regex.test(userAgent))) {
+            return err("ua_not_allowed");
+        }
+    }
+
+    req.rateLimitKey = keyString.toLowerCase();
+    req.rateLimitMax = matchingKey.limit;
+
+    return { success: true };
+}
+
+export const setup = (url) => {
+    if (cluster.isPrimary) {
+        wrapLoad(url, true);
+        if (env.keyReloadInterval > 0) {
+            setInterval(() => wrapLoad(url), env.keyReloadInterval * 1000);
+        }
+    } else if (cluster.isWorker) {
+        process.on('message', (message) => {
+            if ('api_keys' in message) {
+                updateKeys(message.api_keys);
+            }
+        });
+    }
+}

src/security/jwt.js

@@ -0,0 +1,59 @@
+import { nanoid } from "nanoid";
+import { createHmac } from "crypto";
+
+import { env } from "../config.js";
+
+const toBase64URL = (b) => Buffer.from(b).toString("base64url");
+const fromBase64URL = (b) => Buffer.from(b, "base64url").toString();
+
+const makeHmac = (header, payload) =>
+    createHmac("sha256", env.jwtSecret)
+        .update(`${header}.${payload}`)
+        .digest("base64url");
+
+const generate = () => {
+    const exp = Math.floor(new Date().getTime() / 1000) + env.jwtLifetime;
+
+    const header = toBase64URL(JSON.stringify({
+        alg: "HS256",
+        typ: "JWT"
+    }));
+
+    const payload = toBase64URL(JSON.stringify({
+        jti: nanoid(8),
+        exp,
+    }));
+
+    const signature = makeHmac(header, payload);
+
+    return {
+        token: `${header}.${payload}.${signature}`,
+        exp: env.jwtLifetime - 2,
+    };
+}
+
+const verify = (jwt) => {
+    const [header, payload, signature] = jwt.split(".", 3);
+    const timestamp = Math.floor(new Date().getTime() / 1000);
+
+    if ([header, payload, signature].join('.') !== jwt) {
+        return false;
+    }
+
+    const verifySignature = makeHmac(header, payload);
+
+    if (verifySignature !== signature) {
+        return false;
+    }
+
+    if (timestamp >= JSON.parse(fromBase64URL(payload)).exp) {
+        return false;
+    }
+
+    return true;
+}
+
+export default {
+    generate,
+    verify,
+}

src/security/secrets.js

@@ -0,0 +1,62 @@
+import cluster from "node:cluster";
+import { createHmac, randomBytes } from "node:crypto";
+
+const generateSalt = () => {
+    if (cluster.isPrimary)
+        return randomBytes(64);
+
+    return null;
+}
+
+let rateSalt = generateSalt();
+let streamSalt = generateSalt();
+
+export const syncSecrets = () => {
+    return new Promise((resolve, reject) => {
+        if (cluster.isPrimary) {
+            let remaining = Object.values(cluster.workers).length;
+            const handleReady = (worker, m) => {
+                if (m.ready)
+                    worker.send({ rateSalt, streamSalt });
+
+                if (!--remaining)
+                    resolve();
+            }
+
+            for (const worker of Object.values(cluster.workers)) {
+                worker.once(
+                    'message',
+                    (m) => handleReady(worker, m)
+                );
+            }
+        } else if (cluster.isWorker) {
+            if (rateSalt || streamSalt)
+                return reject();
+
+            process.send({ ready: true });
+            process.once('message', (message) => {
+                if (rateSalt || streamSalt)
+                    return reject();
+
+                if (message.rateSalt && message.streamSalt) {
+                    streamSalt = Buffer.from(message.streamSalt);
+                    rateSalt = Buffer.from(message.rateSalt);
+                    resolve();
+                }
+            });
+        } else reject();
+    });
+}
+
+
+export const hashHmac = (value, type) => {
+    let salt;
+    if (type === 'rate')
+        salt = rateSalt;
+    else if (type === 'stream')
+        salt = streamSalt;
+    else
+        throw "unknown salt";
+
+    return createHmac("sha256", salt).update(value).digest();
+}

src/security/turnstile.js

@@ -0,0 +1,19 @@
+import { env } from "../config.js";
+
+export const verifyTurnstileToken = async (turnstileResponse, ip) => {
+    const result = await fetch("https://challenges.cloudflare.com/turnstile/v0/siteverify", {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+            secret: env.turnstileSecret,
+            response: turnstileResponse,
+            remoteip: ip,
+        }),
+    })
+    .then(r => r.json())
+    .catch(() => {});
+
+    return !!result?.success;
+}

src/store/base-store.js

@@ -0,0 +1,48 @@
+const _stores = new Set();
+
+export class Store {
+    id;
+
+    constructor(name) {
+        name = name.toUpperCase();
+
+        if (_stores.has(name))
+            throw `${name} store already exists`;
+        _stores.add(name);
+
+        this.id = name;
+    }
+
+    async _has(_key) { await Promise.reject("needs implementation"); }
+    has(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        return this._has(key);
+    }
+
+    async _get(_key) { await Promise.reject("needs implementation"); }
+    async get(key) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        const val = await this._get(key);
+        if (val === null)
+            return null;
+
+        return val;
+    }
+
+    async _set(_key, _val, _exp_sec = -1) { await Promise.reject("needs implementation") }
+    set(key, val, exp_sec = -1) {
+        if (typeof key !== 'string') {
+            key = key.toString();
+        }
+
+        exp_sec = Math.round(exp_sec);
+
+        return this._set(key, val, exp_sec);
+    }
+};