import { assertConfigIsValid, config } from "./config"; import "source-map-support/register"; import express from "express"; import cors from "cors"; import pinoHttp from "pino-http"; import childProcess from "child_process"; import { logger } from "./logger"; import { keyPool } from "./key-management"; import { adminRouter } from "./admin/routes"; import { proxyRouter } from "./proxy/routes"; import { handleInfoPage } from "./info-page"; import { logQueue } from "./prompt-logging"; import { start as startRequestQueue } from "./proxy/queue"; import { init as initUserStore } from "./proxy/auth/user-store"; import { checkOrigin } from "./proxy/check-origin"; const PORT = config.port; const INTERVAL_TIME = 60000; // Config check interval time in milliseconds (e.g., 60000 ms = 1 minute) const app = express(); // middleware app.use( pinoHttp({ quietReqLogger: true, logger, autoLogging: { ignore: (req) => { const ignored = ["/proxy/kobold/api/v1/model", "/health"]; return ignored.includes(req.url as string); }, }, redact: { paths: [ "req.headers.cookie", 'res.headers["set-cookie"]', //"req.headers.authorization", //'req.headers["x-api-key"]', //'req.headers["x-forwarded-for"]', //'req.headers["x-real-ip"]', //'req.headers["true-client-ip"]', //'req.headers["cf-connecting-ip"]', // Don't log the prompt text on transform errors //"body.messages", //"body.prompt", ], censor: "********", }, }) ); app.get("/health", (_req, res) => res.sendStatus(200)); app.use((req, _res, next) => { req.startTime = Date.now(); req.retryCount = 0; next(); }); app.use(cors({ origin: "*", methods: "GET,HEAD,PUT,PATCH,POST,DELETE,OPTIONS", preflightContinue: false, optionsSuccessStatus: 204, allowedHeaders: "*" })); app.use( express.json({ limit: "10mb" }), express.urlencoded({ extended: true, limit: "10mb" }) ); // TODO: Detect (or support manual configuration of) whether the app is behind // a load balancer/reverse proxy, which is necessary to determine request IP // addresses correctly. app.set("trust proxy", true); // routes app.use(checkOrigin); app.get("/", handleInfoPage); app.use("/admin", adminRouter); app.use("/proxy", proxyRouter); // 500 and 404 app.use((err: any, _req: unknown, res: express.Response, _next: unknown) => { if (err.status) { res.status(err.status).json({ error: err.message }); } else { logger.error(err); res.status(500).json({ error: { type: "proxy_error", message: err.message, stack: err.stack, proxy_note: `Reverse proxy encountered an internal server error.`, }, }); } }); app.use((_req: unknown, res: express.Response) => { res.status(404).json({ error: "Not found" }); }); async function start() { logger.info("Server starting up..."); await setBuildInfo(); logger.info("Checking configs and external dependencies..."); await assertConfigIsValid(); keyPool.init(); if (config.gatekeeper === "user_token") { await initUserStore(); } if (config.promptLogging) { logger.info("Starting prompt logging..."); logQueue.start(); } if (config.queueMode !== "none") { logger.info("Starting request queue..."); startRequestQueue(); } app.listen(PORT, async () => { logger.info({ port: PORT }, "Now listening for connections."); registerUncaughtExceptionHandler(); }); logger.info( { build: process.env.BUILD_INFO, nodeEnv: process.env.NODE_ENV }, "Startup complete." ); setInterval(async () => { logger.info("-!!!-ALERT-!!!- CHECKING ONLINE CONFIG. SERVER MAY HANG. -!!!-ALERT-!!!-"); await assertConfigIsValid(); }, INTERVAL_TIME); } function registerUncaughtExceptionHandler() { process.on("uncaughtException", (err: any) => { logger.error( { err, stack: err?.stack }, "UNCAUGHT EXCEPTION. Please report this error trace." ); }); process.on("unhandledRejection", (err: any) => { logger.error( { err, stack: err?.stack }, "UNCAUGHT PROMISE REJECTION. Please report this error trace." ); }); } /** * Attepts to collect information about the current build from either the * environment or the git repo used to build the image (only works if not * .dockerignore'd). If you're running a sekrit club fork, you can no-op this * function and set the BUILD_INFO env var manually, though I would prefer you * didn't set it to something misleading. */ async function setBuildInfo() { /* // Render .dockerignore's the .git directory but provides info in the env if (process.env.RENDER) { const sha = process.env.RENDER_GIT_COMMIT?.slice(0, 7) || "unknown SHA"; const branch = process.env.RENDER_GIT_BRANCH || "unknown branch"; const repo = process.env.RENDER_GIT_REPO_SLUG || "unknown repo"; const buildInfo = `${sha} (${branch}@${repo})`; //process.env.BUILD_INFO = buildInfo; logger.info({ build: buildInfo }, "Got build info from Render config."); return; } try { // Ignore git's complaints about dubious directory ownership on Huggingface // (which evidently runs dockerized Spaces on Windows with weird NTFS perms) if (process.env.SPACE_ID) { childProcess.execSync("git config --global --add safe.directory /app"); } const promisifyExec = (cmd: string) => new Promise((resolve, reject) => { childProcess.exec(cmd, (err, stdout) => err ? reject(err) : resolve(stdout) ); }); const promises = [ promisifyExec("git rev-parse --short HEAD"), promisifyExec("git rev-parse --abbrev-ref HEAD"), promisifyExec("git config --get remote.origin.url"), promisifyExec("git status --porcelain"), ].map((p) => p.then((result: any) => result.toString().trim())); let [sha, branch, remote, status] = await Promise.all(promises); remote = remote.match(/.*[\/:]([\w-]+)\/([\w\-\.]+?)(?:\.git)?$/) || []; const repo = remote.slice(-2).join("/"); status = status // ignore Dockerfile changes since that's how the user deploys the app .split("\n") .filter((line: string) => !line.endsWith("Dockerfile") && line); const changes = status.length > 0; const build = `${sha}${changes ? " (modified)" : ""} (${branch}@${repo})`; process.env.BUILD_INFO = build; logger.info({ build, status, changes }, "Got build info from Git."); } catch (error: any) { logger.error( { error, stdout: error.stdout.toString(), stderr: error.stderr.toString(), }, "Failed to get commit SHA.", error ); process.env.BUILD_INFO = "unknown"; }*/ process.env.BUILD_INFO = "4335977 (main@khanon/oai-reverse-proxy)"; } start();