import { RequestHandler, Request, Router } from "express"; import * as http from "http"; import { createProxyMiddleware } from "http-proxy-middleware"; import { config } from "../config"; import { keyPool } from "../key-management"; import { logger } from "../logger"; import { createQueueMiddleware } from "./queue"; import { ipLimiter } from "./rate-limit"; import { injectMDReq } from "../proxy/middleware/request/md-request"; import { privilegeCheck } from "../proxy/middleware/request/privilege-check" import { handleProxyError } from "./middleware/common"; import { addKey, milkZoomers, createPreprocessorMiddleware, finalizeBody, languageFilter, limitCompletions, injectMDReq, limitOutputTokens, privilegeCheck, } from "./middleware/request"; import { createOnProxyResHandler, ProxyResHandlerWithBody, } from "./middleware/response"; let modelsCache: any = null; let modelsCacheTime = 0; function getModelsResponse() { if (new Date().getTime() - modelsCacheTime < 1000 * 60) { return modelsCache; } const gptVariants = [ "gpt-4", "gpt-4-0613", "gpt-4-0314", "gpt-4-32k", "gpt-4-32k-0613", "gpt-4-32k-0314", "gpt-3.5-turbo", "gpt-3.5-turbo-0301", "gpt-3.5-turbo-0613", "gpt-3.5-turbo-16k", "gpt-3.5-turbo-16k-0613", ]; const gpt4Available = keyPool.list().filter((key) => { return key.service === "openai" && !key.isDisabled && key.isGpt4; }).length; const models = gptVariants .map((id) => ({ id, object: "model", created: new Date().getTime(), owned_by: "openai", permission: [ { id: "modelperm-" + id, object: "model_permission", created: new Date().getTime(), organization: "*", group: null, is_blocking: false, }, ], root: id, parent: null, })) .filter((model) => { if (model.id.startsWith("gpt-4")) { return gpt4Available > 0; } return true; }); modelsCache = { object: "list", data: models }; modelsCacheTime = new Date().getTime(); return modelsCache; } const handleModelRequest: RequestHandler = (_req, res) => { res.status(200).json(getModelsResponse()); }; const rewriteRequest = ( proxyReq: http.ClientRequest, req: Request, res: http.ServerResponse ) => { const rewriterPipeline = [ addKey, milkZoomers, languageFilter, limitOutputTokens, limitCompletions, injectMDReq, privilegeCheck, finalizeBody, ]; try { for (const rewriter of rewriterPipeline) { rewriter(proxyReq, req, res, {}); } } catch (error) { req.log.error(error, "Error while executing proxy rewriter"); proxyReq.destroy(error as Error); } }; const openaiResponseHandler: ProxyResHandlerWithBody = async ( _proxyRes, req, res, body ) => { if (typeof body !== "object") { throw new Error("Expected body to be an object"); } res.status(200).json(body); }; const openaiProxy = createQueueMiddleware( createProxyMiddleware({ target: "https://api.openai.com", changeOrigin: true, on: { proxyReq: rewriteRequest, proxyRes: createOnProxyResHandler([openaiResponseHandler]), error: handleProxyError, }, selfHandleResponse: true, logger, }) ); const openaiRouter = Router(); // Fix paths because clients don't consistently use the /v1 prefix. openaiRouter.use((req, _res, next) => { if (!req.path.startsWith("/v1/")) { req.url = `/v1${req.url}`; } next(); }); openaiRouter.get("/v1/models", handleModelRequest); openaiRouter.post( "/v1/chat/completions", ipLimiter, createPreprocessorMiddleware({ inApi: "openai", outApi: "openai" }), openaiProxy ); // Redirect browser requests to the homepage. openaiRouter.get("*", (req, res, next) => { const isBrowser = req.headers["user-agent"]?.includes("Mozilla"); if (isBrowser) { res.redirect("/"); } else { next(); } }); openaiRouter.use((req, res) => { req.log.warn(`Blocked openai proxy request: ${req.method} ${req.path}`); res.status(404).json({ error: "Not found" }); }); export const openai = openaiRouter;