Spaces:
No application file
No application file
import fs from "fs"; | |
import { Request, Response } from "express"; | |
import showdown from "showdown"; | |
import { config, listConfig } from "./config"; | |
import { keyPool } from "./key-management"; | |
import { getUniqueIps } from "./proxy/rate-limit"; | |
import { | |
QueuePartition, | |
getEstimatedWaitTime, | |
getQueueLength, | |
} from "./proxy/queue"; | |
const INFO_PAGE_TTL = 5000; | |
let infoPageHtml: string | undefined; | |
let infoPageLastUpdated = 0; | |
export const handleInfoPage = (req: Request, res: Response) => { | |
if (infoPageLastUpdated + INFO_PAGE_TTL > Date.now()) { | |
res.send(infoPageHtml); | |
return; | |
} | |
// Sometimes huggingface doesn't send the host header and makes us guess. | |
const baseUrl = | |
process.env.SPACE_ID && !req.get("host")?.includes("hf.space") | |
? getExternalUrlForHuggingfaceSpaceId(process.env.SPACE_ID) | |
: req.protocol + "://" + req.get("host"); | |
res.send(cacheInfoPageHtml(baseUrl)); | |
}; | |
function cacheInfoPageHtml(baseUrl: string) { | |
const keys = keyPool.list(); | |
const openaiKeys = keys.filter((k) => k.service === "openai").length; | |
const anthropicKeys = keys.filter((k) => k.service === "anthropic").length; | |
const info = { | |
uptime: process.uptime(), | |
endpoints: { | |
...(openaiKeys ? { openai: baseUrl + "/proxy/openai" } : {}), | |
...(anthropicKeys ? { anthropic: baseUrl + "/proxy/anthropic" } : {}), | |
}, | |
proompts: keys.reduce((acc, k) => acc + k.promptCount, 0), | |
...(config.modelRateLimit ? { proomptersNow: getUniqueIps() } : {}), | |
openaiKeys, | |
anthropicKeys, | |
...(openaiKeys ? getOpenAIInfo() : {}), | |
...(anthropicKeys ? getAnthropicInfo() : {}), | |
config: listConfig(), | |
build: process.env.BUILD_INFO || "dev", | |
}; | |
const title = getServerTitle(); | |
const headerHtml = buildInfoPageHeader(new showdown.Converter(), title); | |
const pageBody = `<!DOCTYPE html> | |
<html lang="en"> | |
<head> | |
<meta charset="utf-8" /> | |
<meta name="robots" content="noindex" /> | |
<title>${title}</title> | |
</head> | |
<body style="font-family: sans-serif; background-color: #f0f0f0; padding: 1em;"> | |
${headerHtml} | |
<hr /> | |
<h2>Service Info</h2> | |
<pre>${JSON.stringify(info, null, 2)}</pre> | |
</body> | |
</html>`; | |
infoPageHtml = pageBody; | |
infoPageLastUpdated = Date.now(); | |
return pageBody; | |
} | |
type ServiceInfo = { | |
activeKeys: number; | |
trialKeys?: number; | |
quota: string; | |
proomptersInQueue: number; | |
estimatedQueueTime: string; | |
}; | |
// this has long since outgrown this awful "dump everything in a <pre> tag" approach | |
// but I really don't want to spend time on a proper UI for this right now | |
function getOpenAIInfo() { | |
const info: { [model: string]: Partial<ServiceInfo> } = {}; | |
const keys = keyPool.list().filter((k) => k.service === "openai"); | |
const hasGpt4 = keys.some((k) => k.isGpt4); | |
if (keyPool.anyUnchecked()) { | |
const uncheckedKeys = keys.filter((k) => !k.lastChecked); | |
info.status = `Still checking ${uncheckedKeys.length} keys...` as any; | |
} else { | |
delete info.status; | |
} | |
if (config.checkKeys) { | |
const turboKeys = keys.filter((k) => !k.isGpt4 && !k.isDisabled); | |
const gpt4Keys = keys.filter((k) => k.isGpt4 && !k.isDisabled); | |
const quota: Record<string, string> = { turbo: "", gpt4: "" }; | |
const turboQuota = keyPool.remainingQuota("openai") * 100; | |
const gpt4Quota = keyPool.remainingQuota("openai", { gpt4: true }) * 100; | |
if (config.quotaDisplayMode === "full") { | |
const turboUsage = keyPool.usageInUsd("openai"); | |
const gpt4Usage = keyPool.usageInUsd("openai", { gpt4: true }); | |
quota.turbo = `${turboUsage} (${Math.round(turboQuota)}% remaining)`; | |
quota.gpt4 = `${gpt4Usage} (${Math.round(gpt4Quota)}% remaining)`; | |
} else { | |
quota.turbo = `${Math.round(turboQuota)}%`; | |
quota.gpt4 = `${Math.round(gpt4Quota * 100)}%`; | |
} | |
info.turbo = { | |
activeKeys: turboKeys.filter((k) => !k.isDisabled).length, | |
trialKeys: turboKeys.filter((k) => k.isTrial).length, | |
quota: quota.turbo, | |
}; | |
if (hasGpt4 && true === false) { | |
info.gpt4 = { | |
activeKeys: gpt4Keys.filter((k) => !k.isDisabled).length, | |
trialKeys: gpt4Keys.filter((k) => k.isTrial).length, | |
quota: quota.gpt4, | |
}; | |
} | |
if (config.quotaDisplayMode === "none") { | |
delete info.turbo?.quota; | |
delete info.gpt4?.quota; | |
} | |
delete info.gpt4?.quota; | |
} else { | |
info.status = "Key checking is disabled." as any; | |
info.turbo = { activeKeys: keys.filter((k) => !k.isDisabled).length }; | |
} | |
if (config.queueMode !== "none") { | |
const turboQueue = getQueueInformation("turbo"); | |
info.turbo.proomptersInQueue = turboQueue.proomptersInQueue; | |
info.turbo.estimatedQueueTime = turboQueue.estimatedQueueTime; | |
if (hasGpt4 && true === false) { | |
const gpt4Queue = getQueueInformation("gpt-4"); | |
info.gpt4.proomptersInQueue = gpt4Queue.proomptersInQueue; | |
info.gpt4.estimatedQueueTime = gpt4Queue.estimatedQueueTime; | |
} | |
} | |
return info; | |
} | |
function getAnthropicInfo() { | |
const claudeInfo: Partial<ServiceInfo> = {}; | |
const keys = keyPool.list().filter((k) => k.service === "anthropic"); | |
claudeInfo.activeKeys = keys.filter((k) => !k.isDisabled).length; | |
if (config.queueMode !== "none") { | |
const queue = getQueueInformation("claude"); | |
claudeInfo.proomptersInQueue = queue.proomptersInQueue; | |
claudeInfo.estimatedQueueTime = queue.estimatedQueueTime; | |
} | |
return { claude: claudeInfo }; | |
} | |
/** | |
* If the server operator provides a `greeting.md` file, it will be included in | |
* the rendered info page. | |
**/ | |
function buildInfoPageHeader(converter: showdown.Converter, title: string) { | |
const customGreeting = fs.existsSync("greeting.md") | |
? fs.readFileSync("greeting.md", "utf8") | |
: null; | |
// TODO: use some templating engine instead of this mess | |
let infoBody = `<!-- Header for Showdown's parser, don't remove this line --> | |
# ${title}`; | |
if (config.promptLogging && true === false) { | |
infoBody += `\n## Prompt logging is enabled! | |
The server operator has enabled prompt logging. The prompts you send to this proxy and the AI responses you receive may be saved. | |
Logs are anonymous and do not contain IP addresses or timestamps. [You can see the type of data logged here, along with the rest of the code.](https://gitgud.io/khanon/oai-reverse-proxy/-/blob/main/src/prompt-logging/index.ts). | |
**If you are uncomfortable with this, don't send prompts to this proxy!**`; | |
} | |
if (config.queueMode !== "none") { | |
const waits = []; | |
infoBody += `\n## Estimated Wait Times\nIf the AI is busy, your prompt will processed when a slot frees up.`; | |
if (config.openaiKey) { | |
const turboWait = getQueueInformation("turbo").estimatedQueueTime; | |
const gpt4Wait = getQueueInformation("gpt-4").estimatedQueueTime; | |
waits.push(`**Turbo:** ${turboWait}`); | |
if (keyPool.list().some((k) => k.isGpt4)) { | |
waits.push(`**GPT-4:** ${gpt4Wait}`); | |
} | |
} | |
if (config.anthropicKey) { | |
const claudeWait = getQueueInformation("claude").estimatedQueueTime; | |
waits.push(`**Claude:** ${claudeWait}`); | |
} | |
infoBody += "\n\n" + waits.join(" / "); | |
} | |
if (customGreeting) { | |
infoBody += `\n## Server Greeting\n | |
${customGreeting}`; | |
} | |
return converter.makeHtml(infoBody); | |
} | |
/** Returns queue time in seconds, or minutes + seconds if over 60 seconds. */ | |
function getQueueInformation(partition: QueuePartition) { | |
if (config.queueMode === "none") { | |
return {}; | |
} | |
const waitMs = getEstimatedWaitTime(partition); | |
const waitTime = | |
waitMs < 60000 | |
? `${Math.round(waitMs / 1000)}sec` | |
: `${Math.round(waitMs / 60000)}min, ${Math.round( | |
(waitMs % 60000) / 1000 | |
)}sec`; | |
return { | |
proomptersInQueue: getQueueLength(partition), | |
estimatedQueueTime: waitMs > 2000 ? waitTime : "no wait", | |
}; | |
} | |
function getServerTitle() { | |
// Use manually set title if available | |
if (process.env.SERVER_TITLE) { | |
return process.env.SERVER_TITLE; | |
} | |
// Huggingface | |
if (process.env.SPACE_ID) { | |
return `${process.env.SPACE_AUTHOR_NAME} / ${process.env.SPACE_TITLE}`; | |
} | |
// Render | |
if (process.env.RENDER) { | |
return `Render / ${process.env.RENDER_SERVICE_NAME}`; | |
} | |
return "OAI Reverse Proxy"; | |
} | |
function getExternalUrlForHuggingfaceSpaceId(spaceId: string) { | |
// Huggingface broke their amazon elb config and no longer sends the | |
// x-forwarded-host header. This is a workaround. | |
try { | |
const [username, spacename] = spaceId.split("/"); | |
return `https://${username}-${spacename.replace(/_/g, "-")}.hf.space`; | |
} catch (e) { | |
return ""; | |
} | |
} | |