blind_chat / src /lib /server /modelEndpoint.ts
lauro1's picture
test
faca43f
import {
HF_ACCESS_TOKEN,
HF_API_ROOT,
USE_CLIENT_CERTIFICATE,
CERT_PATH,
KEY_PATH,
CA_PATH,
CLIENT_KEY_PASSWORD,
REJECT_UNAUTHORIZED,
} from "$env/static/private";
import { sum } from "$lib/utils/sum";
import type { BackendModel, Endpoint } from "./models";
import { loadClientCertificates } from "$lib/utils/loadClientCerts";
if (USE_CLIENT_CERTIFICATE === "true") {
loadClientCertificates(
CERT_PATH,
KEY_PATH,
CA_PATH,
CLIENT_KEY_PASSWORD,
REJECT_UNAUTHORIZED === "true"
);
}
/**
* Find a random load-balanced endpoint
*/
export function modelEndpoint(model: BackendModel): Endpoint {
if (model.is_local ?? false) {
return {
host: "local",
model: model.name,
weight: 1,
url: `${HF_API_ROOT}/${model.name}`,
authorization: `Bearer ${HF_ACCESS_TOKEN}`,
};
} else if (!model.endpoints) {
return {
host: "tgi",
url: `${HF_API_ROOT}/${model.name}`,
authorization: `Bearer ${HF_ACCESS_TOKEN}`,
weight: 1,
};
}
const endpoints = model.endpoints;
const totalWeight = sum(endpoints.map((e) => e.weight));
let random = Math.random() * totalWeight;
for (const endpoint of endpoints) {
if (random < endpoint.weight) {
console.log(endpoint);
return endpoint;
}
random -= endpoint.weight;
}
throw new Error("Invalid config, no endpoint found");
}