File size: 1,098 Bytes
66adc5d
 
 
 
 
 
 
 
 
 
9be5ab5
2e6d1bb
9be5ab5
66adc5d
 
 
 
 
 
 
 
 
 
 
 
9be5ab5
 
 
2e6d1bb
cf7ac8d
7764421
2e6d1bb
3baa389
7764421
 
 
 
cf7ac8d
7764421
 
9be5ab5
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import {
	HF_ACCESS_TOKEN,
	HF_API_ROOT,
	USE_CLIENT_CERTIFICATE,
	CERT_PATH,
	KEY_PATH,
	CA_PATH,
	CLIENT_KEY_PASSWORD,
	REJECT_UNAUTHORIZED,
} from "$env/static/private";
import { sum } from "$lib/utils/sum";
import type { BackendModel, Endpoint } from "./models";

import { loadClientCertificates } from "$lib/utils/loadClientCerts";

if (USE_CLIENT_CERTIFICATE === "true") {
	loadClientCertificates(
		CERT_PATH,
		KEY_PATH,
		CA_PATH,
		CLIENT_KEY_PASSWORD,
		REJECT_UNAUTHORIZED === "true"
	);
}

/**
 * Find a random load-balanced endpoint
 */
export function modelEndpoint(model: BackendModel): Endpoint {
	if (!model.endpoints) {
		return {
			host: "tgi",
			url: `${HF_API_ROOT}/${model.name}`,
			authorization: `Bearer ${HF_ACCESS_TOKEN}`,
			weight: 1,
		};
	}
	const endpoints = model.endpoints;
	const totalWeight = sum(endpoints.map((e) => e.weight));

	let random = Math.random() * totalWeight;
	for (const endpoint of endpoints) {
		if (random < endpoint.weight) {
			return endpoint;
		}
		random -= endpoint.weight;
	}

	throw new Error("Invalid config, no endpoint found");
}