File size: 1,312 Bytes
0c4cf03 d2a650e 0c4cf03 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 |
import { defaultModel } from "$lib/server/models";
import { modelEndpoint } from "./modelEndpoint";
import { textGeneration } from "@huggingface/inference";
import { trimSuffix } from "$lib/utils/trimSuffix";
import { trimPrefix } from "$lib/utils/trimPrefix";
import { PUBLIC_SEP_TOKEN } from "$lib/constants/publicSepToken";
interface Parameters {
temperature: number;
truncate: number;
max_new_tokens: number;
stop: string[];
}
export async function generateFromDefaultEndpoint(
prompt: string,
parameters?: Partial<Parameters>
) {
const newParameters = {
...defaultModel.parameters,
...parameters,
return_full_text: false,
};
const endpoint = modelEndpoint(defaultModel);
let { generated_text } = await textGeneration(
{
model: endpoint.url,
inputs: prompt,
parameters: newParameters,
},
{
fetch: (url, options) =>
fetch(url, {
...options,
headers: { ...options?.headers, Authorization: endpoint.authorization },
}),
}
);
generated_text = trimSuffix(
trimPrefix(generated_text, "<|startoftext|>"),
PUBLIC_SEP_TOKEN
).trimEnd();
for (const stop of [...(newParameters?.stop ?? []), "<|endoftext|>"]) {
if (generated_text.endsWith(stop)) {
generated_text = generated_text.slice(0, -stop.length).trimEnd();
}
}
return generated_text;
}
|