File size: 2,106 Bytes
1b66f8d 7482c01 1b66f8d 7482c01 1b66f8d 7f827cf 1b66f8d 7f827cf 2aecb80 1b66f8d 91e621a 1b66f8d 8811ee0 91e621a 1b66f8d 91e621a ad02fa3 7482c01 1b66f8d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
<script lang="ts">
import ChatWindow from '$lib/components/chat/ChatWindow.svelte';
import { pendingMessage } from '$lib/stores/pendingMessage';
import { onMount } from 'svelte';
import type { PageData } from './$types';
import { page } from '$app/stores';
import { HfInference } from '@huggingface/inference';
import { invalidate } from '$app/navigation';
export let data: PageData;
$: messages = data.messages;
const hf = new HfInference();
let loading = false;
async function getTextGenerationStream(inputs: string) {
const response = hf.endpoint($page.url.href).textGenerationStream(
{
inputs,
parameters: {
// Taken from https://huggingface.co/spaces/huggingface/open-assistant-private-testing/blob/main/app.py#L54
temperature: 0.9,
top_p: 0.95,
repetition_penalty: 1.2,
top_k: 50,
// @ts-ignore
truncate: 1024,
watermark: false,
max_new_tokens: 1024,
stop: ['<|endoftext|>'],
return_full_text: false
}
},
{
use_cache: false
}
);
for await (const data of response) {
if (!data) break;
if (!data.token.special) {
const lastMessage = messages.at(-1);
if (lastMessage?.from !== 'assistant') {
// First token has a space at the beginning, trim it
messages = [...messages, { from: 'assistant', content: data.token.text.trimStart() }];
} else {
lastMessage.content += data.token.text;
messages = [...messages];
}
}
}
}
async function writeMessage(message: string) {
if (!message.trim()) return;
try {
loading = true;
messages = [...messages, { from: 'user', content: message }];
await getTextGenerationStream(message);
// Reload conversation order - doesn't seem to work
// await invalidate('/');
} finally {
loading = false;
}
}
onMount(async () => {
if ($pendingMessage) {
const val = $pendingMessage;
$pendingMessage = '';
if (messages.length === 0) {
writeMessage(val);
}
}
});
</script>
<ChatWindow disabled={loading} {messages} on:message={(message) => writeMessage(message.detail)} />
|