Improve config: add top_p, top_k, repetition_penalty and their correct ranges (#29)

src/lib/components/InferencePlayground/InferencePlayground.svelte

@@ -12,6 +12,7 @@
 	import { onDestroy } from 'svelte';
 	import { type ChatCompletionInputMessage } from '@huggingface/tasks';
 	import type { ModelEntryWithTokenizer } from '$lib/types';
+	import { defaultGenerationConfig } from './generationConfigSettings';
 
 	export let models: ModelEntryWithTokenizer[];
 
@@ -21,8 +22,9 @@
 		{
 			id: String(Math.random()),
 			model: models[0],
-			config: { temperature: 0.5, maxTokens: 2048, streaming: true },
-			messages: startMessages
+			config: defaultGenerationConfig,
+			messages: startMessages,
+			streaming: true
 		}
 	];
 
@@ -121,7 +123,7 @@
 			...conversation.messages
 		];
 
-		if (conversation.config.streaming) {
+		if (conversation.streaming) {
 			const streamingMessage = { role: 'assistant', content: '' };
 			conversation.messages = [...conversation.messages, streamingMessage];
 			const abortController = new AbortController();
@@ -400,7 +402,10 @@
 					</select>
 				</div>
 
-				<PlaygroundOptions bind:config={conversations[0].config} />
+				<PlaygroundOptions
+					bind:config={conversations[0].config}
+					bind:streaming={conversations[0].streaming}
+				/>
 				<div class="mt-auto">
 					<div class="mb-3 flex items-center justify-between gap-2">
 						<label

src/lib/components/InferencePlayground/InferencePlaygroundCodeSnippets.svelte

@@ -56,7 +56,7 @@
 # or
 yarn add @huggingface/inference`
 		});
-		if (conversation.config.streaming) {
+		if (conversation.streaming) {
 			snippets.push({
 				label: 'Streaming API',
 				code: `import { HfInference } from "@huggingface/inference"
@@ -111,7 +111,7 @@ console.log(out.choices[0].message);`
 			language: 'bash',
 			code: `pip install huggingface_hub`
 		});
-		if (conversation.config.streaming) {
+		if (conversation.streaming) {
 			snippets.push({
 				label: 'Streaming API',
 				code: `from huggingface_hub import InferenceClient
@@ -154,7 +154,7 @@ print(output.choices[0].message)`
 		const messagesStr = getMessages();
 		const snippets: Snippet[] = [];
 
-		if (conversation.config.streaming) {
+		if (conversation.streaming) {
 			snippets.push({
 				label: 'Streaming API',
 				code: `curl 'https://api-inference.huggingface.co/models/${conversation.model.id}/v1/chat/completions' \

src/lib/components/InferencePlayground/InferencePlaygroundConversation.svelte

@@ -34,7 +34,7 @@
 <div
 	class="flex max-h-[calc(100dvh-5.8rem)] flex-col overflow-y-auto overflow-x-hidden @container"
 	class:pointer-events-none={loading}
-	class:animate-pulse={loading && !conversation.config.streaming}
+	class:animate-pulse={loading && !conversation.streaming}
 	bind:this={messageContainer}
 >
 	{#if sideBySide}
@@ -65,6 +65,7 @@
 				>
 				<PlaygroundOptions
 					bind:config={conversation.config}
+					bind:streaming={conversation.streaming}
 					classNames="absolute top-8 right-0 w-56 invisible group-focus:visible hover:visible border border-gray-200/80 bg-white z-10 px-4 py-6 text-sm shadow-sm dark:border-gray-800 dark:bg-gray-800 rounded-xl"
 				/>
 			</button>

src/lib/components/InferencePlayground/InferencePlaygroundGenerationConfig.svelte

@@ -1,62 +1,45 @@
 <script lang="ts">
+	import { GENERATION_CONFIG_KEYS, GENERATION_CONFIG_SETTINGS } from './generationConfigSettings';
+
 	export let config;
+	export let streaming;
 	export let classNames = '';
 </script>
 
-<div class={classNames}>
-	<div>
-		<div class="flex items-center justify-between">
-			<label
-				for="temperature-range"
-				class="mb-2 block text-sm font-medium text-gray-900 dark:text-white">Temperature</label
-			>
+<div class="flex flex-col gap-y-5 {classNames}">
+	{#each GENERATION_CONFIG_KEYS as key}
+		{@const settings = GENERATION_CONFIG_SETTINGS[key]}
+		<div>
+			<div class="flex items-center justify-between">
+				<label
+					for="temperature-range"
+					class="mb-2 block text-sm font-medium text-gray-900 dark:text-white"
+					>{settings.label}</label
+				>
+				<input
+					type="number"
+					class="w-16 rounded border bg-transparent px-1 py-0.5 text-right text-sm dark:border-gray-700"
+					min={settings.min}
+					max={settings.max}
+					step={settings.step}
+					bind:value={config[key]}
+				/>
+			</div>
 			<input
-				type="number"
-				class="w-16 rounded border bg-transparent px-1 py-0.5 text-right text-sm dark:border-gray-700"
-				bind:value={config.temperature}
-				min="0"
-				max="1"
-				step="0.1"
+				id="temperature-range"
+				type="range"
+				min={settings.min}
+				max={settings.max}
+				step={settings.step}
+				bind:value={config[key]}
+				class="h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-200 accent-black dark:bg-gray-700 dark:accent-blue-500"
 			/>
 		</div>
-		<input
-			id="temperature-range"
-			type="range"
-			bind:value={config.temperature}
-			min="0"
-			max="1"
-			step="0.1"
-			class="h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-200 accent-black dark:bg-gray-700 dark:accent-blue-500"
-		/>
-	</div>
-	<div>
-		<div class="flex items-center justify-between">
-			<label
-				for="max-tokens-range"
-				class="mb-2 block text-sm font-medium text-gray-900 dark:text-white">Max tokens</label
-			>
-			<input
-				type="number"
-				class="w-20 rounded border bg-transparent px-1 py-0.5 text-right text-sm dark:border-gray-700"
-				bind:value={config.maxTokens}
-				min="0"
-				max="4096"
-				step="512"
-			/>
-		</div>
-		<input
-			id="max-tokens-range"
-			type="range"
-			bind:value={config.maxTokens}
-			min="0"
-			max="4096"
-			step="512"
-			class="h-2 w-full cursor-pointer appearance-none rounded-lg bg-gray-200 accent-black dark:bg-gray-700 dark:accent-blue-500"
-		/>
-	</div>
+	{/each}
+
 	<div class="mt-2">
 		<label class="flex cursor-pointer items-center justify-between">
-			<input type="checkbox" bind:checked={config.streaming} class="peer sr-only" />
+			<input type="checkbox" bind:checked={streaming} class="peer sr-only" />
 			<span class="text-sm font-medium text-gray-900 dark:text-gray-300">Streaming</span>
 			<div
 				class="peer relative h-5 w-9 rounded-full bg-gray-200 after:absolute after:start-[2px] after:top-[2px] after:h-4 after:w-4 after:rounded-full after:border after:border-gray-300 after:bg-white after:transition-all after:content-[''] peer-checked:bg-black peer-checked:after:translate-x-full peer-checked:after:border-white peer-focus:outline-none dark:border-gray-600 dark:bg-gray-700 dark:peer-checked:bg-blue-600"

src/lib/components/InferencePlayground/generationConfigSettings.ts

@@ -0,0 +1,61 @@
+interface GenerationKeySettings {
+	default: number;
+	step: number;
+	min: number;
+	max: number;
+	label: string;
+}
+
+export const GENERATION_CONFIG_SETTINGS: Record<string, GenerationKeySettings> = {
+	temperature: {
+		default: 0.7,
+		step: 0.01,
+		min: 0,
+		max: 2,
+		label: 'Temperature'
+	},
+	max_tokens: {
+		default: 0.7,
+		step: 512,
+		min: 1,
+		max: 8192, // changed dynamically based on model
+		label: 'Output Length'
+	},
+	top_p: {
+		default: 0.7,
+		step: 0.01,
+		min: 0,
+		max: 1,
+		label: 'Top-P'
+	},
+	top_k: {
+		default: 50,
+		step: 1,
+		min: 1,
+		max: 100,
+		label: 'Top-K'
+	},
+	repetition_penalty: {
+		default: 1,
+		step: 0.01,
+		min: 1,
+		max: 2,
+		label: 'Repetition Penalty'
+	}
+};
+
+export type GenerationConfigKey = keyof typeof GENERATION_CONFIG_SETTINGS;
+
+export const GENERATION_CONFIG_KEYS: GenerationConfigKey[] = Object.keys(
+	GENERATION_CONFIG_SETTINGS
+);
+
+export type GenerationConfig = Record<GenerationConfigKey, number>;
+
+export const defaultGenerationConfig = Object.keys(GENERATION_CONFIG_SETTINGS).reduce(
+	(acc, key) => {
+		acc[key] = GENERATION_CONFIG_SETTINGS[key].default;
+		return acc;
+	},
+	{} as GenerationConfig
+);

src/lib/types/index.d.ts

@@ -1,19 +1,13 @@
+import type { GenerationConfig } from '$lib/components/InferencePlayground/generationConfigSettings';
 import type { ModelEntry } from '@huggingface/hub';
 import type { ChatCompletionInputMessage } from '@huggingface/tasks';
 
-type Model = string;
-
-type GenerationConfig = {
-	temperature: number;
-	maxTokens: number;
-	streaming: boolean;
-};
-
 type Conversation = {
 	id: string;
 	model: ModelEntryWithTokenizer;
 	config: GenerationConfig;
 	messages: ChatCompletionInputMessage[];
+	streaming: boolean;
 };
 
 interface TokenizerConfig {