auto modes

package.json

@@ -22,9 +22,9 @@
 		"@eslint/js": "^9.22.0",
 		"@floating-ui/dom": "^1.6.13",
 		"@huggingface/hub": "^2.6.12",
-		"@huggingface/inference": "^4.11.1",
-		"@huggingface/tasks": "^0.19.50",
-		"@huggingface/transformers": "^3.7.5",
+		"@huggingface/inference": "^4.13.0",
+		"@huggingface/tasks": "^0.19.62",
+		"@huggingface/transformers": "^3.7.6",
 		"@iconify-json/carbon": "^1.2.8",
 		"@iconify-json/line-md": "^1.2.11",
 		"@iconify-json/logos": "^1.2.9",

pnpm-lock.yaml

@@ -46,14 +46,14 @@ importers:
         specifier: ^2.6.12
         version: 2.6.12
       '@huggingface/inference':
-        specifier: ^4.11.1
-        version: 4.11.1
+        specifier: ^4.13.0
+        version: 4.13.0
       '@huggingface/tasks':
-        specifier: ^0.19.50
-        version: 0.19.50
+        specifier: ^0.19.62
+        version: 0.19.62
       '@huggingface/transformers':
-        specifier: ^3.7.5
-        version: 3.7.5
+        specifier: ^3.7.6
+        version: 3.7.6
       '@iconify-json/carbon':
         specifier: ^1.2.8
         version: 1.2.8
@@ -459,19 +459,19 @@ packages:
     engines: {node: '>=18'}
     hasBin: true
 
-  '@huggingface/inference@4.11.1':
-    resolution: {integrity: sha512-otClcigc7KUQfvG5wav7oVTBEVLsdKelEMuuQGXwBdPqrsJ7WZknAGCXJcXJoO2zLw9v518jgylkQnFwEBpGvw==}
+  '@huggingface/inference@4.13.0':
+    resolution: {integrity: sha512-oqBHbYoLHpNlEUJp/MG28gCR0JOUmLi9iKKd0oJfTAgTqaDSLhoCsIdS1XfLHbT/CrzZch33cARTR3bO4q/2yQ==}
     engines: {node: '>=18'}
 
   '@huggingface/jinja@0.5.1':
     resolution: {integrity: sha512-yUZLld4lrM9iFxHCwFQ7D1HW2MWMwSbeB7WzWqFYDWK+rEb+WldkLdAJxUPOmgICMHZLzZGVcVjFh3w/YGubng==}
     engines: {node: '>=18'}
 
-  '@huggingface/tasks@0.19.50':
-    resolution: {integrity: sha512-kHrfiDsJttkuwpdp7PgFiFHaK9rj+COJTIZ+221gk9vdF4B5QySW7hQT4aOGkwwZP+4qbXGhjMNGg/bxOq+LwA==}
+  '@huggingface/tasks@0.19.62':
+    resolution: {integrity: sha512-sv0ieUqggz2yWfVv14WuvdhmQA8mvSF/OD1quz5RKcysP+dmrPn0E9SwJsSksTsp1QwpAC396PukgPmDymPVhQ==}
 
-  '@huggingface/transformers@3.7.5':
-    resolution: {integrity: sha512-5jvrIwHyRXfOKVaGKYvUZM6ZjJKQXWeKzIOdKBE5pdzPSNzTwBNx5NdWcGElf4Ddv7Dl2mWsvJh+G5RnCUxMmA==}
+  '@huggingface/transformers@3.7.6':
+    resolution: {integrity: sha512-OYlIRY8vj8r/pNx2CdXcDHz4KqpEC+bUMKzdVW5Dx//gp4XRmK+/g8as0h3cssRQYT0vG1A6VCfZy8SV0F4RDQ==}
 
   '@humanfs/core@0.19.1':
     resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==}
@@ -3586,20 +3586,20 @@ snapshots:
 
   '@huggingface/hub@2.6.12':
     dependencies:
-      '@huggingface/tasks': 0.19.50
+      '@huggingface/tasks': 0.19.62
     optionalDependencies:
       cli-progress: 3.12.0
 
-  '@huggingface/inference@4.11.1':
+  '@huggingface/inference@4.13.0':
     dependencies:
       '@huggingface/jinja': 0.5.1
-      '@huggingface/tasks': 0.19.50
+      '@huggingface/tasks': 0.19.62
 
   '@huggingface/jinja@0.5.1': {}
 
-  '@huggingface/tasks@0.19.50': {}
+  '@huggingface/tasks@0.19.62': {}
 
-  '@huggingface/transformers@3.7.5':
+  '@huggingface/transformers@3.7.6':
     dependencies:
       '@huggingface/jinja': 0.5.1
       onnxruntime-node: 1.21.0

src/lib/components/inference-playground/conversation-header.svelte

@@ -27,7 +27,7 @@
 		if (!model) {
 			return;
 		}
-		conversation.update({ modelId: model.id, provider: undefined });
+		conversation.update({ modelId: model.id, provider: "auto", autoPolicy: "default" });
 	}
 
 	let nameSpace = $derived(conversation.model.id.split("/")[0] ?? "");

src/lib/components/inference-playground/provider-select.svelte

@@ -1,14 +1,14 @@
 <script lang="ts">
 	import type { ConversationClass } from "$lib/state/conversations.svelte";
-	import { models } from "$lib/state/models.svelte";
 	import { pricing } from "$lib/state/pricing.svelte";
 	import type { Model } from "$lib/types.js";
-	import { randomPick } from "$lib/utils/array.js";
 	import { cn } from "$lib/utils/cn.js";
 	import { Select } from "melt/builders";
 	import { run } from "svelte/legacy";
 	import IconCaret from "~icons/carbon/chevron-down";
 	import IconProvider from "../icon-provider.svelte";
+	import Tooltip from "../tooltip.svelte";
+	import IconInfo from "~icons/carbon/information";
 
 	interface Props {
 		conversation: ConversationClass & { model: Model };
@@ -20,11 +20,8 @@
 	function reset(providers: typeof conversation.model.inferenceProviderMapping) {
 		const validProvider = providers.find(p => p.provider === conversation.data.provider);
 		if (validProvider || conversation.data.provider === "auto") return;
-		if (providers) {
-			conversation.update({ provider: randomPick(providers)?.provider });
-		} else {
-			conversation.update({ modelId: randomPick(models.all)?.id });
-		}
+		// Default to auto provider if no valid provider is set
+		conversation.update({ provider: "auto", autoPolicy: "default" });
 	}
 
 	let providers = $derived(conversation.model.inferenceProviderMapping);
@@ -39,6 +36,13 @@
 		},
 	});
 
+	const autoPolicySelect = new Select<"default" | "fastest" | "cheapest", false>({
+		value: () => conversation.data.autoPolicy ?? "default",
+		onValueChange(v) {
+			conversation.update({ autoPolicy: v });
+		},
+	});
+
 	const nameMap: Record<string, string> = {
 		"sambanova": "SambaNova",
 		"fal": "fal",
@@ -82,6 +86,28 @@
 		const pd = pricing.getPricing(conversation.model.id, provider);
 		return pricing.formatPricing(pd);
 	}
+
+	function getAutoPolicyLabel(policy: "default" | "fastest" | "cheapest") {
+		switch (policy) {
+			case "default":
+				return "Default";
+			case "fastest":
+				return "Fastest";
+			case "cheapest":
+				return "Cheapest";
+		}
+	}
+
+	function getAutoPolicyDescription(policy: "default" | "fastest" | "cheapest") {
+		switch (policy) {
+			case "default":
+				return "Uses your preference order from Inference Provider settings";
+			case "fastest":
+				return "Selects the provider with highest throughput";
+			case "cheapest":
+				return "Selects the provider with lowest price per output token";
+		}
+	}
 </script>
 
 {#snippet providerDisplay(provider: string)}
@@ -131,4 +157,54 @@
 		{/each}
 		{@render option("auto")}
 	</div>
+
+	{#if conversation.data.provider === "auto"}
+		<div class="flex flex-col gap-1.5">
+			<div class="flex items-center gap-1 text-xs text-gray-600 dark:text-gray-400">
+				<span>Auto Policy</span>
+				<Tooltip>
+					{#snippet trigger(tooltip)}
+						<button class="flex items-center" {...tooltip.trigger}>
+							<IconInfo class="size-3" />
+						</button>
+					{/snippet}
+					{getAutoPolicyDescription(conversation.data.autoPolicy ?? "default")}
+				</Tooltip>
+			</div>
+			<button
+				{...autoPolicySelect.trigger}
+				class={cn(
+					"relative flex items-center justify-between gap-6 overflow-hidden rounded-lg border bg-gray-100/80 px-3 py-1.5 text-sm leading-tight whitespace-nowrap shadow-sm",
+					"hover:brightness-95 dark:border-gray-700 dark:bg-gray-800 dark:hover:brightness-110",
+				)}
+			>
+				{getAutoPolicyLabel(conversation.data.autoPolicy ?? "default")}
+				<div
+					class="absolute right-2 grid size-4 flex-none place-items-center rounded-sm bg-gray-100 text-xs dark:bg-gray-600"
+				>
+					<IconCaret />
+				</div>
+			</button>
+
+			<div {...autoPolicySelect.content} class="rounded-lg border bg-gray-100 dark:border-gray-700 dark:bg-gray-800">
+				{#snippet policyOption(policy: "default" | "fastest" | "cheapest", label: string)}
+					<div {...autoPolicySelect.getOption(policy)} class="group block w-full p-1 text-sm dark:text-white">
+						<div
+							class="rounded-md px-2 py-1.5 group-data-[highlighted]:bg-gray-200 dark:group-data-[highlighted]:bg-gray-700"
+						>
+							<div class="flex flex-col items-start gap-0.5">
+								<span>{label}</span>
+								<span class="text-xs text-gray-500 dark:text-gray-400">
+									{getAutoPolicyDescription(policy)}
+								</span>
+							</div>
+						</div>
+					</div>
+				{/snippet}
+				{@render policyOption("default", "Default")}
+				{@render policyOption("fastest", "Fastest")}
+				{@render policyOption("cheapest", "Cheapest")}
+			</div>
+		</div>
+	{/if}
 </div>

src/lib/state/conversations.svelte.ts

@@ -51,6 +51,9 @@ export class ConversationEntity {
 	@Fields.string()
 	provider?: string;
 
+	@Fields.string()
+	autoPolicy?: "default" | "fastest" | "cheapest";
+
 	@Fields.string()
 	projectId!: string;
 
@@ -87,6 +90,8 @@ function getDefaultConversation(projectId: string) {
 		messages: [],
 		streaming: true,
 		parseMarkdown: false,
+		provider: "auto",
+		autoPolicy: "default" as const,
 		createdAt: new Date(),
 	} satisfies Partial<ConversationEntityMembers>;
 }

src/lib/types.ts

@@ -15,7 +15,7 @@ export type Conversation = {
 	systemMessage: ConversationMessage;
 	streaming: boolean;
 	provider?: string;
-} & Pick<ConversationEntityMembers, "structuredOutput" | "extraParams">;
+} & Pick<ConversationEntityMembers, "structuredOutput" | "extraParams" | "autoPolicy">;
 
 export type ConversationWithCustomModel = Conversation & {
 	model: CustomModel;

src/lib/utils/business.svelte.ts

@@ -123,6 +123,15 @@ const tokenErrMessage = dev
 	? "Please set your Hugging Face token in the .env file"
 	: "Failed to connect to inference providers. Are you logged in?";
 
+function getModelIdWithPolicy(modelId: string, provider?: string, autoPolicy?: "default" | "fastest" | "cheapest") {
+	// When using auto provider, append policy suffix if it's not "default"
+	const policy = autoPolicy ?? "default";
+	if (provider === "auto" && policy !== "default") {
+		return `${modelId}:${policy}`;
+	}
+	return modelId;
+}
+
 export async function handleStreamingResponse(
 	conversation: ConversationClass | Conversation,
 	onChunk: (content: string) => void,
@@ -139,9 +148,11 @@ export async function handleStreamingResponse(
 	];
 	const parsed = await Promise.all(messages.map(parseMessage));
 
+	const modelIdWithPolicy = getModelIdWithPolicy(model.id, data.provider, data.autoPolicy);
+
 	const requestBody = {
 		model: {
-			id: model.id,
+			id: modelIdWithPolicy,
 			isCustom: isCustomModel(model),
 			accessToken: isCustomModel(model) ? model.accessToken : undefined,
 			endpointUrl: isCustomModel(model) ? model.endpointUrl : undefined,
@@ -198,9 +209,11 @@ export async function handleNonStreamingResponse(
 	];
 	const parsed = await Promise.all(messages.map(parseMessage));
 
+	const modelIdWithPolicy = getModelIdWithPolicy(model.id, data.provider, data.autoPolicy);
+
 	const requestBody = {
 		model: {
-			id: model.id,
+			id: modelIdWithPolicy,
 			isCustom: isCustomModel(model),
 			accessToken: isCustomModel(model) ? model.accessToken : undefined,
 			endpointUrl: isCustomModel(model) ? model.endpointUrl : undefined,
@@ -331,11 +344,15 @@ export function getInferenceSnippet(
 
 	const providerMapping = model.inferenceProviderMapping.find(p => p.provider === provider);
 	if (!providerMapping && provider !== "auto") return [];
+
+	// Apply auto policy suffix to model ID if using auto provider
+	const modelIdWithPolicy = getModelIdWithPolicy(model.id, data.provider, data.autoPolicy);
+
 	const allSnippets = snippets.getInferenceSnippets(
-		{ ...model, inference: "" },
+		{ ...model, inference: "", id: modelIdWithPolicy },
 		provider,
 		// eslint-disable-next-line @typescript-eslint/no-explicit-any
-		{ ...providerMapping, hfModelId: model.id } as any,
+		{ ...providerMapping, hfModelId: modelIdWithPolicy } as any,
 		{ ...opts, directRequest: false },
 	);