Img txt 2 txt (#67)

src/lib/components/debug-menu.svelte

@@ -8,6 +8,7 @@
 	import { showQuotaModal } from "./quota-modal.svelte";
 	import type { ToastData } from "./toaster.svelte.js";
 	import { addToast } from "./toaster.svelte.js";
+	import { models } from "$lib/state/models.svelte";
 
 	let innerWidth = $state<number>();
 	let innerHeight = $state<number>();
@@ -31,6 +32,12 @@
 				console.log(session.$);
 			},
 		},
+		{
+			label: "Log models to console",
+			cb: () => {
+				console.log(models.all);
+			},
+		},
 		{
 			label: "Test prompt",
 			cb: async () => {

src/lib/components/inference-playground/conversation.svelte

@@ -1,11 +1,11 @@
 <script lang="ts">
-	import { run } from "svelte/legacy";
-
 	import type { Conversation } from "$lib/types.js";
 
 	import IconPlus from "~icons/carbon/add";
 	import CodeSnippets from "./code-snippets.svelte";
 	import Message from "./message.svelte";
+	import { ScrollState } from "$lib/spells/scroll-state.svelte";
+	import { watch } from "runed";
 
 	interface Props {
 		conversation: Conversation;
@@ -15,35 +15,20 @@
 	}
 
 	let { conversation = $bindable(), loading, viewCode, compareActive }: Props = $props();
-
-	let shouldScrollToBottom = $state(true);
-	let isProgrammaticScroll = $state(true);
-	let conversationLength = $state(conversation.messages.length);
-
 	let messageContainer: HTMLDivElement | null = $state(null);
-
-	function scrollToBottom() {
-		if (messageContainer) {
-			isProgrammaticScroll = true;
-			messageContainer.scrollTop = messageContainer.scrollHeight;
-		}
-	}
-
-	run(() => {
-		if (conversation.messages.at(-1)) {
-			if (shouldScrollToBottom) {
-				scrollToBottom();
-			}
-		}
+	const scrollState = new ScrollState({
+		element: () => messageContainer,
+		offset: { bottom: 100 },
 	});
 
-	run(() => {
-		if (conversation.messages.length !== conversationLength) {
-			// enable automatic scrolling when new message was added
-			conversationLength = conversation.messages.length;
-			shouldScrollToBottom = true;
+	watch(
+		() => conversation.messages.at(-1)?.content,
+		() => {
+			const shouldScroll = scrollState.arrived.bottom && !scrollState.isScrolling;
+			if (!shouldScroll) return;
+			scrollState.scrollToBottom();
 		}
-	});
+	);
 
 	function addMessage() {
 		const msgs = conversation.messages.slice();
@@ -68,19 +53,13 @@
 		: 'max-h-[calc(100dvh-5.8rem-2.5rem-75px)] md:max-h-[calc(100dvh-5.8rem)]'}"
 	class:animate-pulse={loading && !conversation.streaming}
 	bind:this={messageContainer}
-	onscroll={() => {
-		// disable automatic scrolling is user initiates scroll
-		if (!isProgrammaticScroll) {
-			shouldScrollToBottom = false;
-		}
-		isProgrammaticScroll = false;
-	}}
+	id="test-this"
 >
 	{#if !viewCode}
 		{#each conversation.messages as _msg, idx}
 			<Message
-				bind:content={conversation.messages[idx]!.content}
-				role={conversation.messages[idx]!.role}
+				bind:message={conversation.messages[idx]!}
+				{conversation}
 				autofocus={idx === conversation.messages.length - 1}
 				{loading}
 				onDelete={() => deleteMessage(idx)}

src/lib/components/inference-playground/message.svelte

@@ -1,48 +1,140 @@
 <script lang="ts">
+	import Tooltip from "$lib/components/tooltip.svelte";
 	import { TextareaAutosize } from "$lib/spells/textarea-autosize.svelte.js";
-	import type { ConversationMessage } from "$lib/types.js";
+	import { PipelineTag, type Conversation, type ConversationMessage } from "$lib/types.js";
+	import { fileToDataURL } from "$lib/utils/file.js";
+	import { FileUpload } from "melt/builders";
+	import { fade } from "svelte/transition";
+	import IconImage from "~icons/carbon/image-reference";
 
 	type Props = {
-		content: ConversationMessage["content"];
-		role: ConversationMessage["role"];
+		conversation: Conversation;
+		message: ConversationMessage;
 		loading?: boolean;
 		autofocus?: boolean;
 		onDelete?: () => void;
 	};
 
-	let { content = $bindable(""), role, loading, autofocus, onDelete }: Props = $props();
+	let { message = $bindable(), conversation, loading, autofocus, onDelete }: Props = $props();
 
 	let element = $state<HTMLTextAreaElement>();
 	new TextareaAutosize({
-		styleProp: "minHeight",
 		element: () => element,
-		input: () => content,
+		input: () => message.content ?? "",
+	});
+
+	const canUploadImgs = $derived(
+		message.role === "user" && conversation.model.pipeline_tag === PipelineTag.ImageTextToText
+	);
+	const fileUpload = new FileUpload({
+		accept: "image/*",
+		async onAccept(file) {
+			if (!message.images) message.images = [];
+
+			const dataUrl = await fileToDataURL(file);
+			if (message.images.includes(dataUrl)) return;
+
+			message.images.push(await fileToDataURL(file));
+			// We're dealing with files ourselves, so we don't want fileUpload to have any internal state,
+			// to avoid conflicts
+			fileUpload.clear();
+		},
+		disabled: () => !canUploadImgs,
 	});
 </script>
 
 <div
-	class=" group/message group grid grid-cols-[1fr_2.5rem] items-start gap-2 border-b px-3.5 pt-4 pb-6 hover:bg-gray-100/70 @-2xl:grid-cols-[130px_1fr_2.5rem] @2xl:grid-rows-1 @2xl:gap-4 @2xl:px-6 dark:border-gray-800 dark:hover:bg-gray-800/30"
+	class="group/message group relative flex flex-col items-start gap-x-4 gap-y-2 border-b px-3.5 pt-4 pb-6 hover:bg-gray-100/70
+	 @2xl:px-6 dark:border-gray-800 dark:hover:bg-gray-800/30"
 	class:pointer-events-none={loading}
+	{...fileUpload.dropzone}
+	onclick={undefined}
 >
-	<div class="col-span-2 pt-3 pb-1 text-sm font-semibold uppercase @2xl:col-span-1 @2xl:pb-2">
-		{role}
+	<div class=" flex w-full flex-col items-start gap-x-4 gap-y-2 @2xl:flex-row">
+		{#if fileUpload.isDragging}
+			<div
+				class="absolute inset-2 z-10 flex flex-col items-center justify-center rounded-xl bg-gray-800/50 backdrop-blur-md"
+				transition:fade={{ duration: 100 }}
+			>
+				<IconImage />
+				<p>Drop the image here to upload</p>
+			</div>
+		{/if}
+
+		<div class="pt-3 text-sm font-semibold uppercase @2xl:basis-[130px]">
+			{message.role}
+		</div>
+		<div class="flex w-full gap-4">
+			<!-- svelte-ignore a11y_autofocus -->
+			<!-- svelte-ignore a11y_positive_tabindex -->
+			<textarea
+				bind:this={element}
+				{autofocus}
+				bind:value={message.content}
+				placeholder="Enter {message.role} message"
+				class="grow resize-none overflow-hidden rounded-lg bg-transparent px-2 py-2.5 ring-gray-100 outline-none group-hover/message:ring-3 hover:bg-white focus:bg-white focus:ring-3 @2xl:px-3 dark:ring-gray-600 dark:hover:bg-gray-900 dark:focus:bg-gray-900"
+				rows="1"
+				tabindex="2"
+			></textarea>
+
+			{#if canUploadImgs}
+				<Tooltip openDelay={250}>
+					{#snippet trigger(tooltip)}
+						<button
+							tabindex="0"
+							type="button"
+							class="mt-1.5 -mr-2 grid size-8 place-items-center rounded-lg border border-gray-200 bg-white text-xs font-medium
+			text-gray-900 group-focus-within/message:visible group-hover/message:visible
+			hover:bg-gray-100 hover:text-blue-700 focus:z-10
+			focus:ring-4 focus:ring-gray-100 focus:outline-hidden sm:invisible dark:border-gray-600
+			dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
+							{...tooltip.trigger}
+							{...fileUpload.trigger}
+						>
+							<IconImage />
+						</button>
+						<input {...fileUpload.input} />
+					{/snippet}
+					Add image
+				</Tooltip>
+			{/if}
+
+			<Tooltip>
+				{#snippet trigger(tooltip)}
+					<button
+						tabindex="0"
+						onclick={onDelete}
+						type="button"
+						class="mt-1.5 size-8 rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900
+			group-focus-within/message:visible group-hover/message:visible hover:bg-gray-100
+			hover:text-blue-700 focus:z-10 focus:ring-4
+			focus:ring-gray-100 focus:outline-hidden sm:invisible dark:border-gray-600 dark:bg-gray-800
+			dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
+						{...tooltip.trigger}
+					>
+						✕
+					</button>
+				{/snippet}
+				Delete
+			</Tooltip>
+		</div>
 	</div>
-	<!-- svelte-ignore a11y_autofocus -->
-	<!-- svelte-ignore a11y_positive_tabindex -->
-	<textarea
-		bind:this={element}
-		{autofocus}
-		bind:value={content}
-		placeholder="Enter {role} message"
-		class="resize-none overflow-hidden rounded-sm bg-transparent px-2 py-2.5 ring-gray-100 outline-none group-hover/message:ring-3 hover:resize-y hover:bg-white focus:resize-y focus:bg-white focus:ring-3 @2xl:px-3 dark:ring-gray-600 dark:hover:bg-gray-900 dark:focus:bg-gray-900"
-		rows="1"
-		tabindex="2"
-	></textarea>
-	<button
-		tabindex="0"
-		onclick={onDelete}
-		type="button"
-		class="mt-1.5 size-8 rounded-lg border border-gray-200 bg-white text-xs font-medium text-gray-900 group-hover/message:block hover:bg-gray-100 hover:text-blue-700 focus:z-10 focus:ring-4 focus:ring-gray-100 focus:outline-hidden sm:hidden dark:border-gray-600 dark:bg-gray-800 dark:text-gray-400 dark:hover:bg-gray-700 dark:hover:text-white dark:focus:ring-gray-700"
-		>✕</button
-	>
+	{#if message.images?.length}
+		<div class="mt-2">
+			<div class="flex items-center gap-2">
+				{#each message.images as img (img)}
+					<div class="group/img relative">
+						<img src={img} alt="uploaded" class="size-12 rounded-lg object-cover" />
+						<button
+							type="button"
+							onclick={() => (message.images = message.images?.filter(i => i !== img))}
+							class="invisible absolute -top-1 -right-1 grid size-5 place-items-center rounded-full bg-gray-800 text-xs text-white group-hover/img:visible hover:bg-gray-700"
+						>
+							✕
+						</button>
+					</div>
+				{/each}
+			</div>
+		</div>
+	{/if}
 </div>

src/lib/components/inference-playground/model-selector-modal.svelte

@@ -9,6 +9,7 @@
 	import { watch } from "runed";
 	import IconSearch from "~icons/carbon/search";
 	import IconStar from "~icons/carbon/star";
+	import IconEye from "~icons/carbon/view";
 
 	interface Props {
 		onModelSelect?: (model: string) => void;
@@ -136,7 +137,7 @@
 						}}
 					>
 						{#if trending}
-							<div class="lucide lucide-star mr-1.5 size-4 text-yellow-400">
+							<div class=" mr-1.5 size-4 text-yellow-400">
 								<IconStar />
 							</div>
 						{/if}
@@ -145,6 +146,13 @@
 								class="mx-1 text-gray-300 dark:text-gray-700">/</span
 							><span class="text-black dark:text-white">{modelName}</span></span
 						>
+						{#if model.pipeline_tag === "image-text-to-text"}
+							<div
+								class="ml-2 grid size-5 place-items-center rounded bg-gray-500/10 text-gray-500 dark:bg-gray-500/20 dark:text-gray-300"
+							>
+								<IconEye class="size-3.5" />
+							</div>
+						{/if}
 					</button>
 				{/snippet}
 				{#if trending.length > 0}

src/lib/components/inference-playground/utils.ts

@@ -1,8 +1,29 @@
-import type { Conversation, ModelWithTokenizer } from "$lib/types.js";
-import type { InferenceSnippet } from "@huggingface/tasks";
+import type { Conversation, ConversationMessage, ModelWithTokenizer } from "$lib/types.js";
+import type { ChatCompletionInputMessage, InferenceSnippet } from "@huggingface/tasks";
 import { type ChatCompletionOutputMessage } from "@huggingface/tasks";
 
 import { HfInference, snippets, type InferenceProvider } from "@huggingface/inference";
+type ChatCompletionInputMessageChunk =
+	NonNullable<ChatCompletionInputMessage["content"]> extends string | (infer U)[] ? U : never;
+
+function parseMessage(message: ConversationMessage): ChatCompletionInputMessage {
+	if (!message.images) return message;
+	return {
+		...message,
+		content: [
+			{
+				type: "text",
+				text: message.content ?? "",
+			},
+			...message.images.map(img => {
+				return {
+					type: "image_url",
+					image_url: { url: img },
+				} satisfies ChatCompletionInputMessageChunk;
+			}),
+		],
+	};
+}
 
 export async function handleStreamingResponse(
 	hf: HfInference,
@@ -19,7 +40,7 @@ export async function handleStreamingResponse(
 	for await (const chunk of hf.chatCompletionStream(
 		{
 			model: model.id,
-			messages,
+			messages: messages.map(parseMessage),
 			provider: conversation.provider,
 			...conversation.config,
 		},
@@ -44,7 +65,7 @@ export async function handleNonStreamingResponse(
 
 	const response = await hf.chatCompletion({
 		model: model.id,
-		messages,
+		messages: messages.map(parseMessage),
 		provider: conversation.provider,
 		...conversation.config,
 	});

src/lib/components/tooltip.svelte

@@ -0,0 +1,48 @@
+<script lang="ts">
+	import { Tooltip, type TooltipProps } from "melt/builders";
+	import { type ComponentProps, type Extracted } from "melt";
+	import type { Snippet } from "svelte";
+
+	interface Props {
+		children: Snippet;
+		trigger: Snippet<[Tooltip]>;
+		placement?: NonNullable<Extracted<TooltipProps["computePositionOptions"]>>["placement"];
+		openDelay?: ComponentProps<TooltipProps>["openDelay"];
+	}
+	const { children, trigger, placement = "top", openDelay }: Props = $props();
+
+	const tooltip = new Tooltip({
+		forceVisible: true,
+		computePositionOptions: () => ({ placement }),
+		openDelay: () => openDelay,
+	});
+</script>
+
+{@render trigger(tooltip)}
+
+<div {...tooltip.content} class="rounded-xl bg-white p-0 shadow-xl dark:bg-gray-700">
+	<div {...tooltip.arrow} class="rounded-tl"></div>
+	<p class="px-4 py-1 text-gray-700 dark:text-white">{@render children()}</p>
+</div>
+
+<style>
+	[data-melt-tooltip-content] {
+		border: 0;
+
+		position: absolute;
+		pointer-events: none;
+		opacity: 0;
+
+		transform: scale(0.9);
+
+		transition: 0.3s;
+		transition-property: opacity, transform;
+	}
+
+	[data-melt-tooltip-content][data-open] {
+		pointer-events: auto;
+		opacity: 1;
+
+		transform: scale(1);
+	}
+</style>

src/lib/spells/scroll-state.svelte.ts

@@ -0,0 +1,299 @@
+import type { MaybeGetter } from "$lib/types.js";
+import { AnimationFrames, useDebounce, useEventListener } from "runed";
+import { onMount } from "svelte";
+import { extract } from "./extract.svelte.js";
+import { noop } from "$lib/utils/noop.js";
+
+export interface ScrollStateOptions {
+	/**
+	 * The target element.
+	 */
+	element: MaybeGetter<HTMLElement | Window | Document | null | undefined>;
+
+	// /**
+	//  * Throttle time for scroll event, it’s disabled by default.
+	//  *
+	//  * @default 0
+	//  */
+	// throttle?: MaybeGetter<number | undefined>;
+
+	/**
+	 * The check time when scrolling ends.
+	 * This configuration will be setting to (throttle + idle) when the `throttle` is configured.
+	 *
+	 * @default 200
+	 */
+	idle?: MaybeGetter<number | undefined>;
+
+	/**
+	 * Offset arrived states by x pixels
+	 *
+	 */
+	offset?: MaybeGetter<
+		| {
+				left?: number;
+				right?: number;
+				top?: number;
+				bottom?: number;
+		  }
+		| undefined
+	>;
+
+	/**
+	 * Trigger it when scrolling.
+	 *
+	 */
+	onScroll?: (e: Event) => void;
+
+	/**
+	 * Trigger it when scrolling ends.
+	 *
+	 */
+	onStop?: (e: Event) => void;
+
+	/**
+	 * Listener options for scroll event.
+	 *
+	 * @default {capture: false, passive: true}
+	 */
+	eventListenerOptions?: AddEventListenerOptions;
+
+	/**
+	 * Optionally specify a scroll behavior of `auto` (default, not smooth scrolling) or
+	 * `smooth` (for smooth scrolling) which takes effect when changing the `x` or `y` refs.
+	 *
+	 * @default 'auto'
+	 */
+	behavior?: MaybeGetter<ScrollBehavior | undefined>;
+
+	/**
+	 * On error callback
+	 *
+	 * Default log error to `console.error`
+	 */
+	onError?: (error: unknown) => void;
+}
+
+/**
+ * We have to check if the scroll amount is close enough to some threshold in order to
+ * more accurately calculate arrivedState. This is because scrollTop/scrollLeft are non-rounded
+ * numbers, while scrollHeight/scrollWidth and clientHeight/clientWidth are rounded.
+ * https://developer.mozilla.org/en-US/docs/Web/API/Element/scrollHeight#determine_if_an_element_has_been_totally_scrolled
+ */
+const ARRIVED_STATE_THRESHOLD_PIXELS = 1;
+
+/**
+ * Reactive scroll.
+ *
+ * @see https://vueuse.org/useScroll for the inspiration behind this utility.
+ * @param element
+ * @param options
+ */
+export class ScrollState {
+	#options!: ScrollStateOptions;
+	element = $derived(extract(this.#options.element));
+	// throttle = $derived(extract(this.#options.throttle, 0));
+	idle = $derived(extract(this.#options.idle, 200));
+	offset = $derived(
+		extract(this.#options.offset, {
+			left: 0,
+			right: 0,
+			top: 0,
+			bottom: 0,
+		})
+	);
+	onScroll = $derived(this.#options.onScroll ?? noop);
+	onStop = $derived(this.#options.onStop ?? noop);
+	eventListenerOptions = $derived(
+		this.#options.eventListenerOptions ?? {
+			capture: false,
+			passive: true,
+		}
+	);
+	behavior = $derived(extract(this.#options.behavior, "auto"));
+	onError = $derived(
+		this.#options.onError ??
+			((e: unknown) => {
+				console.error(e);
+			})
+	);
+
+	/** State */
+	internalX = $state(0);
+	internalY = $state(0);
+
+	// Use a get/set pair for x and y because we want to write the value to the refs
+	// during a `scrollTo()` without firing additional `scrollTo()`s in the process.
+	#x = $derived(this.internalX);
+	get x() {
+		return this.#x;
+	}
+	set x(v) {
+		this.scrollTo(v, undefined);
+	}
+
+	#y = $derived(this.internalY);
+	get y() {
+		return this.#y;
+	}
+	set y(v) {
+		this.scrollTo(undefined, v);
+	}
+
+	isScrolling = $state(false);
+	arrived = $state({
+		left: true,
+		right: false,
+		top: true,
+		bottom: false,
+	});
+	directions = $state({
+		left: false,
+		right: false,
+		top: false,
+		bottom: false,
+	});
+
+	constructor(options: ScrollStateOptions) {
+		this.#options = options;
+
+		useEventListener(
+			() => this.element,
+			"scroll",
+			// throttle ? useThrottleFn(onScrollHandler, throttle, true, false) : onScrollHandler,
+			this.onScrollHandler,
+			this.eventListenerOptions
+		);
+
+		useEventListener(
+			() => this.element,
+			"scrollend",
+			e => this.onScrollEnd(e),
+			this.eventListenerOptions
+		);
+
+		onMount(() => {
+			this.setArrivedState();
+		});
+
+		// useResizeObserver(
+		// 	() => (isHtmlElement(this.element) ? this.element : null),
+		// 	() => {
+		// 		setTimeout(() => {
+		// 			this.setArrivedState();
+		// 		}, 100);
+		// 	}
+		// );
+
+		// overkill?
+		new AnimationFrames(() => this.setArrivedState());
+	}
+
+	setArrivedState = () => {
+		if (!window || !this.element) return;
+
+		const el: Element = ((this.element as Window)?.document?.documentElement ||
+			(this.element as Document)?.documentElement ||
+			(this.element as HTMLElement | SVGElement)) as Element;
+
+		const { display, flexDirection, direction } = getComputedStyle(el);
+		const directionMultipler = direction === "rtl" ? -1 : 1;
+
+		const scrollLeft = el.scrollLeft;
+		this.directions.left = scrollLeft < this.internalX;
+		this.directions.right = scrollLeft > this.internalX;
+
+		const left = scrollLeft * directionMultipler <= (this.offset.left || 0);
+		const right =
+			scrollLeft * directionMultipler + el.clientWidth >=
+			el.scrollWidth - (this.offset.right || 0) - ARRIVED_STATE_THRESHOLD_PIXELS;
+
+		if (display === "flex" && flexDirection === "row-reverse") {
+			this.arrived.left = right;
+			this.arrived.right = left;
+		} else {
+			this.arrived.left = left;
+			this.arrived.right = right;
+		}
+
+		this.internalX = scrollLeft;
+
+		let scrollTop = el.scrollTop;
+
+		// patch for mobile compatible
+		if (this.element === window.document && !scrollTop) scrollTop = window.document.body.scrollTop;
+
+		this.directions.top = scrollTop < this.internalY;
+		this.directions.bottom = scrollTop > this.internalY;
+		const top = scrollTop <= (this.offset.top || 0);
+		const bottom =
+			scrollTop + el.clientHeight >= el.scrollHeight - (this.offset.bottom || 0) - ARRIVED_STATE_THRESHOLD_PIXELS;
+
+		/**
+		 * reverse columns and rows behave exactly the other way around,
+		 * bottom is treated as top and top is treated as the negative version of bottom
+		 */
+		if (display === "flex" && flexDirection === "column-reverse") {
+			this.arrived.top = bottom;
+			this.arrived.bottom = top;
+		} else {
+			this.arrived.top = top;
+			this.arrived.bottom = bottom;
+		}
+
+		this.internalY = scrollTop;
+	};
+
+	onScrollHandler = (e: Event) => {
+		if (!window) return;
+
+		this.setArrivedState();
+
+		this.isScrolling = true;
+		this.onScrollEndDebounced(e);
+		this.onScroll(e);
+	};
+
+	scrollTo(x: number | undefined, y: number | undefined) {
+		if (!window) return;
+
+		(this.element instanceof Document ? window.document.body : this.element)?.scrollTo({
+			top: y ?? this.y,
+			left: x ?? this.x,
+			behavior: this.behavior,
+		});
+		const scrollContainer =
+			(this.element as Window)?.document?.documentElement ||
+			(this.element as Document)?.documentElement ||
+			(this.element as Element);
+		if (x != null) this.internalX = scrollContainer.scrollLeft;
+		if (y != null) this.internalY = scrollContainer.scrollTop;
+	}
+
+	scrollToTop() {
+		this.scrollTo(undefined, 0);
+	}
+
+	scrollToBottom() {
+		if (!window) return;
+
+		const scrollContainer =
+			(this.element as Window)?.document?.documentElement ||
+			(this.element as Document)?.documentElement ||
+			(this.element as Element);
+		this.scrollTo(undefined, scrollContainer.scrollHeight);
+	}
+
+	onScrollEnd = (e: Event) => {
+		// dedupe if support native scrollend event
+		if (!this.isScrolling) return;
+
+		this.isScrolling = false;
+		this.directions.left = false;
+		this.directions.right = false;
+		this.directions.top = false;
+		this.directions.bottom = false;
+		this.onStop(e);
+	};
+	onScrollEndDebounced = useDebounce(this.onScrollEnd, () => this.idle);
+}

src/lib/spells/textarea-autosize.svelte.ts

@@ -1,7 +1,7 @@
 import type { Getter } from "melt";
 import { extract } from "./extract.svelte.js";
 import { useResizeObserver, watch } from "runed";
-import { tick } from "svelte";
+import { onDestroy, tick } from "svelte";
 
 export interface TextareaAutosizeOptions {
 	/** Textarea element to autosize. */
@@ -15,26 +15,37 @@ export interface TextareaAutosizeOptions {
 	 * @default `height`
 	 **/
 	styleProp?: "height" | "minHeight";
+	/**
+	 * Maximum height of the textarea before enabling scrolling.
+	 * @default `undefined` (no maximum)
+	 */
+	maxHeight?: number;
 }
 
 export class TextareaAutosize {
 	#options: TextareaAutosizeOptions;
+	#resizeTimeout: number | null = null;
+	#hiddenTextarea: HTMLTextAreaElement | null = null;
+
 	element = $derived.by(() => extract(this.#options.element));
 	input = $derived.by(() => extract(this.#options.input));
 	styleProp = $derived.by(() => extract(this.#options.styleProp, "height"));
-
-	textareaScrollHeight = $state(1);
+	maxHeight = $derived.by(() => extract(this.#options.maxHeight, undefined));
+	textareaHeight = $state(0);
 	textareaOldWidth = $state(0);
 
 	constructor(options: TextareaAutosizeOptions) {
 		this.#options = options;
 
+		// Create hidden textarea for measurements
+		this.#createHiddenTextarea();
+
 		watch([() => this.input, () => this.element], () => {
 			tick().then(() => this.triggerResize());
 		});
 
 		watch(
-			() => this.textareaScrollHeight,
+			() => this.textareaHeight,
 			() => options?.onResize?.()
 		);
 
@@ -45,23 +56,106 @@ export class TextareaAutosize {
 				const { contentRect } = entry;
 				if (this.textareaOldWidth === contentRect.width) return;
 
-				requestAnimationFrame(() => {
-					this.textareaOldWidth = contentRect.width;
-					this.triggerResize();
-				});
+				this.textareaOldWidth = contentRect.width;
+				this.triggerResize();
 			}
 		);
+
+		onDestroy(() => {
+			// Clean up
+			if (this.#hiddenTextarea) {
+				this.#hiddenTextarea.remove();
+				this.#hiddenTextarea = null;
+			}
+
+			if (this.#resizeTimeout) {
+				window.cancelAnimationFrame(this.#resizeTimeout);
+				this.#resizeTimeout = null;
+			}
+		});
+	}
+
+	#createHiddenTextarea() {
+		// Create a hidden textarea that will be used for measurements
+		// This avoids layout shifts caused by manipulating the actual textarea
+		if (typeof window === "undefined") return;
+
+		this.#hiddenTextarea = document.createElement("textarea");
+		const style = this.#hiddenTextarea.style;
+
+		// Make it invisible but keep same text layout properties
+		style.visibility = "hidden";
+		style.position = "absolute";
+		style.overflow = "hidden";
+		style.height = "0";
+		style.top = "0";
+		style.left = "-9999px";
+
+		document.body.appendChild(this.#hiddenTextarea);
+	}
+
+	#copyStyles() {
+		if (!this.element || !this.#hiddenTextarea) return;
+
+		const computed = window.getComputedStyle(this.element);
+
+		// Copy all the styles that affect text layout
+		const stylesToCopy = [
+			"box-sizing",
+			"width",
+			"padding-top",
+			"padding-right",
+			"padding-bottom",
+			"padding-left",
+			"border-top-width",
+			"border-right-width",
+			"border-bottom-width",
+			"border-left-width",
+			"font-family",
+			"font-size",
+			"font-weight",
+			"font-style",
+			"letter-spacing",
+			"text-indent",
+			"text-transform",
+			"line-height",
+			"word-spacing",
+			"word-wrap",
+			"word-break",
+			"white-space",
+		];
+
+		stylesToCopy.forEach(style => {
+			this.#hiddenTextarea!.style.setProperty(style, computed.getPropertyValue(style));
+		});
+
+		// Ensure the width matches exactly
+		this.#hiddenTextarea.style.width = `${this.element.clientWidth}px`;
 	}
 
 	triggerResize = () => {
-		if (!this.element) return;
+		if (!this.element || !this.#hiddenTextarea) return;
+
+		// Copy current styles and content to hidden textarea
+		this.#copyStyles();
+		this.#hiddenTextarea.value = this.input || "";
 
-		let height = "";
+		// Measure the hidden textarea
+		const scrollHeight = this.#hiddenTextarea.scrollHeight;
 
-		this.element.style[this.styleProp] = "1px";
-		this.textareaScrollHeight = this.element?.scrollHeight;
-		height = `${this.textareaScrollHeight}px`;
+		// Apply the height, respecting maxHeight if set
+		let newHeight = scrollHeight;
+		if (this.maxHeight && newHeight > this.maxHeight) {
+			newHeight = this.maxHeight;
+			this.element.style.overflowY = "auto";
+		} else {
+			this.element.style.overflowY = "hidden";
+		}
 
-		this.element.style[this.styleProp] = height;
+		// Only update if height actually changed
+		if (this.textareaHeight !== newHeight) {
+			this.textareaHeight = newHeight;
+			this.element.style[this.styleProp] = `${newHeight}px`;
+		}
 	};
 }

src/lib/types.ts

@@ -3,6 +3,7 @@ import type { ChatCompletionInputMessage } from "@huggingface/tasks";
 
 export type ConversationMessage = Pick<ChatCompletionInputMessage, "name" | "role" | "tool_calls"> & {
 	content?: string;
+	images?: string[];
 };
 
 export type Conversation = {
@@ -169,6 +170,7 @@ export enum LibraryName {
 
 export enum PipelineTag {
 	TextGeneration = "text-generation",
+	ImageTextToText = "image-text-to-text",
 }
 
 export type MaybeGetter<T> = T | (() => T);

src/lib/utils/file.ts

@@ -0,0 +1,15 @@
+export function fileToDataURL(file: File): Promise<string> {
+	return new Promise((resolve, reject) => {
+		const reader = new FileReader();
+
+		reader.onload = function (event) {
+			resolve(event.target?.result as string);
+		};
+
+		reader.onerror = function (error) {
+			reject(error);
+		};
+
+		reader.readAsDataURL(file);
+	});
+}

src/routes/+layout.svelte

@@ -3,6 +3,7 @@
 	import Prompts from "$lib/components/prompts.svelte";
 	import QuotaModal from "$lib/components/quota-modal.svelte";
 	import "../app.css";
+
 	interface Props {
 		children?: import("svelte").Snippet;
 	}

src/routes/api/models/+server.ts

@@ -5,68 +5,117 @@ import { dev } from "$app/environment";
 
 let cache: ModelWithTokenizer[] | undefined;
 
+const headers: HeadersInit = {
+	"Upgrade-Insecure-Requests": "1",
+	"Sec-Fetch-Dest": "document",
+	"Sec-Fetch-Mode": "navigate",
+	"Sec-Fetch-Site": "none",
+	"Sec-Fetch-User": "?1",
+	"Priority": "u=0, i",
+	"Pragma": "no-cache",
+	"Cache-Control": "no-cache",
+};
+
+const requestInit: RequestInit = {
+	credentials: "include",
+	headers,
+	method: "GET",
+	mode: "cors",
+};
+
+interface ApiQueryParams {
+	pipeline_tag?: "text-generation" | "image-text-to-text";
+	filter: string;
+	inference_provider: string;
+	limit: number;
+	expand: string[];
+}
+
+const queryParams: ApiQueryParams = {
+	filter: "conversational",
+	inference_provider: "all",
+	limit: 100,
+	expand: ["inferenceProviderMapping", "config", "library_name", "pipeline_tag", "tags", "mask_token", "trendingScore"],
+};
+
+const baseUrl = "https://huggingface.co/api/models";
+
+function buildApiUrl(params: ApiQueryParams): string {
+	const url = new URL(baseUrl);
+	// Add simple params
+	Object.entries(params).forEach(([key, value]) => {
+		if (!Array.isArray(value)) {
+			url.searchParams.append(key, String(value));
+		}
+	});
+	// Handle array params specially
+	params.expand.forEach(item => {
+		url.searchParams.append("expand[]", item);
+	});
+	return url.toString();
+}
+
 export const GET: RequestHandler = async ({ fetch }) => {
 	if (cache?.length && dev) {
 		console.log("Skipping load, using in memory cache");
 		return json(cache);
 	}
 
-	const apiUrl =
-		"https://huggingface.co/api/models?pipeline_tag=text-generation&filter=conversational&inference_provider=all&limit=100&expand[]=inferenceProviderMapping&expand[]=config&expand[]=library_name&expand[]=pipeline_tag&expand[]=tags&expand[]=mask_token&expand[]=trendingScore";
-
-	const res = await fetch(apiUrl, {
-		credentials: "include",
-		headers: {
-			"Upgrade-Insecure-Requests": "1",
-			"Sec-Fetch-Dest": "document",
-			"Sec-Fetch-Mode": "navigate",
-			"Sec-Fetch-Site": "none",
-			"Sec-Fetch-User": "?1",
-			"Priority": "u=0, i",
-			"Pragma": "no-cache",
-			"Cache-Control": "no-cache",
-		},
-		method: "GET",
-		mode: "cors",
-	});
-
-	if (!res.ok) {
-		console.error(`Error fetching warm models`, res.status, res.statusText);
-		return json({ models: [] });
-	}
+	try {
+		// Fetch both types of models in parallel
+		const textGenPromise = fetch(buildApiUrl({ ...queryParams, pipeline_tag: "text-generation" }), requestInit);
+		const imgText2TextPromise = fetch(buildApiUrl({ ...queryParams, pipeline_tag: "image-text-to-text" }), requestInit);
+		const [textGenResponse, imgText2TextResponse] = await Promise.all([textGenPromise, imgText2TextPromise]);
 
-	const compatibleModels: Model[] = await res.json();
-	compatibleModels.sort((a, b) => a.id.toLowerCase().localeCompare(b.id.toLowerCase()));
-
-	const promises = compatibleModels.map(async model => {
-		const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
-		const res = await fetch(configUrl, {
-			credentials: "include",
-			headers: {
-				"Upgrade-Insecure-Requests": "1",
-				"Sec-Fetch-Dest": "document",
-				"Sec-Fetch-Mode": "navigate",
-				"Sec-Fetch-Site": "none",
-				"Sec-Fetch-User": "?1",
-				"Priority": "u=0, i",
-				"Pragma": "no-cache",
-				"Cache-Control": "no-cache",
-			},
-			method: "GET",
-			mode: "cors",
-		});
+		if (!textGenResponse.ok) {
+			console.error(`Error fetching text-generation models`, textGenResponse.status, textGenResponse.statusText);
+		}
 
-		if (!res.ok) {
-			// console.error(`Error fetching tokenizer file for ${model.id}`, res.status, res.statusText);
-			return null; // Ignore failed requests by returning null
+		if (!imgText2TextResponse.ok) {
+			console.error(
+				`Error fetching image-text-to-text models`,
+				imgText2TextResponse.status,
+				imgText2TextResponse.statusText
+			);
 		}
 
-		const tokenizerConfig = await res.json();
-		return { ...model, tokenizerConfig } satisfies ModelWithTokenizer;
-	});
+		// Parse the responses
+		const textGenModels: Model[] = textGenResponse.ok ? await textGenResponse.json() : [];
+		const imgText2TextModels: Model[] = imgText2TextResponse.ok ? await imgText2TextResponse.json() : [];
+
+		// Combine the models
+		const compatibleModels: Model[] = [...textGenModels, ...imgText2TextModels];
+
+		// Sort the models
+		compatibleModels.sort((a, b) => a.id.toLowerCase().localeCompare(b.id.toLowerCase()));
 
-	const models: ModelWithTokenizer[] = (await Promise.all(promises)).filter(model => model !== null);
-	cache = models;
+		// Fetch tokenizer configs for each model
+		const promises = compatibleModels.map(async model => {
+			const configUrl = `https://huggingface.co/${model.id}/raw/main/tokenizer_config.json`;
+			const res = await fetch(configUrl, {
+				credentials: "include",
+				headers,
+				method: "GET",
+				mode: "cors",
+			});
 
-	return json(cache);
+			if (!res.ok) {
+				// console.error(`Error fetching tokenizer file for ${model.id}`, res.status, res.statusText);
+				return null; // Ignore failed requests by returning null
+			}
+
+			const tokenizerConfig = await res.json();
+			return { ...model, tokenizerConfig } satisfies ModelWithTokenizer;
+		});
+
+		const models: ModelWithTokenizer[] = (await Promise.all(promises)).filter(
+			(model): model is ModelWithTokenizer => model !== null
+		);
+		cache = models;
+
+		return json(cache);
+	} catch (error) {
+		console.error("Error fetching models:", error);
+		return json([]);
+	}
 };