move some tasks to small model (#479)

.env

@@ -61,6 +61,7 @@ MODELS=`[
   }
 ]`
 OLD_MODELS=`[]`# any removed models, `{ name: string, displayName?: string, id?: string }`
+TASK_MODEL='' # name of the model used for tasks such as summarizing title, creating query, etc.
 
 PUBLIC_ORIGIN=#https://huggingface.co
 PUBLIC_SHARE_PREFIX=#https://hf.co/chat

src/lib/server/generateFromDefaultEndpoint.ts

@@ -1,4 +1,4 @@
-import { defaultModel } from "$lib/server/models";
+import { smallModel } from "$lib/server/models";
 import { modelEndpoint } from "./modelEndpoint";
 import { trimSuffix } from "$lib/utils/trimSuffix";
 import { trimPrefix } from "$lib/utils/trimPrefix";
@@ -16,12 +16,12 @@ export async function generateFromDefaultEndpoint(
 	parameters?: Partial<Parameters>
 ): Promise<string> {
 	const newParameters = {
-		...defaultModel.parameters,
+		...smallModel.parameters,
 		...parameters,
 		return_full_text: false,
 	};
 
-	const randomEndpoint = modelEndpoint(defaultModel);
+	const randomEndpoint = modelEndpoint(smallModel);
 
 	const abortController = new AbortController();
 

src/lib/server/models.ts

@@ -1,4 +1,4 @@
-import { HF_ACCESS_TOKEN, MODELS, OLD_MODELS } from "$env/static/private";
+import { HF_ACCESS_TOKEN, MODELS, OLD_MODELS, TASK_MODEL } from "$env/static/private";
 import type { ChatTemplateInput, WebSearchQueryTemplateInput } from "$lib/types/Template";
 import { compileTemplate } from "$lib/utils/template";
 import { z } from "zod";
@@ -133,6 +133,8 @@ export type Endpoint = z.infer<typeof endpoint>;
 
 export const defaultModel = models[0];
 
+export const smallModel = models.find((m) => m.name === TASK_MODEL) || defaultModel;
+
 export const validateModel = (_models: BackendModel[]) => {
 	// Zod enum function requires 2 parameters
 	return z.enum([_models[0].id, ..._models.slice(1).map((m) => m.id)]);

src/lib/server/summarize.ts

@@ -7,8 +7,18 @@ export async function summarize(prompt: string) {
 
 	const summaryPrompt = await buildPrompt({
 		messages: [{ from: "user", content: userPrompt }],
-		preprompt:
-			"You are a summarization AI. Your task is to summarize user requests, in a single sentence of less than 5 words. Do not try to answer questions, just summarize the user's request.",
+		preprompt: `
+You are a summarization AI. Your task is to summarize user requests, in a single sentence of less than 5 words. Do not try to answer questions, just summarize the user's request. Start your answer with an emoji relevant to the summary."
+
+Example: "Who is the president of France ?"
+Summary: "🇫🇷 President of France request"
+
+Example: "What are the latest news ?"
+Summary: "📰 Latest news"
+
+Example: "Can you debug this python code?"
+Summary: "🐍 Python code debugging request"
+`,
 		model: defaultModel,
 	});
 

src/routes/conversation/[id]/+page.svelte

@@ -161,8 +161,6 @@
 								}
 							} else if (update.type === "webSearch") {
 								webSearchMessages = [...webSearchMessages, update];
-							} else {
-								console.log();
 							}
 						} catch (parseError) {
 							// in case of parsing error we wait for the next message