Update websearch prompting & summary prompting (#503)

* Use the task model for websearch query generation

* summary with examples in chat format

* Update to websearch prompting

- Removed web search template
- Added examples in `generateQuery`
- Passed previous questions when generating the web search query
- Added previous questions to the bottom of the context window in `buildPrompt` when using websearch

* wait for model if needed

* rm console.log

* get rid of regex

* added a few time related examples

* Update src/lib/server/websearch/generateQuery.ts

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>

* reformat yesterday

* test

* fix whitespace

* remove extra whitespace for mistral

* removed mistral extra whitespace

* Update src/lib/server/summarize.ts

Co-authored-by: Victor Muštar <victor.mustar@gmail.com>

* Update src/lib/server/summarize.ts

Co-authored-by: Victor Muštar <victor.mustar@gmail.com>

* add an emoji if none is found in the first three characters

---------

Co-authored-by: Mishig <mishig.davaadorj@coloradocollege.edu>
Co-authored-by: Victor Muštar <victor.mustar@gmail.com>

.env.template

@@ -99,7 +99,7 @@ MODELS=`[
       "description": "Mistral 7B is a new Apache 2.0 model, released by Mistral AI that outperforms Llama2 13B in benchmarks.",
       "websiteUrl": "https://mistral.ai/news/announcing-mistral-7b/",
       "preprompt": "",
-      "chatPromptTemplate" : "<s>{{#each messages}}{{#ifUser}}[INST] {{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}} {{content}} [/INST]{{/ifUser}}{{#ifAssistant}}{{content}}</s> {{/ifAssistant}}{{/each}}",
+      "chatPromptTemplate" : "<s>{{#each messages}}{{#ifUser}}[INST] {{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}}{{content}} [/INST]{{/ifUser}}{{#ifAssistant}}{{content}}</s>{{/ifAssistant}}{{/each}}",
       "parameters": {
         "temperature": 0.1,
         "top_p": 0.95,

src/lib/buildPrompt.ts

@@ -23,18 +23,28 @@ export async function buildPrompt({
 	preprompt,
 }: buildPromptOptions): Promise<string> {
 	if (webSearch && webSearch.context) {
+		const lastMsg = messages.slice(-1)[0];
 		const messagesWithoutLastUsrMsg = messages.slice(0, -1);
-		const lastUserMsg = messages.slice(-1)[0];
+		const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
+
+		const previousQuestions =
+			previousUserMessages.length > 0
+				? `Previous questions: \n${previousUserMessages
+						.map(({ content }) => `- ${content}`)
+						.join("\n")}`
+				: "";
 		const currentDate = format(new Date(), "MMMM d, yyyy");
 		messages = [
 			...messagesWithoutLastUsrMsg,
 			{
 				from: "user",
-				content: `Please answer my question "${lastUserMsg.content}" using the supplied context below (paragraphs from various websites). For the context, today is ${currentDate}: 
+				content: `I searched the web using the query: ${webSearch.searchQuery}. Today is ${currentDate} and here are the results:
 				=====================
 				${webSearch.context}
 				=====================
-				So my question is "${lastUserMsg.content}"`,
+				${previousQuestions}
+				Answer the question: ${lastMsg.content} 
+				`,
 			},
 		];
 	}

src/lib/server/generateFromDefaultEndpoint.ts

@@ -19,6 +19,7 @@ export async function generateFromDefaultEndpoint(
 		...smallModel.parameters,
 		...parameters,
 		return_full_text: false,
+		wait_for_model: true,
 	};
 
 	const randomEndpoint = modelEndpoint(smallModel);

src/lib/server/models.ts

@@ -1,5 +1,5 @@
 import { HF_ACCESS_TOKEN, MODELS, OLD_MODELS, TASK_MODEL } from "$env/static/private";
-import type { ChatTemplateInput, WebSearchQueryTemplateInput } from "$lib/types/Template";
+import type { ChatTemplateInput } from "$lib/types/Template";
 import { compileTemplate } from "$lib/utils/template";
 import { z } from "zod";
 
@@ -67,15 +67,6 @@ const modelsRaw = z
 						"{{/each}}" +
 						"{{assistantMessageToken}}"
 				),
-			webSearchQueryPromptTemplate: z
-				.string()
-				.default(
-					"{{userMessageToken}}" +
-						'My question is: "{{message.content}}". ' +
-						"Based on the conversation history (my previous questions are: {{previousMessages}}), give me an appropriate query to answer my question for google search. You should not say more than query. You should not say any words except the query. For the context, today is {{currentDate}}" +
-						"{{userMessageEndToken}}" +
-						"{{assistantMessageToken}}"
-				),
 			promptExamples: z
 				.array(
 					z.object({
@@ -104,10 +95,6 @@ export const models = await Promise.all(
 		userMessageEndToken: m?.userMessageEndToken || m?.messageEndToken,
 		assistantMessageEndToken: m?.assistantMessageEndToken || m?.messageEndToken,
 		chatPromptRender: compileTemplate<ChatTemplateInput>(m.chatPromptTemplate, m),
-		webSearchQueryPromptRender: compileTemplate<WebSearchQueryTemplateInput>(
-			m.webSearchQueryPromptTemplate,
-			m
-		),
 		id: m.id || m.name,
 		displayName: m.displayName || m.name,
 		preprompt: m.prepromptUrl ? await fetch(m.prepromptUrl).then((r) => r.text()) : m.preprompt,

src/lib/server/summarize.ts

@@ -1,39 +1,48 @@
-import { buildPrompt } from "$lib/buildPrompt";
-import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
-import { defaultModel } from "$lib/server/models";
 import { LLM_SUMMERIZATION } from "$env/static/private";
+import { generateFromDefaultEndpoint } from "$lib/server/generateFromDefaultEndpoint";
+import { smallModel } from "$lib/server/models";
+import type { Message } from "$lib/types/Message";
 
 export async function summarize(prompt: string) {
 	if (!LLM_SUMMERIZATION) {
 		return prompt.split(/\s+/g).slice(0, 5).join(" ");
 	}
-	const userPrompt = `Please summarize the following message: \n` + prompt;
-
-	const summaryPrompt = await buildPrompt({
-		messages: [{ from: "user", content: userPrompt }],
-		preprompt: `
-You are a summarization AI. Your task is to summarize user requests, in a single sentence of less than 5 words. Do not try to answer questions, just summarize the user's request. Start your answer with an emoji relevant to the summary."
-
-Example: "Who is the president of France ?"
-Summary: "🇫🇷 President of France request"
-
-Example: "What are the latest news ?"
-Summary: "📰 Latest news"
 
-Example: "Can you debug this python code?"
-Summary: "🐍 Python code debugging request"
-`,
-		model: defaultModel,
+	const messages: Array<Omit<Message, "id">> = [
+		{ from: "user", content: "Who is the president of Gabon?" },
+		{ from: "assistant", content: "🇬🇦 President of Gabon" },
+		{ from: "user", content: "Who is Julien Chaumond?" },
+		{ from: "assistant", content: "🧑 Julien Chaumond" },
+		{ from: "user", content: "what is 1 + 1?" },
+		{ from: "assistant", content: "🔢 Simple math operation" },
+		{ from: "user", content: "What are the latest news?" },
+		{ from: "assistant", content: "📰 Latest news" },
+		{ from: "user", content: "How to make a great cheesecake?" },
+		{ from: "assistant", content: "🍰 Cheesecake recipe" },
+		{ from: "user", content: "what is your favorite movie? do a short answer." },
+		{ from: "assistant", content: "🎥 Favorite movie" },
+		{ from: "user", content: "Explain the concept of artificial intelligence in one sentence" },
+		{ from: "assistant", content: "🤖 AI definition" },
+		{ from: "user", content: "Answer all my questions like chewbacca from now ok?" },
+		{ from: "assistant", content: "🐒 Answer as Chewbacca" },
+		{ from: "user", content: prompt },
+	];
+
+	const summaryPrompt = smallModel.chatPromptRender({
+		messages,
+		preprompt: `You are a summarization AI. You'll never answer a user's question directly, but instead summarize the user's request into a single short sentence of four words or less. Always start your answer with an emoji relevant to the summary.`,
 	});
 
-	const generated_text = await generateFromDefaultEndpoint(summaryPrompt).catch((e) => {
-		console.error(e);
-		return null;
-	});
-
-	if (generated_text) {
-		return generated_text;
-	}
-
-	return null;
+	return await generateFromDefaultEndpoint(summaryPrompt)
+		.then((summary) => {
+			// add an emoji if none is found in the first three characters
+			if (!/\p{Emoji}/u.test(summary.slice(0, 3))) {
+				return "💬 " + summary;
+			}
+			return summary;
+		})
+		.catch((e) => {
+			console.error(e);
+			return null;
+		});
 }

src/lib/server/websearch/generateQuery.ts

@@ -1,46 +1,71 @@
 import type { Message } from "$lib/types/Message";
 import { format } from "date-fns";
 import { generateFromDefaultEndpoint } from "../generateFromDefaultEndpoint";
-import { defaultModel } from "../models";
+import { smallModel } from "../models";
 
 export async function generateQuery(messages: Message[]) {
 	const currentDate = format(new Date(), "MMMM d, yyyy");
 	const userMessages = messages.filter(({ from }) => from === "user");
 	const previousUserMessages = userMessages.slice(0, -1);
+
 	const lastMessage = userMessages.slice(-1)[0];
-	const promptSearchQuery = defaultModel.webSearchQueryPromptRender({
-		message: lastMessage,
-		previousMessages: previousUserMessages.map(({ content }) => content).join(" "),
-		currentDate,
-	});
-	const searchQuery = await generateFromDefaultEndpoint(promptSearchQuery).then((query) => {
-		// example of generating google query:
-		// case 1
-		// user: tell me what happened yesterday
-		// LLM: google query is "news september 12, 2023"
-		// the regex below will try to capture the last "phrase" (i.e. words between quotes or double quotes or ticks)
-		// in this case, it is "news september 12, 2023"
-		// if there is no "phrase", we will just use the user query, which was "tell me what happened yesterday"
-		const regexLastPhrase = /("|'|`)((?:(?!\1).)+)\1$/;
-		let match = query.match(regexLastPhrase);
-		if (match) {
-			return match[2];
-		}
 
-		// case 2
-		// user: tell me what happened yesterday
-		// LLM: Here is a query: news september 12, 2023
-		// the regex below will try to capture the last sentences starting from :
-		// in this case, it is "news september 12, 2023"
-		// if there is no math, we will just use the user query, which was "tell me what happened yesterday"
-		const regexColon = /:\s(.*)$/;
-		match = query.match(regexColon);
-		if (match) {
-			return match[1];
-		}
+	const convQuery: Array<Omit<Message, "id">> = [
+		{
+			from: "user",
+			content: `Previous Questions:
+- Who is the president of France?
+
+Current Question: What about Mexico?
+`,
+		},
+		{
+			from: "assistant",
+			content: "President of Mexico",
+		},
+		{
+			from: "user",
+			content: `Previous questions: 
+- When is the next formula 1 grand prix?
+
+Current Question: Where is it being hosted ?`,
+		},
+		{
+			from: "assistant",
+			content: "location of next formula 1 grand prix",
+		},
+		{
+			from: "user",
+			content: "Current Question: What type of printhead does the Epson F2270 DTG printer use?",
+		},
+		{
+			from: "assistant",
+			content: "Epson F2270 DTG printer printhead",
+		},
+		{ from: "user", content: "What were the news yesterday ?" },
+		{
+			from: "assistant",
+			content: `news ${format(new Date(Date.now() - 864e5), "MMMM d, yyyy")}`,
+		},
+		{ from: "user", content: "What is the current weather in Paris ?" },
+		{ from: "assistant", content: `weather in Paris ${currentDate}` },
+		{
+			from: "user",
+			content:
+				(previousUserMessages.length > 0
+					? `Previous questions: \n${previousUserMessages
+							.map(({ content }) => `- ${content}`)
+							.join("\n")}`
+					: "") +
+				"\n\nCurrent Question:" +
+				lastMessage.content,
+		},
+	];
 
-		return lastMessage.content;
+	const promptQuery = smallModel.chatPromptRender({
+		preprompt: `You are tasked with generating web search queries. Give me an appropriate query to answer my question for google search. Answer with only the query. Today is ${currentDate}`,
+		messages: convQuery,
 	});
 
-	return searchQuery;
+	return await generateFromDefaultEndpoint(promptQuery);
 }

src/lib/types/Template.ts

@@ -12,9 +12,3 @@ export type ChatTemplateInput = {
 	messages: Pick<Message, "from" | "content">[];
 	preprompt?: string;
 };
-
-export type WebSearchQueryTemplateInput = {
-	message: Pick<Message, "from" | "content">;
-	previousMessages: string;
-	currentDate: string;
-};