Mishig commited on
Commit
2e2f16c
1 Parent(s): 506e596

[Websearch] change context schema (#944)

Browse files
src/lib/server/preprocessMessages.ts CHANGED
@@ -11,8 +11,14 @@ export async function preprocessMessages(
11
  ): Promise<Message[]> {
12
  return await Promise.all(
13
  structuredClone(messages).map(async (message, idx) => {
 
 
 
 
 
 
14
  // start by adding websearch to the last message
15
- if (idx === messages.length - 1 && webSearch && webSearch.context) {
16
  const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? "";
17
  const previousQuestions = messages
18
  .filter((el) => el.from === "user")
@@ -23,7 +29,7 @@ export async function preprocessMessages(
23
  message.content = `I searched the web using the query: ${webSearch.searchQuery}.
24
  Today is ${currentDate} and here are the results:
25
  =====================
26
- ${webSearch.context}
27
  =====================
28
  ${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""}
29
  Answer the question: ${lastQuestion}`;
 
11
  ): Promise<Message[]> {
12
  return await Promise.all(
13
  structuredClone(messages).map(async (message, idx) => {
14
+ const webSearchContext = webSearch?.contextSources
15
+ .map(({ context }) => context)
16
+ .flat()
17
+ .sort((a, b) => a.idx - b.idx)
18
+ .map(({ text }) => text)
19
+ .join(" ");
20
  // start by adding websearch to the last message
21
+ if (idx === messages.length - 1 && webSearch && webSearchContext?.trim()) {
22
  const lastQuestion = messages.findLast((el) => el.from === "user")?.content ?? "";
23
  const previousQuestions = messages
24
  .filter((el) => el.from === "user")
 
29
  message.content = `I searched the web using the query: ${webSearch.searchQuery}.
30
  Today is ${currentDate} and here are the results:
31
  =====================
32
+ ${webSearchContext}
33
  =====================
34
  ${previousQuestions.length > 0 ? `Previous questions: \n- ${previousQuestions.join("\n- ")}` : ""}
35
  Answer the question: ${lastQuestion}`;
src/lib/server/websearch/runWebSearch.ts CHANGED
@@ -36,7 +36,6 @@ export async function runWebSearch(
36
  prompt,
37
  searchQuery: "",
38
  results: [],
39
- context: "",
40
  contextSources: [],
41
  createdAt: new Date(),
42
  updatedAt: new Date(),
@@ -153,14 +152,15 @@ export async function runWebSearch(
153
  const indices = await findSimilarSentences(embeddingModel, prompt, texts, {
154
  topK: topKClosestParagraphs,
155
  });
156
- webSearch.context = indices.map((idx) => texts[idx]).join("");
157
 
158
- const usedSources = new Set<string>();
159
  for (const idx of indices) {
160
  const { source } = paragraphChunks[idx];
161
- if (!usedSources.has(source.link)) {
162
- usedSources.add(source.link);
163
- webSearch.contextSources.push(source);
 
 
 
164
  }
165
  }
166
  updatePad({
 
36
  prompt,
37
  searchQuery: "",
38
  results: [],
 
39
  contextSources: [],
40
  createdAt: new Date(),
41
  updatedAt: new Date(),
 
152
  const indices = await findSimilarSentences(embeddingModel, prompt, texts, {
153
  topK: topKClosestParagraphs,
154
  });
 
155
 
 
156
  for (const idx of indices) {
157
  const { source } = paragraphChunks[idx];
158
+ const contextWithId = { idx, text: texts[idx] };
159
+ const usedSource = webSearch.contextSources.find((cSource) => cSource.link === source.link);
160
+ if (usedSource) {
161
+ usedSource.context.push(contextWithId);
162
+ } else {
163
+ webSearch.contextSources.push({ ...source, context: [contextWithId] });
164
  }
165
  }
166
  updatePad({
src/lib/types/WebSearch.ts CHANGED
@@ -10,8 +10,7 @@ export interface WebSearch extends Timestamps {
10
 
11
  searchQuery: string;
12
  results: WebSearchSource[];
13
- context: string;
14
- contextSources: WebSearchSource[];
15
  }
16
 
17
  export interface WebSearchSource {
@@ -21,6 +20,10 @@ export interface WebSearchSource {
21
  text?: string; // You.com provides text of webpage right away
22
  }
23
 
 
 
 
 
24
  export type WebSearchMessageSources = {
25
  type: "sources";
26
  sources: WebSearchSource[];
 
10
 
11
  searchQuery: string;
12
  results: WebSearchSource[];
13
+ contextSources: WebSearchUsedSource[];
 
14
  }
15
 
16
  export interface WebSearchSource {
 
20
  text?: string; // You.com provides text of webpage right away
21
  }
22
 
23
+ export interface WebSearchUsedSource extends WebSearchSource {
24
+ context: { idx: number; text: string }[];
25
+ }
26
+
27
  export type WebSearchMessageSources = {
28
  type: "sources";
29
  sources: WebSearchSource[];