Option for running websearch locally (#563)

* Fully local websearch

* better comments

* fix else if

.env

@@ -14,6 +14,7 @@ OPENAI_API_KEY=#your openai api key here
 YDC_API_KEY=#your docs.you.com api key here
 SERPER_API_KEY=#your serper.dev api key here
 SERPAPI_KEY=#your serpapi key here
+USE_LOCAL_WEBSEARCH=#set to true to parse google results yourself, overrides other API keys
 
 # Parameters to enable open id login
 OPENID_CONFIG=`{

src/lib/server/websearch/runWebSearch.ts

@@ -15,6 +15,8 @@ import { getWebSearchProvider } from "./searchWeb";
 const MAX_N_PAGES_SCRAPE = 10 as const;
 const MAX_N_PAGES_EMBED = 5 as const;
 
+const DOMAIN_BLOCKLIST = ["youtube.com", "twitter.com"];
+
 export async function runWebSearch(
 	conv: Conversation,
 	prompt: string,
@@ -45,14 +47,14 @@ export async function runWebSearch(
 		const results = await searchWeb(webSearch.searchQuery);
 		webSearch.results =
 			(results.organic_results &&
-				results.organic_results.map((el: { title: string; link: string; text?: string }) => {
+				results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
 					const { title, link, text } = el;
 					const { hostname } = new URL(link);
 					return { title, link, hostname, text };
 				})) ??
 			[];
 		webSearch.results = webSearch.results
-			.filter(({ link }) => !link.includes("youtube.com")) // filter out youtube links
+			.filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
 			.slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
 
 		let paragraphChunks: { source: WebSearchSource; text: string }[] = [];

src/lib/server/websearch/searchWeb.ts

@@ -1,8 +1,9 @@
 import type { YouWebSearch } from "../../types/WebSearch";
 import { WebSearchProvider } from "../../types/WebSearch";
-import { SERPAPI_KEY, SERPER_API_KEY, YDC_API_KEY } from "$env/static/private";
+import { SERPAPI_KEY, SERPER_API_KEY, USE_LOCAL_WEBSEARCH, YDC_API_KEY } from "$env/static/private";
 import { getJson } from "serpapi";
 import type { GoogleParameters } from "serpapi";
+import { searchWebLocal } from "./searchWebLocal";
 
 // get which SERP api is providing web results
 export function getWebSearchProvider() {
@@ -11,6 +12,9 @@ export function getWebSearchProvider() {
 
 // Show result as JSON
 export async function searchWeb(query: string) {
+	if (USE_LOCAL_WEBSEARCH) {
+		return await searchWebLocal(query);
+	}
 	if (SERPER_API_KEY) {
 		return await searchWebSerper(query);
 	}

src/lib/server/websearch/searchWebLocal.ts

@@ -0,0 +1,45 @@
+import { JSDOM, VirtualConsole } from "jsdom";
+
+export async function searchWebLocal(query: string) {
+	const abortController = new AbortController();
+	setTimeout(() => abortController.abort(), 10000);
+
+	const htmlString = await fetch("https://www.google.com/search?hl=en&q=" + query, {
+		signal: abortController.signal,
+	})
+		.then((response) => response.text())
+		.catch();
+
+	const virtualConsole = new VirtualConsole();
+
+	virtualConsole.on("error", () => {
+		// No-op to skip console errors.
+	});
+
+	// put the html string into a DOM
+	const dom = new JSDOM(htmlString ?? "", {
+		virtualConsole,
+	});
+
+	const { document } = dom.window;
+	// get all a documents with href tag
+
+	const links = document.querySelectorAll("a");
+
+	if (!links.length) {
+		throw new Error(`webpage doesn't have any "a" element`);
+	}
+
+	// take url that start wirth /url?q=
+	// and do not contain google.com links
+	// and strip them up to '&sa='
+	const linksHref = Array.from(links)
+		.filter((el) => el.href?.startsWith("/url?q=") && !el.href.includes("google.com/"))
+		.map((el) => {
+			const link = el.href;
+			return link.slice("/url?q=".length, link.indexOf("&sa="));
+		});
+
+	// remove duplicate links and map links to the correct object shape
+	return { organic_results: [...new Set(linksHref)].map((link) => ({ link })) };
+}