Baqir MuhammadBaqir nsarrazin HF staff commited on
Commit
e63c1d7
1 Parent(s): f09c5f3

Fixes "invalid url" error in websearch (#675)

Browse files

* Fixed #656

* lint

---------

Co-authored-by: muhammadbaqir1327 <muhammad.baqir@arbisoft.com>
Co-authored-by: Nathan Sarrazin <sarrazin.nathan@gmail.com>

src/lib/server/websearch/runWebSearch.ts CHANGED
@@ -48,11 +48,17 @@ export async function runWebSearch(
48
  webSearch.results =
49
  (results.organic_results &&
50
  results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
51
- const { title, link, text } = el;
52
- const { hostname } = new URL(link);
53
- return { title, link, hostname, text };
 
 
 
 
 
54
  })) ??
55
  [];
 
56
  webSearch.results = webSearch.results
57
  .filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
58
  .slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
 
48
  webSearch.results =
49
  (results.organic_results &&
50
  results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
51
+ try {
52
+ const { title, link, text } = el;
53
+ const { hostname } = new URL(link);
54
+ return { title, link, hostname, text };
55
+ } catch (e) {
56
+ // Ignore Errors
57
+ return null;
58
+ }
59
  })) ??
60
  [];
61
+ webSearch.results = webSearch.results.filter((value) => value !== null);
62
  webSearch.results = webSearch.results
63
  .filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
64
  .slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only