nsarrazin's picture
nsarrazin HF staff
Refactor of websearch (#281)
6434339 unverified
raw
history blame
1.66 kB
import { JSDOM, VirtualConsole } from "jsdom";
function removeTags(node: Node) {
if (node.hasChildNodes()) {
node.childNodes.forEach((childNode) => {
if (node.nodeName === "SCRIPT" || node.nodeName === "STYLE") {
node.removeChild(childNode);
} else {
removeTags(childNode);
}
});
}
}
function naiveInnerText(node: Node): string {
const Node = node; // We need Node(DOM's Node) for the constants, but Node doesn't exist in the nodejs global space, and any Node instance references the constants through the prototype chain
return [...node.childNodes]
.map((childNode) => {
switch (childNode.nodeType) {
case Node.TEXT_NODE:
return node.textContent;
case Node.ELEMENT_NODE:
return naiveInnerText(childNode);
default:
return "";
}
})
.join("\n");
}
export async function parseWeb(url: string) {
const abortController = new AbortController();
setTimeout(() => abortController.abort(), 10000);
const htmlString = await fetch(url, { signal: abortController.signal })
.then((response) => response.text())
.catch((err) => console.log(err));
const virtualConsole = new VirtualConsole();
virtualConsole.on("error", () => {
// No-op to skip console errors.
});
// put the html string into a DOM
const dom = new JSDOM(htmlString ?? "", {
virtualConsole,
});
const body = dom.window.document.querySelector("body");
if (!body) throw new Error("body of the webpage is null");
removeTags(body);
// recursively extract text content from the body and then remove newlines and multiple spaces
const text = (naiveInnerText(body) ?? "").replace(/ {2}|\r\n|\n|\r/gm, "");
return text;
}