Repository: Mintplex-Labs/anything-llm
File: collector/processLink/convert/generic.js (JavaScript)

Code before:
const {
  PuppeteerWebBaseLoader,
} = require("langchain/document_loaders/web/puppeteer");
const { writeToServerDocuments } = require("../../utils/files");
const { tokenizeString } = require("../../utils/tokenizer");
const { default: slugify } = require("slugify");
const { getContentType } = require("../helpers/getContentType");
const { downloadFileToHotDir } = require("../helpers/downloadFileToHotDir");
const { processSingleFile } = require("../../processSingleFile");
const path = require("path");
const { ACCEPTED_MIMES } = require("../../utils/constants");

/**
 * Scrape a generic URL and return the content in the specified format
 * @param {Object} config - The configuration object
 * @param {string} config.link - The URL to scrape
 * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
 * @param {boolean} config.processAsDocument - Whether to process the content as a document or return the content directly. Default is true
 * @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
 * @param {{[key: string]: string}} config.metadata - Metadata to use when creating the document
 * @returns {Promise<Object>} - The content of the page
 */
async function scrapeGenericUrl({
  link,
  processAsDocument = true,
  scraperHeaders = {},
  metadata = {},
}) {
  // Get the content type of the link
  const contentTypeResult = await getContentType(link);
  // If the retrieving the content type failed, return an error
  if (!contentTypeResult.success || !contentTypeResult.contentType) {
    return {
      success: false,
      reason: contentTypeResult.reason,
      documents: [],
    };
  }
  const contentType = contentTypeResult.contentType;

  if (contentType in ACCEPTED_MIMES) {
    // If the content type is HTML or text, get the content of the web page with puppeteer
    if (contentType === "text/html" || contentType === "text/plain") {
      const content = await getPageContent({
        link,
        captureAs: "text",
        headers: scraperHeaders,
      });

      if (!content.length) {
        console.error(`Resulting URL content was empty at ${link}.`);

Review comment (bug):
This log is actually very useful for debugging, observability, and when someone opens a GH issue "why did my website not get scraped" and it allows us to easily repro.

It is also just nice to have just to know the collector is working via logs.

Code after:
const path = require("path");
const {
  PuppeteerWebBaseLoader,
} = require("langchain/document_loaders/web/puppeteer");
const { writeToServerDocuments } = require("../../utils/files");
const { tokenizeString } = require("../../utils/tokenizer");
const { default: slugify } = require("slugify");
const { getContentTypeFromURL, returnResult } = require("../helpers");
const { processSingleFile } = require("../../processSingleFile");
const { downloadURIToFile } = require("../../utils/downloadURIToFile");
const { ACCEPTED_MIMES } = require("../../utils/constants");
const RuntimeSettings = require("../../utils/runtimeSettings");

/**
 * Scrape a generic URL and return the content in the specified format
 * @param {Object} config - The configuration object
 * @param {string} config.link - The URL to scrape
 * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
 * @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
 * @param {{[key: string]: string}} config.metadata - Metadata to use when creating the document
 * @param {boolean} config.saveAsDocument - Whether to save the content as a document. Default is true
 * @returns {Promise<Object>} - The content of the page
 */
async function scrapeGenericUrl({
  link,
  captureAs = "text",
  scraperHeaders = {},
  metadata = {},
  saveAsDocument = true,
}) {
  /** @type {'web' | 'file'} */
  let processVia = "web";
  console.log(`-- Working URL ${link} => (captureAs: ${captureAs}) --`);

  const contentType = await getContentTypeFromURL(link)
    .then((result) => {
      // If there is a reason, log it, but continue with the process
      if (!!result.reason) console.error(result.reason);
      return result.contentType;
    })
    .catch((error) => {
      console.error("Error getting content type from URL", error);
      return null;
    });

  // If the content is unlikely to be a webpage, assume it is a file and process it as a file
  if (
    !["text/html", "text/plain"].includes(contentType) &&
    contentType in ACCEPTED_MIMES
  )
    processVia = "file";