Repository: Mintplex-Labs/anything-llm
File: collector/processLink/convert/generic.js (JavaScript)

Code before:
const { v4 } = require("uuid");
const {
  PuppeteerWebBaseLoader,
} = require("langchain/document_loaders/web/puppeteer");
const { writeToServerDocuments } = require("../../utils/files");
const { tokenizeString } = require("../../utils/tokenizer");
const { default: slugify } = require("slugify");
const { getContentType } = require("../helpers/getContentType");
const { downloadFileToHotDir } = require("../helpers/downloadFileToHotDir");
const { processSingleFile } = require("../../processSingleFile");
const path = require("path");
const { ACCEPTED_MIMES } = require("../../utils/constants");

/**
 * Scrape a generic URL and return the content in the specified format
 * @param {Object} config - The configuration object
 * @param {string} config.link - The URL to scrape
 * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
 * @param {boolean} config.processAsDocument - Whether to process the content as a document or return the content directly. Default is true
 * @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
 * @param {{[key: string]: string}} config.metadata - Metadata to use when creating the document
 * @returns {Promise<Object>} - The content of the page
 */
async function scrapeGenericUrl({
  link,
  processAsDocument = true,
  scraperHeaders = {},
  metadata = {},
}) {
  // Get the content type of the link
  const contentTypeResult = await getContentType(link);
  // If the retrieving the content type failed, return an error
  if (!contentTypeResult.success || !contentTypeResult.contentType) {
    return {
      success: false,
      reason: contentTypeResult.reason,
      documents: [],
    };
  }
  const contentType = contentTypeResult.contentType;

  if (contentType in ACCEPTED_MIMES) {
    // If the content type is HTML or text, get the content of the web page with puppeteer
    if (contentType === "text/html" || contentType === "text/plain") {
      const content = await getPageContent({
        link,
        captureAs: "text",

Review comment (suggestion):
The removal of `captureAs` breaks the collector here in the agent web-scrape and agent-flow webscraper. Since this function will also now _always_ generate a document.

 However, it seems that this is supposed to be replaced by the `processAsDocument` key.
 
 These two flags do two separate functions and can work together, though. `captureAs` basically determines what type of [content will we scrape from a site](https://github.com/Mintplex-Labs/anything-llm/blob/3c61e400d0f579b543246903baef532d0e0d5d57/collector/processLink/convert/generic.js#L170-L171)
 
 However,  because `captureAs` was removed as an option.arg we can no longer use it and `getPageContent` will always only get text.
 
 [`getLinkText`](https://github.com/Mintplex-Labs/anything-llm/blob/3c61e400d0f579b543246903baef532d0e0d5d57/collector/processLink/index.js#L33) still uses this arg so it would just be dropped
 
 



Code after:
const { v4 } = require("uuid");
const path = require("path");
const {
  PuppeteerWebBaseLoader,
} = require("langchain/document_loaders/web/puppeteer");
const { writeToServerDocuments } = require("../../utils/files");
const { tokenizeString } = require("../../utils/tokenizer");
const { default: slugify } = require("slugify");
const { getContentTypeFromURL, returnResult } = require("../helpers");
const { processSingleFile } = require("../../processSingleFile");
const { downloadURIToFile } = require("../../utils/downloadURIToFile");
const { ACCEPTED_MIMES } = require("../../utils/constants");
const RuntimeSettings = require("../../utils/runtimeSettings");

/**
 * Scrape a generic URL and return the content in the specified format
 * @param {Object} config - The configuration object
 * @param {string} config.link - The URL to scrape
 * @param {('html' | 'text')} config.captureAs - The format to capture the page content as. Default is 'text'
 * @param {{[key: string]: string}} config.scraperHeaders - Custom headers to use when making the request
 * @param {{[key: string]: string}} config.metadata - Metadata to use when creating the document
 * @param {boolean} config.saveAsDocument - Whether to save the content as a document. Default is true
 * @returns {Promise<Object>} - The content of the page
 */
async function scrapeGenericUrl({
  link,
  captureAs = "text",
  scraperHeaders = {},
  metadata = {},
  saveAsDocument = true,
}) {
  /** @type {'web' | 'file'} */
  let processVia = "web";
  console.log(`-- Working URL ${link} => (captureAs: ${captureAs}) --`);

  const contentType = await getContentTypeFromURL(link)
    .then((result) => {
      // If there is a reason, log it, but continue with the process
      if (!!result.reason) console.error(result.reason);
      return result.contentType;
    })
    .catch((error) => {
      console.error("Error getting content type from URL", error);
      return null;
    });

  // If the content is unlikely to be a webpage, assume it is a file and process it as a file