Repository: Mintplex-Labs/anything-llm
File: collector/processLink/convert/generic.js (JavaScript)

Code before:
        success: false,
        reason: fileContentResult.reason,
        documents: [],
      };
    }

    const fileFilePath = fileContentResult.data;
    const targetFilename = path.basename(fileFilePath);

    const processSingleFileResult = await processSingleFile(targetFilename);

    if (!processSingleFileResult.success) {
      return {
        success: false,
        reason: processSingleFileResult.reason,
        documents: [],
      };
    }
    return processSingleFileResult;
  }

  // If the content type is not supported, return an error
  return {
    success: false,
    reason: `Unsupported content type: ${contentType}`,
    documents: [],
  };
}

/**
 * Validate the headers object
 * - Keys & Values must be strings and not empty
 * - Assemble a new object with only the valid keys and values
 * @param {{[key: string]: string}} headers - The headers object to validate
 * @returns {{[key: string]: string}} - The validated headers object
 */
function validatedHeaders(headers = {}) {
  try {
    if (Object.keys(headers).length === 0) return {};
    let validHeaders = {};
    for (const key of Object.keys(headers)) {
      if (!key?.trim()) continue;
      if (typeof headers[key] !== "string" || !headers[key]?.trim()) continue;
      validHeaders[key] = headers[key].trim();
    }
    return validHeaders;
  } catch (error) {
    console.error("Error validating headers", error);
    return {};
  }
}

Review comment (suggestion):
Since the entire logic is now in a big `if` it would make more sense to check this as a guard clause at the top of the function so one big `if` is not wrapping the core of the function.

eg: 
```js
if(!(contentType in ACCEPTED_MIMES)) { return .... }
// rest of code
```


Code after:
  }

  // Otherwise, assume the content is a webpage and scrape the content from the webpage
  const content = await getPageContent({
    link,
    captureAs,
    headers: scraperHeaders,
  });

  if (!content.length) {
    console.error(`Resulting URL content was empty at ${link}.`);
    return returnResult({
      success: false,
      reason: `No URL content found at ${link}.`,
      documents: [],
      content: null,
      saveAsDocument,
    });
  }

  // If the captureAs is text, return the content as a string immediately
  // so that we dont save the content as a document
  if (!saveAsDocument) {
    return returnResult({
      success: true,
      content,
      saveAsDocument,
    });
  }

  // Save the content as a document from the URL
  const url = new URL(link);
  const decodedPathname = decodeURIComponent(url.pathname);
  const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`;
  const data = {
    id: v4(),
    url: "file://" + slugify(filename) + ".html",
    title: metadata.title || slugify(filename) + ".html",
    docAuthor: metadata.docAuthor || "no author found",
    description: metadata.description || "No description found.",
    docSource: metadata.docSource || "URL link uploaded by the user.",
    chunkSource: `link://${link}`,
    published: new Date().toLocaleString(),
    wordCount: content.split(" ").length,
    pageContent: content,
    token_count_estimate: tokenizeString(content),
  };

  const document = writeToServerDocuments({
    data,
    filename: `url-${slugify(filename)}-${data.id}`,