CVE-2024-3149: Server-Side Request Forgery (SSRF) (CWE-918)
value: A Server-Side Request Forgery (SSRF) vulnerability exists in the upload link feature of mintplex-labs/anything-llm. This feature, intended for users with manager or admin roles, processes uploaded links through an internal Collector API using a headless browser. An attacker can exploit this by hosting a malicious website and using it to perform actions such as internal port scanning, accessing internal web applications not exposed externally, and interacting with the Collector API. This interaction can lead to unauthorized actions such as arbitrary file deletion and limited Local File Inclusion (LFI), including accessing NGINX access logs which may contain sensitive information.
CWE description: The web server receives a URL or similar request from an upstream component and retrieves the contents of this URL, but it does not sufficiently ensure that the request is being sent to the expected destination.
Published: 2024-06-06T19:16Z
Repository: https://github.com/mintplex-labs/anything-llm
Language: JavaScript
Files changed: .vscode/settings.json, collector/extensions/index.js, collector/index.js, collector/middleware/verifyIntegrity.js, collector/processSingleFile/index.js, collector/utils/comKey/index.js, collector/utils/files/index.js, server/.gitignore, server/utils/boot/index.js, server/utils/collectorApi/index.js, server/utils/comKey/index.js
Commit date: 2024-04-01 13:56:35 -0700
Commit message: RSA-Signing on server<->collector communication via API (#1005)

* WIP integrity check between processes

* Implement integrity checking on document processor payloads
Diff: diff --git a/.vscode/settings.json b/.vscode/settings.json
index 72b612b86..a87e960bf 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,9 +1,10 @@
 {
   "cSpell.words": [
     "anythingllm",
     "Astra",
+    "comkey",
     "Dockerized",
     "Embeddable",
     "GROQ",
     "hljs",
     "inferencing",
@@ -18,6 +19,6 @@
     "Weaviate",
     "Zilliz"
   ],
   "eslint.experimental.useFlatConfig": true,
   "docker.languageserver.formatter.ignoreMultilineInstructions": true
-}
+}
\ No newline at end of file
diff --git a/collector/extensions/index.js b/collector/extensions/index.js
index bcf2229f2..0e91d1731 100644
--- a/collector/extensions/index.js
+++ b/collector/extensions/index.js
@@ -1,11 +1,12 @@
+const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity");
 const { reqBody } = require("../utils/http");
 
 function extensions(app) {
   if (!app) return;
 
-  app.post("/ext/github-repo", async function (request, response) {
+  app.post("/ext/github-repo", [verifyPayloadIntegrity], async function (request, response) {
     try {
       const loadGithubRepo = require("../utils/extensions/GithubRepo");
       const { success, reason, data } = await loadGithubRepo(reqBody(request));
       response.status(200).json({
         success,
@@ -22,11 +23,11 @@ function extensions(app) {
     }
     return;
   });
 
   // gets all branches for a specific repo
-  app.post("/ext/github-repo/branches", async function (request, response) {
+  app.post("/ext/github-repo/branches", [verifyPayloadIntegrity], async function (request, response) {
     try {
       const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader");
       const allBranches = await (new GithubRepoLoader(reqBody(request))).getRepoBranches()
       response.status(200).json({
         success: true,
@@ -46,11 +47,11 @@ function extensions(app) {
       });
     }
     return;
   });
 
-  app.post("/ext/youtube-transcript", async function (request, response) {
+  app.post("/ext/youtube-transcript", [verifyPayloadIntegrity], async function (request, response) {
     try {
       const loadYouTubeTranscript = require("../utils/extensions/YoutubeTranscript");
       const { success, reason, data } = await loadYouTubeTranscript(reqBody(request));
       response.status(200).json({ success, reason, data });
     } catch (e) {
diff --git a/collector/index.js b/collector/index.js
index a1142d756..f574730c6 100644
--- a/collector/index.js
+++ b/collector/index.js
@@ -11,10 +11,11 @@ const { reqBody } = require("./utils/http");
 const { processSingleFile } = require("./processSingleFile");
 const { processLink } = require("./processLink");
 const { wipeCollectorStorage } = require("./utils/files");
 const extensions = require("./extensions");
 const { processRawText } = require("./processRawText");
+const { verifyPayloadIntegrity } = require("./middleware/verifyIntegrity");
 const app = express();
 
 app.use(cors({ origin: true }));
 app.use(
   bodyParser.text(),
@@ -22,75 +23,87 @@ app.use(
   bodyParser.urlencoded({
     extended: true,
   })
 );
 
-app.post("/process", async function (request, response) {
-  const { filename, options = {} } = reqBody(request);
-  try {
-    const targetFilename = path
-      .normalize(filename)
-      .replace(/^(\.\.(\/|\\|$))+/, "");
-    const {
-      success,
-      reason,
-      documents = [],
-    } = await processSingleFile(targetFilename, options);
-    response
-      .status(200)
-      .json({ filename: targetFilename, success, reason, documents });
-  } catch (e) {
-    console.error(e);
-    response.status(200).json({
-      filename: filename,
-      success: false,
-      reason: "A processing error occurred.",
-      documents: [],
-    });
+app.post(
+  "/process",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { filename, options = {} } = reqBody(request);
+    try {
+      const targetFilename = path
+        .normalize(filename)
+        .replace(/^(\.\.(\/|\\|$))+/, "");
+      const {
+        success,
+        reason,
+        documents = [],
+      } = await processSingleFile(targetFilename, options);
+      response
+        .status(200)
+        .json({ filename: targetFilename, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        filename: filename,
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
   }
-  return;
-});
+);
 
-app.post("/process-link", async function (request, response) {
-  const { link } = reqBody(request);
-  try {
-    const { success, reason, documents = [] } = await processLink(link);
-    response.status(200).json({ url: link, success, reason, documents });
-  } catch (e) {
-    console.error(e);
-    response.status(200).json({
-      url: link,
-      success: false,
-      reason: "A processing error occurred.",
-      documents: [],
-    });
+app.post(
+  "/process-link",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { link } = reqBody(request);
+    try {
+      const { success, reason, documents = [] } = await processLink(link);
+      response.status(200).json({ url: link, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        url: link,
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
   }
-  return;
-});
+);
 
-app.post("/process-raw-text", async function (request, response) {
-  const { textContent, metadata } = reqBody(request);
-  try {
-    const {
-      success,
-      reason,
-      documents = [],
-    } = await processRawText(textContent, metadata);
-    response
-      .status(200)
-      .json({ filename: metadata.title, success, reason, documents });
-  } catch (e) {
-    console.error(e);
-    response.status(200).json({
-      filename: metadata?.title || "Unknown-doc.txt",
-      success: false,
-      reason: "A processing error occurred.",
-      documents: [],
-    });
+app.post(
+  "/process-raw-text",
+  [verifyPayloadIntegrity],
+  async function (request, response) {
+    const { textContent, metadata } = reqBody(request);
+    try {
+      const {
+        success,
+        reason,
+        documents = [],
+      } = await processRawText(textContent, metadata);
+      response
+        .status(200)
+        .json({ filename: metadata.title, success, reason, documents });
+    } catch (e) {
+      console.error(e);
+      response.status(200).json({
+        filename: metadata?.title || "Unknown-doc.txt",
+        success: false,
+        reason: "A processing error occurred.",
+        documents: [],
+      });
+    }
+    return;
   }
-  return;
-});
+);
 
 extensions(app);
 
 app.get("/accepts", function (_, response) {
   response.status(200).json(ACCEPTED_MIMES);
diff --git a/collector/middleware/verifyIntegrity.js b/collector/middleware/verifyIntegrity.js
new file mode 100644
index 000000000..0dcb3f75d
--- /dev/null
+++ b/collector/middleware/verifyIntegrity.js
@@ -0,0 +1,21 @@
+const { CommunicationKey } = require("../utils/comKey");
+
+function verifyPayloadIntegrity(request, response, next) {
+  const comKey = new CommunicationKey();
+  if (process.env.NODE_ENV === "development") {
+    comKey.log('verifyPayloadIntegrity is skipped in development.')
+    next();
+    return;
+  }
+
+  const signature = request.header("X-Integrity");
+  if (!signature) return response.status(400).json({ msg: 'Failed integrity signature check.' })
+
+  const validSignedPayload = comKey.verify(signature, request.body);
+  if (!validSignedPayload) return response.status(400).json({ msg: 'Failed integrity signature check.' })
+  next();
+}
+
+module.exports = {
+  verifyPayloadIntegrity
+}
\ No newline at end of file
diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js
index 5d9e6a38a..bdefb79e0 100644
--- a/collector/processSingleFile/index.js
+++ b/collector/processSingleFile/index.js
@@ -2,15 +2,30 @@ const path = require("path");
 const fs = require("fs");
 const {
   WATCH_DIRECTORY,
   SUPPORTED_FILETYPE_CONVERTERS,
 } = require("../utils/constants");
-const { trashFile, isTextType } = require("../utils/files");
+const {
+  trashFile,
+  isTextType,
+  normalizePath,
+  isWithin,
+} = require("../utils/files");
 const RESERVED_FILES = ["__HOTDIR__.md"];
 
 async function processSingleFile(targetFilename, options = {}) {
-  const fullFilePath = path.resolve(WATCH_DIRECTORY, targetFilename);
+  const fullFilePath = path.resolve(
+    WATCH_DIRECTORY,
+    normalizePath(targetFilename)
+  );
+  if (!isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath))
+    return {
+      success: false,
+      reason: "Filename is a not a valid path to process.",
+      documents: [],
+    };
+
   if (RESERVED_FILES.includes(targetFilename))
     return {
       success: false,
       reason: "Filename is a reserved filename and cannot be processed.",
       documents: [],
diff --git a/collector/utils/comKey/index.js b/collector/utils/comKey/index.js
new file mode 100644
index 000000000..0e96a6972
--- /dev/null
+++ b/collector/utils/comKey/index.js
@@ -0,0 +1,42 @@
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+
+const keyPath =
+  process.env.NODE_ENV === "development"
+    ? path.resolve(__dirname, `../../../server/storage/comkey`)
+    : path.resolve(process.env.STORAGE_DIR, `comkey`);
+
+class CommunicationKey {
+  #pubKeyName = "ipc-pub.pem";
+  #storageLoc = keyPath;
+
+  constructor() {}
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[CommunicationKeyVerify]\x1b[0m ${text}`, ...args);
+  }
+
+  #readPublicKey() {
+    return fs.readFileSync(path.resolve(this.#storageLoc, this.#pubKeyName));
+  }
+
+  // Given a signed payload from private key from /app/server/ this signature should
+  // decode to match the textData provided. This class does verification only in collector.
+  // Note: The textData is typically the JSON stringified body sent to the document processor API.
+  verify(signature = "", textData = "") {
+    try {
+      let data = textData;
+      if (typeof textData !== "string") data = JSON.stringify(data);
+      return crypto.verify(
+        "RSA-SHA256",
+        Buffer.from(data),
+        this.#readPublicKey(),
+        Buffer.from(signature, "hex")
+      );
+    } catch {}
+    return false;
+  }
+}
+
+module.exports = { CommunicationKey };
diff --git a/collector/utils/files/index.js b/collector/utils/files/index.js
index 4bca62f9f..9b56bb5b4 100644
--- a/collector/utils/files/index.js
+++ b/collector/utils/files/index.js
@@ -106,12 +106,35 @@ async function wipeCollectorStorage() {
   await Promise.all([cleanHotDir, cleanTmpDir]);
   console.log(`Collector hot directory and tmp storage wiped!`);
   return;
 }
 
+/**
+ * Checks if a given path is within another path.
+ * @param {string} outer - The outer path (should be resolved).
+ * @param {string} inner - The inner path (should be resolved).
+ * @returns {boolean} - Returns true if the inner path is within the outer path, false otherwise.
+ */
+function isWithin(outer, inner) {
+  if (outer === inner) return false;
+  const rel = path.relative(outer, inner);
+  return !rel.startsWith("../") && rel !== "..";
+}
+
+function normalizePath(filepath = "") {
+  const result = path
+    .normalize(filepath.trim())
+    .replace(/^(\.\.(\/|\\|$))+/, "")
+    .trim();
+  if (["..", ".", "/"].includes(result)) throw new Error("Invalid path.");
+  return result;
+}
+
 module.exports = {
   trashFile,
   isTextType,
   createdDate,
   writeToServerDocuments,
   wipeCollectorStorage,
+  normalizePath,
+  isWithin,
 };
diff --git a/server/.gitignore b/server/.gitignore
index f9d92ce17..b22a054fc 100644
--- a/server/.gitignore
+++ b/server/.gitignore
@@ -1,10 +1,11 @@
 .env.production
 .env.development
 storage/assets/*
 !storage/assets/anything-llm.png
 storage/documents/*
+storage/comkey/*
 storage/tmp/*
 storage/vector-cache/*.json
 storage/exports
 storage/imports
 !storage/documents/DOCUMENTS.md
diff --git a/server/utils/boot/index.js b/server/utils/boot/index.js
index 7134198e7..ea95e1f52 100644
--- a/server/utils/boot/index.js
+++ b/server/utils/boot/index.js
@@ -1,6 +1,7 @@
 const { Telemetry } = require("../../models/telemetry");
+const { CommunicationKey } = require("../comKey");
 const setupTelemetry = require("../telemetry");
 
 function bootSSL(app, port = 3001) {
   try {
     console.log(
@@ -14,10 +15,11 @@ function bootSSL(app, port = 3001) {
 
     https
       .createServer(credentials, app)
       .listen(port, async () => {
         await setupTelemetry();
+        new CommunicationKey(true);
         console.log(`Primary server in HTTPS mode listening on port ${port}`);
       })
       .on("error", catchSigTerms);
     return app;
   } catch (e) {
@@ -38,10 +40,11 @@ function bootHTTP(app, port = 3001) {
   if (!app) throw new Error('No "app" defined - crashing!');
 
   app
     .listen(port, async () => {
       await setupTelemetry();
+      new CommunicationKey(true);
       console.log(`Primary server in HTTP mode listening on port ${port}`);
     })
     .on("error", catchSigTerms);
   return app;
 }
diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js
index ed27a9289..d96cd1e6d 100644
--- a/server/utils/collectorApi/index.js
+++ b/server/utils/collectorApi/index.js
@@ -3,10 +3,12 @@
 // so no additional security is needed on the endpoint directly. Auth is done however by the express
 // middleware prior to leaving the node-side of the application so that is good enough >:)
 
 class CollectorApi {
   constructor() {
+    const { CommunicationKey } = require("../comKey");
+    this.comkey = new CommunicationKey();
     this.endpoint = `http://0.0.0.0:${process.env.COLLECTOR_PORT || 8888}`;
   }
 
   log(text, ...args) {
     console.log(`\x1b[36m[CollectorApi]\x1b[0m ${text}`, ...args);
@@ -38,19 +40,23 @@ class CollectorApi {
       });
   }
 
   async processDocument(filename = "") {
     if (!filename) return false;
+
+    const data = JSON.stringify({
+      filename,
+      options: this.#attachOptions(),
+    });
+
     return await fetch(`${this.endpoint}/process`, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
+        "X-Integrity": this.comkey.sign(data),
       },
-      body: JSON.stringify({
-        filename,
-        options: this.#attachOptions(),
-      }),
+      body: data,
     })
       .then((res) => {
         if (!res.ok) throw new Error("Response could not be completed");
         return res.json();
       })
@@ -62,16 +68,18 @@ class CollectorApi {
   }
 
   async processLink(link = "") {
     if (!link) return false;
 
+    const data = JSON.stringify({ link });
     return await fetch(`${this.endpoint}/process-link`, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
+        "X-Integrity": this.comkey.sign(data),
       },
-      body: JSON.stringify({ link }),
+      body: data,
     })
       .then((res) => {
         if (!res.ok) throw new Error("Response could not be completed");
         return res.json();
       })
@@ -81,16 +89,18 @@ class CollectorApi {
         return { success: false, reason: e.message, documents: [] };
       });
   }
 
   async processRawText(textContent = "", metadata = {}) {
+    const data = JSON.stringify({ textContent, metadata });
     return await fetch(`${this.endpoint}/process-raw-text`, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
+        "X-Integrity": this.comkey.sign(data),
       },
-      body: JSON.stringify({ textContent, metadata }),
+      body: data,
     })
       .then((res) => {
         if (!res.ok) throw new Error("Response could not be completed");
         return res.json();
       })
@@ -108,10 +118,11 @@ class CollectorApi {
     return await fetch(`${this.endpoint}${endpoint}`, {
       method,
       body, // Stringified JSON!
       headers: {
         "Content-Type": "application/json",
+        "X-Integrity": this.comkey.sign(body),
       },
     })
       .then((res) => {
         if (!res.ok) throw new Error("Response could not be completed");
         return res.json();
diff --git a/server/utils/comKey/index.js b/server/utils/comKey/index.js
new file mode 100644
index 000000000..e545a6be5
--- /dev/null
+++ b/server/utils/comKey/index.js
@@ -0,0 +1,75 @@
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
+const keyPath =
+  process.env.NODE_ENV === "development"
+    ? path.resolve(__dirname, `../../storage/comkey`)
+    : path.resolve(process.env.STORAGE_DIR, `comkey`);
+
+// What does this class do?
+// This class generates a hashed version of some text (typically a JSON payload) using a rolling RSA key
+// that can then be appended as a header value to do integrity checking on a payload. Given the
+// nature of this class and that keys are rolled constantly, this protects the request
+// integrity of requests sent to the collector as only the server can sign these requests.
+// This keeps accidental misconfigurations of AnythingLLM that leaving port 8888 open from
+// being abused or SSRF'd by users scraping malicious sites who have a loopback embedded in a <script>, for example.
+// Since each request to the collector must be signed to be valid, unsigned requests directly to the collector
+// will be dropped and must go through the /server endpoint directly.
+class CommunicationKey {
+  #privKeyName = "ipc-priv.pem";
+  #pubKeyName = "ipc-pub.pem";
+  #storageLoc = keyPath;
+
+  // Init the class and determine if keys should be rolled.
+  // This typically occurs on boot up so key is fresh each boot.
+  constructor(generate = false) {
+    if (generate) this.#generate();
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[CommunicationKey]\x1b[0m ${text}`, ...args);
+  }
+
+  #readPrivateKey() {
+    return fs.readFileSync(path.resolve(this.#storageLoc, this.#privKeyName));
+  }
+
+  #generate() {
+    const keyPair = crypto.generateKeyPairSync("rsa", {
+      modulusLength: 2048,
+      publicKeyEncoding: {
+        type: "pkcs1",
+        format: "pem",
+      },
+      privateKeyEncoding: {
+        type: "pkcs1",
+        format: "pem",
+      },
+    });
+
+    if (!fs.existsSync(this.#storageLoc))
+      fs.mkdirSync(this.#storageLoc, { recursive: true });
+    fs.writeFileSync(
+      `${path.resolve(this.#storageLoc, this.#privKeyName)}`,
+      keyPair.privateKey
+    );
+    fs.writeFileSync(
+      `${path.resolve(this.#storageLoc, this.#pubKeyName)}`,
+      keyPair.publicKey
+    );
+    this.log(
+      "RSA key pair generated for signed payloads within AnythingLLM services."
+    );
+  }
+
+  // This instance of ComKey on server is intended for generation of Priv/Pub key for signing and decoding.
+  // this resource is shared with /collector/ via a class of the same name in /utils which does decoding/verification only
+  // while this server class only does signing with the private key.
+  sign(textData = "") {
+    return crypto
+      .sign("RSA-SHA256", Buffer.from(textData), this.#readPrivateKey())
+      .toString("hex");
+  }
+}
+
+module.exports = { CommunicationKey };

Vulnerable code: }
  app.post("/ext/github-repo", async function (request, response) {
  app.post("/ext/github-repo/branches", async function (request, response) {
  app.post("/ext/youtube-transcript", async function (request, response) {
app.post("/process", async function (request, response) {
  const { filename, options = {} } = reqBody(request);
  try {
    const targetFilename = path
      .normalize(filename)
      .replace(/^(\.\.(\/|\\|$))+/, "");
    const {
      success,
      reason,
      documents = [],
    } = await processSingleFile(targetFilename, options);
    response
      .status(200)
      .json({ filename: targetFilename, success, reason, documents });
  } catch (e) {
    console.error(e);
    response.status(200).json({
      filename: filename,
      success: false,
      reason: "A processing error occurred.",
      documents: [],
    });
  return;
});
app.post("/process-link", async function (request, response) {
  const { link } = reqBody(request);
  try {
    const { success, reason, documents = [] } = await processLink(link);
    response.status(200).json({ url: link, success, reason, documents });
  } catch (e) {
    console.error(e);
    response.status(200).json({
      url: link,
      success: false,
      reason: "A processing error occurred.",
      documents: [],
    });
  return;
});
app.post("/process-raw-text", async function (request, response) {
  const { textContent, metadata } = reqBody(request);
  try {
    const {
      success,
      reason,
      documents = [],
    } = await processRawText(textContent, metadata);
    response
      .status(200)
      .json({ filename: metadata.title, success, reason, documents });
  } catch (e) {
    console.error(e);
    response.status(200).json({
      filename: metadata?.title || "Unknown-doc.txt",
      success: false,
      reason: "A processing error occurred.",
      documents: [],
    });
  return;
});
const { trashFile, isTextType } = require("../utils/files");
  const fullFilePath = path.resolve(WATCH_DIRECTORY, targetFilename);
      body: JSON.stringify({
        filename,
        options: this.#attachOptions(),
      }),
      body: JSON.stringify({ link }),
      body: JSON.stringify({ textContent, metadata }),
Fixed code:     "comkey",
}
const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity");
  app.post("/ext/github-repo", [verifyPayloadIntegrity], async function (request, response) {
  app.post("/ext/github-repo/branches", [verifyPayloadIntegrity], async function (request, response) {
  app.post("/ext/youtube-transcript", [verifyPayloadIntegrity], async function (request, response) {
const { verifyPayloadIntegrity } = require("./middleware/verifyIntegrity");
app.post(
  "/process",
  [verifyPayloadIntegrity],
  async function (request, response) {
    const { filename, options = {} } = reqBody(request);
    try {
      const targetFilename = path
        .normalize(filename)
        .replace(/^(\.\.(\/|\\|$))+/, "");
      const {
        success,
        reason,
        documents = [],
      } = await processSingleFile(targetFilename, options);
      response
        .status(200)
        .json({ filename: targetFilename, success, reason, documents });
    } catch (e) {
      console.error(e);
      response.status(200).json({
        filename: filename,
        success: false,
        reason: "A processing error occurred.",
        documents: [],
      });
    }
    return;
);
app.post(
  "/process-link",
  [verifyPayloadIntegrity],
  async function (request, response) {
    const { link } = reqBody(request);
    try {
      const { success, reason, documents = [] } = await processLink(link);
      response.status(200).json({ url: link, success, reason, documents });
    } catch (e) {
      console.error(e);
      response.status(200).json({
        url: link,
        success: false,
        reason: "A processing error occurred.",
        documents: [],
      });
    }
    return;
);
app.post(
  "/process-raw-text",
  [verifyPayloadIntegrity],
  async function (request, response) {
    const { textContent, metadata } = reqBody(request);
    try {
      const {
        success,
        reason,
        documents = [],
      } = await processRawText(textContent, metadata);
      response
        .status(200)
        .json({ filename: metadata.title, success, reason, documents });
    } catch (e) {
      console.error(e);
      response.status(200).json({
        filename: metadata?.title || "Unknown-doc.txt",
        success: false,
        reason: "A processing error occurred.",
        documents: [],
      });
    }
    return;
);
const { CommunicationKey } = require("../utils/comKey");

function verifyPayloadIntegrity(request, response, next) {
  const comKey = new CommunicationKey();
  if (process.env.NODE_ENV === "development") {
    comKey.log('verifyPayloadIntegrity is skipped in development.')
    next();
    return;
  }

  const signature = request.header("X-Integrity");
  if (!signature) return response.status(400).json({ msg: 'Failed integrity signature check.' })

  const validSignedPayload = comKey.verify(signature, request.body);
  if (!validSignedPayload) return response.status(400).json({ msg: 'Failed integrity signature check.' })
  next();
}

module.exports = {
  verifyPayloadIntegrity
}
const {
  trashFile,
  isTextType,
  normalizePath,
  isWithin,
} = require("../utils/files");
  const fullFilePath = path.resolve(
    WATCH_DIRECTORY,
    normalizePath(targetFilename)
  );
  if (!isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath))
    return {
      success: false,
      reason: "Filename is a not a valid path to process.",
      documents: [],
    };

const crypto = require("crypto");
const fs = require("fs");
const path = require("path");

const keyPath =
  process.env.NODE_ENV === "development"
    ? path.resolve(__dirname, `../../../server/storage/comkey`)
    : path.resolve(process.env.STORAGE_DIR, `comkey`);

class CommunicationKey {
  #pubKeyName = "ipc-pub.pem";
  #storageLoc = keyPath;

  constructor() {}

  log(text, ...args) {
    console.log(`\x1b[36m[CommunicationKeyVerify]\x1b[0m ${text}`, ...args);
  }

  #readPublicKey() {
    return fs.readFileSync(path.resolve(this.#storageLoc, this.#pubKeyName));
  }

  // Given a signed payload from private key from /app/server/ this signature should
  // decode to match the textData provided. This class does verification only in collector.
  // Note: The textData is typically the JSON stringified body sent to the document processor API.
  verify(signature = "", textData = "") {
    try {
      let data = textData;
      if (typeof textData !== "string") data = JSON.stringify(data);
      return crypto.verify(
        "RSA-SHA256",
        Buffer.from(data),
        this.#readPublicKey(),
        Buffer.from(signature, "hex")
      );
    } catch {}
    return false;
  }
}

module.exports = { CommunicationKey };
/**
 * Checks if a given path is within another path.
 * @param {string} outer - The outer path (should be resolved).
 * @param {string} inner - The inner path (should be resolved).
 * @returns {boolean} - Returns true if the inner path is within the outer path, false otherwise.
 */
function isWithin(outer, inner) {
  if (outer === inner) return false;
  const rel = path.relative(outer, inner);
  return !rel.startsWith("../") && rel !== "..";
}

function normalizePath(filepath = "") {
  const result = path
    .normalize(filepath.trim())
    .replace(/^(\.\.(\/|\\|$))+/, "")
    .trim();
  if (["..", ".", "/"].includes(result)) throw new Error("Invalid path.");
  return result;
}

  normalizePath,
  isWithin,
storage/comkey/*
const { CommunicationKey } = require("../comKey");
        new CommunicationKey(true);
      new CommunicationKey(true);
    const { CommunicationKey } = require("../comKey");
    this.comkey = new CommunicationKey();

    const data = JSON.stringify({
      filename,
      options: this.#attachOptions(),
    });

        "X-Integrity": this.comkey.sign(data),
      body: data,
    const data = JSON.stringify({ link });
        "X-Integrity": this.comkey.sign(data),
      body: data,
    const data = JSON.stringify({ textContent, metadata });
        "X-Integrity": this.comkey.sign(data),
      body: data,
        "X-Integrity": this.comkey.sign(body),
const crypto = require("crypto");
const fs = require("fs");
const path = require("path");
const keyPath =
  process.env.NODE_ENV === "development"
    ? path.resolve(__dirname, `../../storage/comkey`)
    : path.resolve(process.env.STORAGE_DIR, `comkey`);

// What does this class do?
// This class generates a hashed version of some text (typically a JSON payload) using a rolling RSA key
// that can then be appended as a header value to do integrity checking on a payload. Given the
// nature of this class and that keys are rolled constantly, this protects the request
// integrity of requests sent to the collector as only the server can sign these requests.
// This keeps accidental misconfigurations of AnythingLLM that leaving port 8888 open from
// being abused or SSRF'd by users scraping malicious sites who have a loopback embedded in a <script>, for example.
// Since each request to the collector must be signed to be valid, unsigned requests directly to the collector
// will be dropped and must go through the /server endpoint directly.
class CommunicationKey {
  #privKeyName = "ipc-priv.pem";
  #pubKeyName = "ipc-pub.pem";
  #storageLoc = keyPath;

  // Init the class and determine if keys should be rolled.
  // This typically occurs on boot up so key is fresh each boot.
  constructor(generate = false) {
    if (generate) this.#generate();
  }

  log(text, ...args) {
    console.log(`\x1b[36m[CommunicationKey]\x1b[0m ${text}`, ...args);
  }

  #readPrivateKey() {
    return fs.readFileSync(path.resolve(this.#storageLoc, this.#privKeyName));
  }

  #generate() {
    const keyPair = crypto.generateKeyPairSync("rsa", {
      modulusLength: 2048,
      publicKeyEncoding: {
        type: "pkcs1",
        format: "pem",
      },
      privateKeyEncoding: {
        type: "pkcs1",
        format: "pem",
      },
    });

    if (!fs.existsSync(this.#storageLoc))
      fs.mkdirSync(this.#storageLoc, { recursive: true });
    fs.writeFileSync(
      `${path.resolve(this.#storageLoc, this.#privKeyName)}`,
      keyPair.privateKey
    );
    fs.writeFileSync(
      `${path.resolve(this.#storageLoc, this.#pubKeyName)}`,
      keyPair.publicKey
    );
    this.log(
      "RSA key pair generated for signed payloads within AnythingLLM services."
    );
  }

  // This instance of ComKey on server is intended for generation of Priv/Pub key for signing and decoding.
  // this resource is shared with /collector/ via a class of the same name in /utils which does decoding/verification only
  // while this server class only does signing with the private key.
  sign(textData = "") {
    return crypto
      .sign("RSA-SHA256", Buffer.from(textData), this.#readPrivateKey())
      .toString("hex");
  }
}

module.exports = { CommunicationKey };