add docHandle

package-lock.json

@@ -17,6 +17,7 @@
                 "@xenova/transformers": "^2.1.1",
                 "eslint": "8.40.0",
                 "eslint-config-next": "13.4.2",
+                "fs-extra": "^11.1.1",
                 "langchain": "^0.0.90",
                 "next": "13.4.2",
                 "react": "18.2.0",
@@ -25,6 +26,7 @@
                 "uuid": "^9.0.0"
             },
             "devDependencies": {
+                "@types/fs-extra": "^11.0.1",
                 "@types/uuid": "^9.0.1"
             }
         },
@@ -1354,11 +1356,30 @@
             "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.1.tgz",
             "integrity": "sha512-LG4opVs2ANWZ1TJoKc937iMmNstM/d0ae1vNbnBvBhqCSezgVUOzcLCqbI5elV8Vy6WKwKjaqR+zO9VKirBBCA=="
         },
+        "node_modules/@types/fs-extra": {
+            "version": "11.0.1",
+            "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.1.tgz",
+            "integrity": "sha512-MxObHvNl4A69ofaTRU8DFqvgzzv8s9yRtaPPm5gud9HDNvpB3GPQFvNuTWAI59B9huVGV5jXYJwbCsmBsOGYWA==",
+            "dev": true,
+            "dependencies": {
+                "@types/jsonfile": "*",
+                "@types/node": "*"
+            }
+        },
         "node_modules/@types/json5": {
             "version": "0.0.29",
             "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
             "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ=="
         },
+        "node_modules/@types/jsonfile": {
+            "version": "6.1.1",
+            "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.1.tgz",
+            "integrity": "sha512-GSgiRCVeapDN+3pqA35IkQwasaCh/0YFH5dEF6S88iDvEn901DjOeH3/QPY+XYP1DFzDZPvIvfeEgk+7br5png==",
+            "dev": true,
+            "dependencies": {
+                "@types/node": "*"
+            }
+        },
         "node_modules/@types/long": {
             "version": "4.0.2",
             "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz",
@@ -3144,16 +3165,16 @@
             "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
         },
         "node_modules/fs-extra": {
-            "version": "10.1.0",
-            "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
-            "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
+            "version": "11.1.1",
+            "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz",
+            "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==",
             "dependencies": {
                 "graceful-fs": "^4.2.0",
                 "jsonfile": "^6.0.1",
                 "universalify": "^2.0.0"
             },
             "engines": {
-                "node": ">=12"
+                "node": ">=14.14"
             }
         },
         "node_modules/fs.realpath": {
@@ -5422,6 +5443,19 @@
                 "node": ">= 8.0.0"
             }
         },
+        "node_modules/rollup-plugin-typescript2/node_modules/fs-extra": {
+            "version": "10.1.0",
+            "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz",
+            "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==",
+            "dependencies": {
+                "graceful-fs": "^4.2.0",
+                "jsonfile": "^6.0.1",
+                "universalify": "^2.0.0"
+            },
+            "engines": {
+                "node": ">=12"
+            }
+        },
         "node_modules/run-applescript": {
             "version": "5.0.0",
             "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-5.0.0.tgz",

package.json

@@ -17,6 +17,7 @@
         "@xenova/transformers": "^2.1.1",
         "eslint": "8.40.0",
         "eslint-config-next": "13.4.2",
+        "fs-extra": "^11.1.1",
         "langchain": "^0.0.90",
         "next": "13.4.2",
         "react": "18.2.0",
@@ -25,6 +26,7 @@
         "uuid": "^9.0.0"
     },
     "devDependencies": {
+        "@types/fs-extra": "^11.0.1",
         "@types/uuid": "^9.0.1"
     }
 }

src/pages/api/docHandle.ts

@@ -0,0 +1,57 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+
+import {
+  readHNSWLibModelFromLocal,
+  storesDir,
+  vectorStoreToHNSWLibModel,
+} from '@/utils/file-handler';
+import fs from 'fs-extra';
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { HNSWLib } from 'langchain/vectorstores/hnswlib';
+import { XenovaTransformersEmbeddings } from '../../embed/hf'
+
+async function handleDocs(text: string) {
+  const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
+  const docs = await textSplitter.createDocuments([text]);
+  console.log(docs);
+
+  const vectorStore = await HNSWLib.fromDocuments(docs, new XenovaTransformersEmbeddings());
+  console.log(vectorStore);
+
+  return vectorStore;
+}
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse,
+) {
+  const { text } = JSON.parse(req.body);
+  // console.log(text);
+
+  if (!text) {
+    return res.status(400).json({ message: 'No question in the request' });
+  }
+
+  const exists = await fs.exists(storesDir);
+  console.log(exists);
+
+  if (exists) {
+    console.log('read from ' + storesDir);
+    const model = await readHNSWLibModelFromLocal();
+    return res.status(200).send({
+      ...model,
+    });
+  }
+
+  const vectorStore = await handleDocs(text);
+  const model = await vectorStoreToHNSWLibModel(vectorStore);
+  res.status(200).send({
+    ...model,
+  });
+}
+
+export const config = {
+  api: {
+    bodyParser: true, // Disallow body parsing, consume as stream
+  },
+};

src/utils/file-handler.ts

@@ -0,0 +1,75 @@
+import fs from 'fs-extra';
+import type { OpenAIEmbeddings } from 'langchain/embeddings/openai';
+import {
+  HNSWLib,
+  type HNSWLib as StoreTypeHNSWLib,
+} from 'langchain/vectorstores/hnswlib';
+import path from 'path';
+
+const ifDev = process.env.NODE_ENV === 'development';
+// in prod mode, only allowed to write to /tmp/
+// https://vercel.com/guides/how-can-i-use-files-in-serverless-functions
+export const storesDir = ifDev ? 'tmp/hnswlib-stores' : '/tmp/hnswlib-stores';
+
+type HNSWLibModel = {
+  args: string;
+  docstore: string;
+  hnswlibIndex: string;
+};
+
+const HNSWLibModelFilesName = {
+  args: 'args.json',
+  docstore: 'docstore.json',
+  hnswlibIndex: 'hnswlib.index',
+};
+
+// looking forward to a better way to transfrom hnswlibStore <=> indexes
+export async function HNSWLibModelToVectorStore(
+  model: HNSWLibModel,
+  embeddings: OpenAIEmbeddings,
+) {
+  await saveHNSWLibModelToLocal(model);
+  // load from dir
+  const vectorStore = await HNSWLib.load(storesDir, embeddings);
+  return vectorStore;
+}
+
+export async function saveHNSWLibModelToLocal(model: HNSWLibModel) {
+  // save model to /tmp/
+  await Promise.all(
+    Object.keys(HNSWLibModelFilesName).map((key) => {
+      const fullPath = path.join(
+        storesDir,
+        (HNSWLibModelFilesName as Record<string, string>)[key],
+      );
+      console.log(fullPath);
+      const data = (model as Record<string, string>)[key];
+      console.log(data);
+
+      return fs.writeFile(fullPath, data);
+    }),
+  );
+}
+
+export async function vectorStoreToHNSWLibModel(
+  store: StoreTypeHNSWLib,
+): Promise<HNSWLibModel> {
+  await store.save(storesDir);
+  return await readHNSWLibModelFromLocal();
+}
+
+export async function readHNSWLibModelFromLocal(): Promise<HNSWLibModel> {
+  const [args, docstore, hnswlibIndex] = await Promise.all([
+    fs.readFile(path.join(storesDir, HNSWLibModelFilesName.args), 'utf-8'),
+    fs.readFile(path.join(storesDir, HNSWLibModelFilesName.docstore), 'utf-8'),
+    fs.readFile(
+      path.join(storesDir, HNSWLibModelFilesName.hnswlibIndex),
+      'hex',
+    ),
+  ]);
+  return {
+    args,
+    docstore,
+    hnswlibIndex,
+  };
+}

src/utils/index.ts

@@ -89,3 +89,4 @@ export function throttle<T extends (...args: any[]) => any>(
 
 export const DEFAULT_TEMPERATURE = 
   parseFloat(process.env.NEXT_PUBLIC_DEFAULT_TEMPERATURE || "1");
+