cleanup

package-lock.json

@@ -16,11 +16,11 @@
                 "@types/react-dom": "18.2.4",
                 "@xenova/transformers": "^2.1.1",
                 "chromadb": "^1.5.2",
+                "cohere-ai": "^5.1.0",
                 "dexie": "^3.2.4",
                 "eslint": "8.40.0",
                 "eslint-config-next": "13.4.2",
                 "fs-extra": "^11.1.1",
-                "hnswlib-node": "^1.4.2",
                 "langchain": "^0.0.90",
                 "next": "13.4.2",
                 "pdfjs-dist": "^3.7.107",
@@ -1968,6 +1968,8 @@
             "version": "1.5.0",
             "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
             "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+            "optional": true,
+            "peer": true,
             "dependencies": {
                 "file-uri-to-path": "1.0.0"
             }
@@ -2220,6 +2222,11 @@
             "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
             "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
         },
+        "node_modules/cohere-ai": {
+            "version": "5.1.0",
+            "resolved": "https://registry.npmjs.org/cohere-ai/-/cohere-ai-5.1.0.tgz",
+            "integrity": "sha512-7q3z3w6GSoPxQqRL9G6QTaQ0e513auVE1JlNDnqnoFEXGtDbkVfaTOliR5qrMoK//74Csb0NW669evqngwPx3g=="
+        },
         "node_modules/color": {
             "version": "4.2.3",
             "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
@@ -3285,7 +3292,9 @@
         "node_modules/file-uri-to-path": {
             "version": "1.0.0",
             "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
-            "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
+            "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
+            "optional": true,
+            "peer": true
         },
         "node_modules/filelist": {
             "version": "1.0.4",
@@ -3802,6 +3811,8 @@
             "resolved": "https://registry.npmjs.org/hnswlib-node/-/hnswlib-node-1.4.2.tgz",
             "integrity": "sha512-76PIzOaNcX8kOpKwlFPl07uelpctqDMzbiC+Qsk2JWNVkzeU/6iXRk4tfE9z3DoK1RCBrOaFXmQ6RFb1BVF9LA==",
             "hasInstallScript": true,
+            "optional": true,
+            "peer": true,
             "dependencies": {
                 "bindings": "^1.5.0",
                 "node-addon-api": "^6.0.0"

package.json

@@ -16,6 +16,7 @@
         "@types/react-dom": "18.2.4",
         "@xenova/transformers": "^2.1.1",
         "chromadb": "^1.5.2",
+        "cohere-ai": "^5.1.0",
         "dexie": "^3.2.4",
         "eslint": "8.40.0",
         "eslint-config-next": "13.4.2",

src/components/ChatWindow.jsx

@@ -5,9 +5,11 @@ import MessageList from './MessageList';
 import {FileLoader} from './FileLoader';
 import Loader from "./Loader";
 import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { TransformersEmbeddingFunction } from '../embed/hf';
 import { ChromaClient } from "chromadb";
 
 const client = new ChromaClient();
+const embedder = new TransformersEmbeddingFunction({});
 
 function ChatWindow({
   stopStrings,
@@ -29,19 +31,28 @@ function ChatWindow({
     }
     
     if (fileText) {
+      console.log('found file text splitting into chunks')
       const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000 });
-      const docs = await textSplitter.createDocuments([text]);
-      const collection = await client.createCollection({name: "docs", embeddingFunction: TransformersEmbeddingFunction})
-      await collection.add({
+      const docs = await textSplitter.createDocuments([fileText]);
+      console.log(`split docs: ${docs}`);
+      const collection = await client.createCollection({name: "docs", embeddingFunction: embedder })
+      console.log(`collection: ${collection}`);
+      let queryResult;
+      try {
+        await collection.add({
           ids: [...docs.map((v, k) => k)],
           metadatas: [...docs.map(doc => doc.metadata)],
           documents: [...docs.map(doc => doc.pageContent)],
-      });
-      const result = await collection.query({
-        nResults: 2, 
-        queryTexts: [userPrompt]
-      });
-      console.log(result);
+        });
+        const queryResult = await collection.query({
+          nResults: 2, 
+          queryTexts: [userPrompt]
+        });
+        console.log(queryResult);
+      } catch (err) {
+        console.log(err);
+      }
+      
 
       const qaPrompt = 
       `You are an AI assistant providing helpful advice. You are given the following extracted parts of a long document and a question. Provide a conversational answer based on the context provided.
@@ -50,7 +61,7 @@ function ChatWindow({
         If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context.
         Question: ${userInput}
         =========
-        ${result}
+        ${queryResult}
         =========
         Answer:
         `
@@ -85,7 +96,7 @@ function ChatWindow({
   }, [handleSubmit]);
 
   const loadFile = async () => {
-    console.log('test');
+    console.log('file loaded');
   }
 
   useEffect(() => {

src/components/FileLoader.jsx

@@ -5,6 +5,13 @@ import * as PDFJS from 'pdfjs-dist/build/pdf';
 
 PDFJS.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${PDFJS.version}/pdf.worker.min.js`;
 
+const readFile = (blob) => new Promise((resolve, reject) => {
+  const reader = new FileReader();
+  reader.onload = (event) => resolve(event.target.result);
+  reader.onerror = reject;
+  reader.readAsText(blob);
+});
+
 export default class Pdf {
   static async getPageText(pdf, pageNo) {
     const page = await pdf.getPage(pageNo);
@@ -43,16 +50,11 @@ export const FileLoader = ({ setFileText }) => {
             const blob = new Blob([file], { type: 'text/plain' });
             if (file.type === "application/pdf") {
               text = await Pdf.getPDFText(URL.createObjectURL(blob));
-            } else {
-              let reader = new FileReader();
-    
-              reader.addEventListener('load', function (e) {
-                text = e.target.result;
-              });
-              
-              reader.readAsBinaryString(file);
+            } else { 
+              text = await readFile(file)
             }
-            setFileText(text)
+            console.log(`file text: ${text}`);
+            setFileText(text);
             setUploadStatus("Embed Complete");
           }
         }}