added loading existing files

app.py

@@ -6,10 +6,11 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from langchain_community.embeddings import HuggingFaceInferenceAPIEmbeddings
 import google.generativeai as genai
 from langchain.text_splitter import RecursiveCharacterTextSplitter
-from utils import add_file_to_chroma,remove_file_from_chroma,generate_query_response
+from utils import add_file_to_chroma, remove_file_from_chroma, generate_query_response
 
-from langchain_chroma import Chroma
+from db import init_db,add_file_to_db,remove_file_from_db,get_all_files
 
+from langchain_chroma import Chroma
 import os
 from dotenv import load_dotenv
 import logging
@@ -25,6 +26,7 @@ HF_TOKEN = os.getenv('HF_TOKEN')
 GOOGLE_API_KEY = os.getenv('GOOGLE_API_KEY')
 CHROMA_PATH = "chroma"
 UPLOAD_FOLDER = "uploads"
+DB_PATH = "files.db"
 PROMPT_TEMPLATE = """
 Answer the given query based only on the context given below.
 context:
@@ -54,10 +56,30 @@ CORS(app)
 # Initialize ChromaDB client
 db = Chroma(persist_directory=CHROMA_PATH, embedding_function=hugging_face_ef)
 
+
+# Initialize database
+init_db()
+
 @app.route('/')
 def index():
     return render_template('index.html')
 
+@app.route('/get-files', methods=['GET'])
+def get_files():
+    """Get list of all files."""
+    try:
+        files = get_all_files()
+        return jsonify({
+            'status': 'success',
+            'files': files
+        }), 200
+    except Exception as e:
+        logger.error(f"Error fetching files: {str(e)}")
+        return jsonify({
+            'status': 'error',
+            'error': str(e)
+        }), 500
+
 @app.route('/upload-file', methods=['POST'])
 def upload_file():
     """Handle file uploads."""
@@ -71,6 +93,7 @@ def upload_file():
         return jsonify({'error': 'No file selected', 'status': 'error'}), 400
     
     filename = secure_filename(file.filename)
+    file_type = filename.split('.')[-1].lower()
     file_path = os.path.join(UPLOAD_FOLDER, filename)
     file.save(file_path)
         
@@ -78,12 +101,16 @@ def upload_file():
         # Add file chunks to ChromaDB
         add_file_to_chroma(file_path, file_id, hugging_face_ef, db, logger)
         
+        # Add file to SQLite database
+        add_file_to_db(file_id, filename, file_type)
+        
         return jsonify({
             'message': 'File uploaded successfully',
             'status': 'success',
             'file_info': {
                 'file_id': file_id,
-                'file_name': filename
+                'file_name': filename,
+                'file_type': file_type
             }
         }), 200
     except ValueError as e:
@@ -101,16 +128,26 @@ def upload_file():
 def remove_file():
     file_id = request.form.get('file_id')
     
-    if remove_file_from_chroma(file_id, db):
-        return jsonify({
-            'message': 'File deleted successfully',
-            'status': 'success'
-        }), 200
-    else:
+    try:
+        # Remove from ChromaDB
+        if remove_file_from_chroma(file_id, db):
+            # Remove from SQLite database
+            remove_file_from_db(file_id)
+            return jsonify({
+                'message': 'File deleted successfully',
+                'status': 'success'
+            }), 200
+        else:
+            return jsonify({
+                'message': 'File Not Found',
+                'status': 'fail'
+            }), 404
+    except Exception as e:
+        logger.error(f"Error removing file: {str(e)}")
         return jsonify({
-            'message': 'File Not Found',
-            'status': 'fail'
-        }), 404
+            'error': str(e),
+            'status': 'error'
+        }), 500
 
 @app.route("/ask_query", methods=['POST'])
 def ask_query():

db.py

@@ -0,0 +1,39 @@
+import sqlite3
+
+# Initialize SQLite database for file tracking
+def init_db(DB_PATH = "files.db"):
+    conn = sqlite3.connect(DB_PATH)
+    c = conn.cursor()
+    c.execute('''CREATE TABLE IF NOT EXISTS files
+                 (file_id TEXT PRIMARY KEY, 
+                  file_name TEXT,
+                  file_type TEXT,
+                  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)''')
+    conn.commit()
+    conn.close()
+
+def add_file_to_db(file_id, file_name, file_type,DB_PATH = "files.db"):
+    conn = sqlite3.connect(DB_PATH)
+    c = conn.cursor()
+    c.execute('INSERT INTO files (file_id, file_name, file_type) VALUES (?, ?, ?)',
+              (file_id, file_name, file_type))
+    conn.commit()
+    conn.close()
+
+def remove_file_from_db(file_id,DB_PATH = "files.db"):
+    conn = sqlite3.connect(DB_PATH)
+    c = conn.cursor()
+    c.execute('DELETE FROM files WHERE file_id = ?', (file_id,))
+    conn.commit()
+    conn.close()
+
+def get_all_files(DB_PATH = "files.db"):
+    conn = sqlite3.connect(DB_PATH)
+    c = conn.cursor()
+    c.execute('SELECT file_id, file_name, file_type, created_at FROM files ORDER BY created_at DESC')
+    files = [{'file_id': row[0], 
+              'file_name': row[1], 
+              'file_type': row[2],
+              'created_at': row[3]} for row in c.fetchall()]
+    conn.close()
+    return files

templates/index.html

@@ -421,6 +421,36 @@
             console.log("After Ajax")
         }
 
+        document.addEventListener("DOMContentLoaded", function () {
+            set_loading("Loading Files");
+
+            fetch('/get-files')
+                .then(response => response.json())
+                .then(data => {
+                    if (data.status == 'success') {
+                        let max_file_cnt = 0;
+                        Array.from(data.files).forEach(file => {
+                            file_list[file.file_id] = { name: file.file_name };
+                            const listItem = document.createElement('li');
+                            listItem.id = `file_item_${file.file_id}`
+                            listItem.className = 'flex justify-between items-center bg-gray-200 dark:bg-gray-700 p-2 rounded';
+                            listItem.innerHTML = `
+                    <span style="width: 90%;overflow: hidden;">${file.file_name}</span>
+                    <button class="text-red-500 w-20" onclick="removeFile(${file.file_id})">
+                        <i class="fas fa-times"></i>
+                    </button>`;
+                            fileList.appendChild(listItem);
+
+                            max_file_cnt = max_file_cnt < file.file_id ? file.file_id : max_file_cnt;
+                        });
+                        file_count = max_file_cnt + 1;
+                        manageEmptyFileList();
+                    }
+                }).then(() => {
+                    remove_loading();
+                })
+                .catch(error => console.error("Error fetching files:", error));
+        });
 
         // Toggle sidebar for mobile view
         document.getElementById('menu-toggle').addEventListener('click', function () {