Upload 2 files

App_Function_Libraries/DB/DB_Manager.py

@@ -5,16 +5,14 @@
 import configparser
 import os
 import logging
-import threading
-from contextlib import contextmanager
 from typing import Tuple, List, Union, Dict
-import sqlite3
 import time
 #
 # 3rd-Party Libraries
 from elasticsearch import Elasticsearch
 #
 # Import your existing SQLite functions
+from App_Function_Libraries.DB.SQLite_DB import DatabaseError
 from App_Function_Libraries.DB.SQLite_DB import (
     update_media_content as sqlite_update_media_content,
     list_prompts as sqlite_list_prompts,
@@ -49,166 +47,93 @@ from App_Function_Libraries.DB.SQLite_DB import (
     search_and_display_items as sqlite_search_and_display_items,
     get_conversation_name as sqlite_get_conversation_name,
     add_media_with_keywords as sqlite_add_media_with_keywords,
-    check_media_and_whisper_model as sqlite_check_media_and_whisper_model,
-    DatabaseError, create_document_version as sqlite_create_document_version,
-    get_document_version as sqlite_get_document_version, sqlite_search_db, sqlite_add_media_chunk,
+    check_media_and_whisper_model as sqlite_check_media_and_whisper_model, \
+    create_document_version as sqlite_create_document_version,
+    get_document_version as sqlite_get_document_version, sqlite_search_db, add_media_chunk as sqlite_add_media_chunk,
     sqlite_update_fts_for_media, sqlite_get_unprocessed_media, fetch_item_details as sqlite_fetch_item_details, \
     search_media_database as sqlite_search_media_database, mark_as_trash as sqlite_mark_as_trash, \
     get_media_transcripts as sqlite_get_media_transcripts, get_specific_transcript as sqlite_get_specific_transcript, \
     get_media_summaries as sqlite_get_media_summaries, get_specific_summary as sqlite_get_specific_summary, \
     get_media_prompts as sqlite_get_media_prompts, get_specific_prompt as sqlite_get_specific_prompt, \
-    delete_specific_transcript as sqlite_delete_specific_transcript, delete_specific_summary as sqlite_delete_specific_summary, \
-    delete_specific_prompt as sqlite_delete_specific_prompt, fetch_keywords_for_media as sqlite_fetch_keywords_for_media, \
+    delete_specific_transcript as sqlite_delete_specific_transcript,
+    delete_specific_summary as sqlite_delete_specific_summary, \
+    delete_specific_prompt as sqlite_delete_specific_prompt,
+    fetch_keywords_for_media as sqlite_fetch_keywords_for_media, \
     update_keywords_for_media as sqlite_update_keywords_for_media, check_media_exists as sqlite_check_media_exists, \
     search_prompts as sqlite_search_prompts, get_media_content as sqlite_get_media_content, \
     get_paginated_files as sqlite_get_paginated_files, get_media_title as sqlite_get_media_title, \
     get_all_content_from_database as sqlite_get_all_content_from_database,
+    get_next_media_id as sqlite_get_next_media_id, \
+    batch_insert_chunks as sqlite_batch_insert_chunks, Database,
 )
 #
 # Local Imports
 from App_Function_Libraries.Utils.Utils import load_comprehensive_config, get_database_path, get_project_relative_path
 #
 # End of imports
+############################################################################################################
+
+
 ############################################################################################################
 #
-# Globals
+# Database Config loading
+
+logger = logging.getLogger(__name__)
 
-# Load configuration from config file
 config_path = get_project_relative_path('Config_Files/config.txt')
 config = configparser.ConfigParser()
 config.read(config_path)
 
 db_path: str = config.get('Database', 'sqlite_path', fallback='./Databases/media_summary.db')
-
 backup_path: str = config.get('Database', 'backup_path', fallback='database_backups')
-
 backup_dir: Union[str, bytes] = os.environ.get('DB_BACKUP_DIR', backup_path)
 
-#
-# End of Globals
-############################################################################################################
-#
-# Database Manager Class
+def get_db_config():
+    try:
+        config = load_comprehensive_config()
 
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
+        if 'Database' not in config:
+            print("Warning: 'Database' section not found in config. Using default values.")
+            return default_db_config()
 
+        return {
+            'type': config.get('Database', 'type', fallback='sqlite'),
+            'sqlite_path': config.get('Database', 'sqlite_path', fallback='Databases/media_summary.db'),
+            'elasticsearch_host': config.get('Database', 'elasticsearch_host', fallback='localhost'),
+            'elasticsearch_port': config.getint('Database', 'elasticsearch_port', fallback=9200)
+        }
+    except FileNotFoundError:
+        print("Warning: Config file not found. Using default database configuration.")
+        return default_db_config()
+    except Exception as e:
+        print(f"Error reading config: {str(e)}. Using default database configuration.")
+        return default_db_config()
 
-class Database:
-    def __init__(self, db_name='media_summary.db'):
-        self.db_path = get_database_path(db_name)
-        self.pool = []
-        self.pool_size = 10
-        self.lock = threading.Lock()
-        self.timeout = 60.0  # 60 seconds timeout
-
-    @contextmanager
-    def get_connection(self):
-        retry_count = 5
-        retry_delay = 1
-        while retry_count > 0:
-            try:
-                if self.pool:
-                    conn = self.pool.pop()
-                else:
-                    conn = sqlite3.connect(self.db_path, timeout=self.timeout, check_same_thread=False)
-                    conn.execute("PRAGMA journal_mode=WAL;")  # Enable WAL mode
-                yield conn
-                self.pool.append(conn)
-                return
-            except sqlite3.OperationalError as e:
-                if 'database is locked' in str(e):
-                    logger.warning(f"Database is locked, retrying in {retry_delay} seconds...")
-                    retry_count -= 1
-                    time.sleep(retry_delay)
-                    retry_delay *= 2  # Exponential backoff
-                else:
-                    raise DatabaseError(f"Database error: {e}")
-            except Exception as e:
-                raise DatabaseError(f"Unexpected error: {e}")
-        raise DatabaseError("Database is locked and retries have been exhausted")
-
-    def execute_query(self, query: str, params: Tuple = ()) -> None:
-        with self.lock:  # Use a global lock for write operations
-            with self.get_connection() as conn:
-                try:
-                    cursor = conn.cursor()
-                    cursor.execute(query, params)
-                    conn.commit()
-                except sqlite3.Error as e:
-                    logger.error(f"Database error: {e}, Query: {query}")
-                    raise DatabaseError(f"Database error: {e}, Query: {query}")
-
-    def execute_many(self, query: str, params_list: List[Tuple]) -> None:
-        with self.lock:  # Use a global lock for write operations
-            with self.get_connection() as conn:
-                try:
-                    cursor = conn.cursor()
-                    cursor.executemany(query, params_list)
-                    conn.commit()
-                except sqlite3.Error as e:
-                    logger.error(f"Database error: {e}, Query: {query}")
-                    raise DatabaseError(f"Database error: {e}, Query: {query}")
-
-    def close_all_connections(self):
-        for conn in self.pool:
-            conn.close()
-        self.pool.clear()
+def default_db_config():
+    return {
+        'type': 'sqlite',
+        'sqlite_path': get_database_path('media_summary.db'),
+        'elasticsearch_host': 'localhost',
+        'elasticsearch_port': 9200
+    }
 
+def ensure_directory_exists(file_path):
+    directory = os.path.dirname(file_path)
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+        print(f"Created directory: {directory}")
 
-#
-# class Database:
-#     def __init__(self, db_name='media_summary.db'):
-#         self.db_path = get_database_path(db_name)
-#         self.pool = []
-#         self.pool_size = 10
-#
-#     @contextmanager
-#     def get_connection(self):
-#         retry_count = 5
-#         retry_delay = 1
-#         while retry_count > 0:
-#             try:
-#                 if self.pool:
-#                     conn = self.pool.pop()
-#                 else:
-#                     conn = sqlite3.connect(self.db_path, check_same_thread=False)
-#                 yield conn
-#                 self.pool.append(conn)
-#                 return
-#             except sqlite3.OperationalError as e:
-#                 if 'database is locked' in str(e):
-#                     logger.warning(f"Database is locked, retrying in {retry_delay} seconds...")
-#                     retry_count -= 1
-#                     time.sleep(retry_delay)
-#                     retry_delay *= 2  # Exponential backoff
-#                 else:
-#                     raise DatabaseError(f"Database error: {e}")
-#             except Exception as e:
-#                 raise DatabaseError(f"Unexpected error: {e}")
-#         raise DatabaseError("Database is locked and retries have been exhausted")
-#
-#     def execute_query(self, query: str, params: Tuple = ()) -> None:
-#         with self.get_connection() as conn:
-#             try:
-#                 cursor = conn.cursor()
-#                 cursor.execute(query, params)
-#                 conn.commit()
-#             except sqlite3.Error as e:
-#                 logger.error(f"Database error: {e}, Query: {query}")
-#                 raise DatabaseError(f"Database error: {e}, Query: {query}")
-#
-#     def close_all_connections(self):
-#         for conn in self.pool:
-#             conn.close()
-#         self.pool.clear()
-#
+db_config = get_db_config()
+db_type = db_config['type']
 
-#
-# End of Database Manager Class
-############################################################################################################
-#
-# Database Config loading
+if db_type == 'sqlite':
+    db = Database(os.path.basename(db_config['sqlite_path']))
+elif db_type == 'elasticsearch':
+    raise NotImplementedError("Elasticsearch support not yet implemented")
+else:
+    raise ValueError(f"Unsupported database type: {db_type}")
+
+print(f"Database path: {db.db_path}")
 
 def get_db_config():
     try:
@@ -335,6 +260,12 @@ def get_media_title(*args, **kwargs):
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
+def get_next_media_id():
+    if db_type == 'sqlite':
+        return sqlite_get_next_media_id()
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
 #
 # End of DB-Searching functions
@@ -441,9 +372,18 @@ def ingest_article_to_db(url, title, author, content, keywords, summary, ingesti
         raise ValueError(f"Unsupported database type: {db_type}")
 
 
-def add_media_chunk(media_id: int, chunk_text: str, start_index: int, end_index: int, chunk_id: str):
+def add_media_chunk(*args, **kwargs):
     if db_type == 'sqlite':
-        sqlite_add_media_chunk(db, media_id, chunk_text, start_index, end_index, chunk_id)
+        sqlite_add_media_chunk(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version not yet implemented")
+    else:
+        raise ValueError(f"Unsupported database type: {db_type}")
+
+def batch_insert_chunks(*args, **kwargs):
+    if db_type == 'sqlite':
+        sqlite_batch_insert_chunks(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version not yet implemented")
@@ -850,16 +790,9 @@ def get_document_version(*args, **kwargs):
 # End of Document Versioning Functions
 ############################################################################################################
 
-
-
-############################################################################################################
-#
-# Function to close the database connection for SQLite
-
 def close_connection():
     if db_type == 'sqlite':
-        db.close_all_connections()
-    # Elasticsearch doesn't need explicit closing
+        db.get_connection().close()
 
 #
 # End of file

App_Function_Libraries/DB/SQLite_DB.py

@@ -49,24 +49,22 @@ import csv
 import html
 import logging
 import os
+import queue
 import re
 import shutil
 import sqlite3
-import time
 import traceback
-from contextlib import contextmanager
 from datetime import datetime, timedelta
-from typing import List, Tuple, Dict, Any
+from typing import List, Tuple, Dict, Any, Optional
 # Local Libraries
-from App_Function_Libraries.Utils.Utils import is_valid_url, get_project_relative_path, get_database_path, \
-    get_database_dir, ensure_directory_exists
+from App_Function_Libraries.Utils.Utils import get_project_relative_path, get_database_path, \
+    get_database_dir
+from App_Function_Libraries.Chunk_Lib import chunk_options, chunk_text
+#
 # Third-Party Libraries
 import gradio as gr
 import pandas as pd
 import yaml
-
-
-# Import Local Libraries
 #
 #######################################################################################################################
 # Function Definitions
@@ -78,8 +76,6 @@ def ensure_database_directory():
 ensure_database_directory()
 
 # Set up logging
-#logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-#logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 logger = logging.getLogger(__name__)
 
 # FIXME - Setup properly and test/add documentation for its existence...
@@ -117,6 +113,9 @@ print(f"Backup directory: {backup_dir}")
 
 #
 #
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Backup-related functions
@@ -180,6 +179,9 @@ def rotate_backups(backup_dir, max_backups=10):
 
 #
 #
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # DB-Integrity Check Functions
@@ -204,88 +206,50 @@ def check_database_integrity(db_path):
 
 #
 # End of DB-Integrity Check functions
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
-# Media-related Functions
+# DB Setup Functions
 
-# Custom exceptions
 class DatabaseError(Exception):
     pass
 
-
 class InputError(Exception):
     pass
 
-
-# Database connection function with connection pooling
-
 class Database:
-    def __init__(self, db_name=None):
-        self.db_name = db_name or os.getenv('DB_NAME', 'media_summary.db')
-        self.db_path = get_database_path(self.db_name)
-        ensure_directory_exists(os.path.dirname(self.db_path))
-        self.pool = []
-        self.pool_size = 10
-        logging.info(f"Database initialized with path: {self.db_path}")
-
-    @contextmanager
-    def get_connection(self):
-        conn = None
-        try:
-            conn = self._get_connection_from_pool()
-            yield conn
-        except Exception as e:
-            if conn:
-                conn.rollback()
-            raise e
-        finally:
-            if conn:
-                conn.commit()
-                self.release_connection(conn)
+    def __init__(self, db_name='media_summary.db'):
+        self.db_path = get_database_path(db_name)
+        self.timeout = 60.0  # 60 seconds timeout
 
-    def _get_connection_from_pool(self):
-        retry_count = 5
-        retry_delay = 1
-        while retry_count > 0:
-            try:
-                if self.pool:
-                    return self.pool.pop()
-                else:
-                    return sqlite3.connect(self.db_path, check_same_thread=False)
-            except sqlite3.OperationalError as e:
-                if 'database is locked' in str(e):
-                    logging.warning(f"Database is locked, retrying in {retry_delay} seconds...")
-                    retry_count -= 1
-                    time.sleep(retry_delay)
-                else:
-                    logging.error(f"Database error: {e}")
-                    raise DatabaseError(f"Database error: {e}")
-            except Exception as e:
-                logging.error(f"Unexpected error: {e}")
-                raise DatabaseError(f"Unexpected error: {e}")
-        raise DatabaseError("Database is locked and retries have been exhausted")
-
-    def release_connection(self, conn):
-        if len(self.pool) < self.pool_size:
-            self.pool.append(conn)
-        else:
-            conn.close()
+    def get_connection(self):
+        return sqlite3.connect(self.db_path, timeout=self.timeout)
 
     def execute_query(self, query: str, params: Tuple = ()) -> None:
         with self.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute(query, params)
-
-    def close_all_connections(self):
-        for conn in self.pool:
-            conn.close()
-        self.pool.clear()
-        logging.info("All database connections closed")
+            try:
+                cursor = conn.cursor()
+                cursor.execute(query, params)
+                conn.commit()
+            except sqlite3.Error as e:
+                logging.error(f"Database error: {e}, Query: {query}")
+                raise DatabaseError(f"Database error: {e}, Query: {query}")
 
+    def execute_many(self, query: str, params_list: List[Tuple]) -> None:
+        with self.get_connection() as conn:
+            try:
+                cursor = conn.cursor()
+                cursor.executemany(query, params_list)
+                conn.commit()
+            except sqlite3.Error as e:
+                logging.error(f"Database error: {e}, Query: {query}")
+                raise DatabaseError(f"Database error: {e}, Query: {query}")
 
 db = Database()
 
-def instantiate_SQLite_db():
+def instantiate_sqlite_db():
     global sqlite_db
     sqlite_db = Database()
 
@@ -308,7 +272,8 @@ def create_tables(db) -> None:
             transcription_model TEXT,
             is_trash BOOLEAN DEFAULT 0,
             trash_date DATETIME,
-            vector_embedding BLOB
+            vector_embedding BLOB,
+            chunking_status TEXT DEFAULT 'pending'
         )
         ''',
         '''
@@ -462,13 +427,21 @@ def create_tables(db) -> None:
 create_tables(db)
 
 
-def check_media_exists(title, url):
-    """Check if media with the given title or URL exists in the database."""
-    with db.get_connection() as conn:
-        cursor = conn.cursor()
-        cursor.execute("SELECT id FROM Media WHERE title = ? OR url = ?", (title, url))
-        result = cursor.fetchone()
-        return result is not None
+def check_media_exists(title: str, url: str) -> Optional[int]:
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            query = 'SELECT id FROM Media WHERE title = ? AND url = ?'
+            cursor.execute(query, (title, url))
+            result = cursor.fetchone()
+            logging.debug(f"check_media_exists query: {query}")
+            logging.debug(f"check_media_exists params: title={title}, url={url}")
+            logging.debug(f"check_media_exists result: {result}")
+            return result[0] if result else None
+    except Exception as e:
+        logging.error(f"Error checking if media exists: {str(e)}")
+        logging.error(f"Exception details: {traceback.format_exc()}")
+        return None
 
 
 def check_media_and_whisper_model(title=None, url=None, current_whisper_model=None):
@@ -540,7 +513,7 @@ def check_media_and_whisper_model(title=None, url=None, current_whisper_model=No
         return False, f"Media found with same whisper model (ID: {media_id})"
 
 
-def sqlite_add_media_chunk(db, media_id: int, chunk_text: str, start_index: int, end_index: int, chunk_id: str):
+def add_media_chunk(media_id: int, chunk_text: str, start_index: int, end_index: int, chunk_id: str):
     with db.get_connection() as conn:
         cursor = conn.cursor()
         cursor.execute(
@@ -562,8 +535,21 @@ def sqlite_get_unprocessed_media(db):
         cursor.execute("SELECT id, content, type FROM Media WHERE id NOT IN (SELECT DISTINCT media_id FROM MediaChunks)")
         return cursor.fetchall()
 
+def get_next_media_id():
+    try:
+        conn = sqlite3.connect(db_path)
+        cursor = conn.cursor()
+        cursor.execute("SELECT MAX(media_id) FROM media")
+        max_id = cursor.fetchone()[0]
+        return (max_id or 0) + 1
+    finally:
+        conn.close()
+
 #
 # End of Media-related Functions
+#######################################################################################################################
+
+
 #######################################################################################################################
 # Keyword-related Functions
 #
@@ -612,8 +598,12 @@ def delete_keyword(keyword: str) -> str:
 # Function to add media with keywords
 def add_media_with_keywords(url, title, media_type, content, keywords, prompt, summary, transcription_model, author,
                             ingestion_date):
+    logging.debug(f"Entering add_media_with_keywords: URL={url}, Title={title}")
     # Set default values for missing fields
-    url = url or 'Unknown'
+    if url is None:
+        url = 'localhost'
+    elif url is not None:
+        url = url
     title = title or 'Untitled'
     media_type = media_type or 'Unknown'
     content = content or 'No content available'
@@ -624,10 +614,6 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
     author = author or 'Unknown'
     ingestion_date = ingestion_date or datetime.now().strftime('%Y-%m-%d')
 
-    # Ensure URL is valid
-    if not is_valid_url(url):
-        url = 'localhost'
-
     if media_type not in ['article', 'audio', 'document', 'mediawiki_article', 'mediawiki_dump', 'obsidian_note', 'podcast', 'text', 'video', 'unknown']:
         raise InputError("Invalid media type. Allowed types: article, audio file, document, obsidian_note podcast, text, video, unknown.")
 
@@ -653,68 +639,64 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
 
     try:
         with db.get_connection() as conn:
-            conn.execute("BEGIN TRANSACTION")
             cursor = conn.cursor()
 
-            # Check if media already exists
-            cursor.execute('SELECT id FROM Media WHERE url = ?', (url,))
-            existing_media = cursor.fetchone()
-
-            if existing_media:
-                media_id = existing_media[0]
-                logging.info(f"Updating existing media with ID: {media_id}")
+            # Check if media already exists using both title and URL
+            existing_media_id = check_media_exists(title, url)
+            logging.debug(f"Existing media ID for {url}: {existing_media_id}")
 
+            if existing_media_id:
+                media_id = existing_media_id
+                logging.debug(f"Updating existing media with ID: {media_id}")
                 cursor.execute('''
                 UPDATE Media 
-                SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ?
+                SET content = ?, transcription_model = ?, type = ?, author = ?, ingestion_date = ?
                 WHERE id = ?
-                ''', (content, transcription_model, title, media_type, author, ingestion_date, media_id))
+                ''', (content, transcription_model, media_type, author, ingestion_date, media_id))
             else:
-                logging.info("Creating new media entry")
-
+                logging.debug("Inserting new media")
                 cursor.execute('''
                 INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model)
                 VALUES (?, ?, ?, ?, ?, ?, ?)
                 ''', (url, title, media_type, content, author, ingestion_date, transcription_model))
                 media_id = cursor.lastrowid
+                logging.debug(f"New media inserted with ID: {media_id}")
 
-            logging.info(f"Adding new modification to MediaModifications for media ID: {media_id}")
             cursor.execute('''
             INSERT INTO MediaModifications (media_id, prompt, summary, modification_date)
             VALUES (?, ?, ?, ?)
             ''', (media_id, prompt, summary, ingestion_date))
-            logger.info("New modification added to MediaModifications")
 
-            # Insert keywords and associate with media item
-            logging.info("Processing keywords")
-            for keyword in keyword_list:
-                keyword = keyword.strip().lower()
-                cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,))
-                cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
-                keyword_id = cursor.fetchone()[0]
-                cursor.execute('INSERT OR IGNORE INTO MediaKeywords (media_id, keyword_id) VALUES (?, ?)',
-                               (media_id, keyword_id))
+            # Batch insert keywords
+            keyword_params = [(keyword.strip().lower(),) for keyword in keyword_list]
+            cursor.executemany('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', keyword_params)
+
+            # Get keyword IDs
+            placeholder = ','.join(['?'] * len(keyword_list))
+            cursor.execute(f'SELECT id, keyword FROM Keywords WHERE keyword IN ({placeholder})', keyword_list)
+            keyword_ids = cursor.fetchall()
+
+            # Batch insert media-keyword associations
+            media_keyword_params = [(media_id, keyword_id) for keyword_id, _ in keyword_ids]
+            cursor.executemany('INSERT OR IGNORE INTO MediaKeywords (media_id, keyword_id) VALUES (?, ?)', media_keyword_params)
 
             # Update full-text search index
-            logging.info("Updating full-text search index")
             cursor.execute('INSERT OR REPLACE INTO media_fts (rowid, title, content) VALUES (?, ?, ?)',
                            (media_id, title, content))
 
-            logging.info("Adding new media version")
-            add_media_version(media_id, prompt, summary)
+            # Add media version
+            add_media_version(conn, media_id, prompt, summary)
 
             conn.commit()
             logging.info(f"Media '{title}' successfully added/updated with ID: {media_id}")
 
-            return f"Media '{title}' added/updated successfully with keywords: {', '.join(keyword_list)}"
+        return media_id, f"Media '{title}' added/updated successfully with keywords: {', '.join(keyword_list)}"
 
     except sqlite3.Error as e:
-        conn.rollback()
-        logging.error(f"SQL Error: {e}")
+        logging.error(f"SQL Error in add_media_with_keywords: {e}")
         raise DatabaseError(f"Error adding media with keywords: {e}")
     except Exception as e:
-        conn.rollback()
-        logging.error(f"Unexpected Error: {e}")
+        logging.error(f"Unexpected Error in add_media_with_keywords: {e}")
         raise DatabaseError(f"Unexpected error: {e}")
 
 
@@ -866,6 +848,9 @@ def update_keywords_for_media(media_id, keyword_list):
 
 #
 # End of Keyword-related functions
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Media-related Functions
@@ -922,7 +907,10 @@ def fetch_item_details(media_id: int):
         return "", "", ""
 
 #
-#
+#  End of Media-related Functions
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Media-related Functions
@@ -930,21 +918,19 @@ def fetch_item_details(media_id: int):
 
 
 # Function to add a version of a prompt and summary
-def add_media_version(media_id: int, prompt: str, summary: str) -> None:
+def add_media_version(conn, media_id: int, prompt: str, summary: str) -> None:
     try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
+        cursor = conn.cursor()
 
-            # Get the current version number
-            cursor.execute('SELECT MAX(version) FROM MediaVersion WHERE media_id = ?', (media_id,))
-            current_version = cursor.fetchone()[0] or 0
+        # Get the current version number
+        cursor.execute('SELECT MAX(version) FROM MediaVersion WHERE media_id = ?', (media_id,))
+        current_version = cursor.fetchone()[0] or 0
 
-            # Insert the new version
-            cursor.execute('''
-            INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at)
-            VALUES (?, ?, ?, ?, ?)
-            ''', (media_id, current_version + 1, prompt, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
-            conn.commit()
+        # Insert the new version
+        cursor.execute('''
+        INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at)
+        VALUES (?, ?, ?, ?, ?)
+        ''', (media_id, current_version + 1, prompt, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
     except DatabaseError as e:
         logging.error(f"Error adding media version: {e}")
         raise
@@ -1127,7 +1113,6 @@ def is_valid_date(date_string: str) -> bool:
     except ValueError:
         return False
 
-
 def add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model, media_type='video'):
     db = Database()
     try:
@@ -1158,16 +1143,16 @@ def add_media_to_database(url, info_dict, segments, summary, keywords, custom_pr
                 media_id = existing_media[0]
                 cursor.execute('''
                 UPDATE Media 
-                SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ?
+                SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ?, chunking_status = ?
                 WHERE id = ?
                 ''', (content, whisper_model, info_dict.get('title', 'Untitled'), media_type,
-                      info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), media_id))
+                      info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), 'pending', media_id))
             else:
                 cursor.execute('''
-                INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model)
-                VALUES (?, ?, ?, ?, ?, ?, ?)
+                INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model, chunking_status)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                 ''', (url, info_dict.get('title', 'Untitled'), media_type, content,
-                      info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), whisper_model))
+                      info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), whisper_model, 'pending'))
                 media_id = cursor.lastrowid
 
             # Add modification
@@ -1194,120 +1179,54 @@ def add_media_to_database(url, info_dict, segments, summary, keywords, custom_pr
             cursor.execute('''
             INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at)
             VALUES (?, ?, ?, ?, ?)
-            ''', (
-            media_id, current_version + 1, custom_prompt_input, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
+            ''', (media_id, current_version + 1, custom_prompt_input, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
 
-        return f"Media '{info_dict.get('title', 'Untitled')}' added/updated successfully with keywords: {', '.join(keyword_list)}"
+            conn.commit()
 
-    except sqlite3.Error as e:
+        # Schedule chunking
+        schedule_chunking(media_id, content, info_dict.get('title', 'Untitled'))
+
+        return f"Media '{info_dict.get('title', 'Untitled')}' added/updated successfully with keywords: {', '.join(keyword_list)}. Chunking scheduled."
+
+    except DatabaseError as e:
         logging.error(f"Database error: {e}")
-        raise DatabaseError(f"Error adding media with keywords: {e}")
+        raise
     except Exception as e:
         logging.error(f"Unexpected error: {e}")
         raise DatabaseError(f"Unexpected error: {e}")
 
-# Add ingested media to DB
-# def add_media_to_database(url, info_dict, segments, summary, keywords, custom_prompt_input, whisper_model, media_type='video'):
-#     max_retries = 5
-#     base_delay = 0.1
-#
-#     for attempt in range(max_retries):
-#         try:
-#             with db.get_connection() as conn:
-#                 cursor = conn.cursor()
-#                 conn.execute("BEGIN TRANSACTION")
-#
-#                 try:
-#                     # Extract content from segments
-#                     if isinstance(segments, list):
-#                         content = ' '.join([segment.get('Text', '') for segment in segments if 'Text' in segment])
-#                     elif isinstance(segments, dict):
-#                         content = segments.get('text', '') or segments.get('content', '')
-#                     else:
-#                         content = str(segments)
-#
-#                     # Process keywords
-#                     if isinstance(keywords, str):
-#                         keyword_list = [keyword.strip().lower() for keyword in keywords.split(',')]
-#                     elif isinstance(keywords, (list, tuple)):
-#                         keyword_list = [keyword.strip().lower() for keyword in keywords]
-#                     else:
-#                         keyword_list = ['default']
-#
-#                     # Check if media already exists
-#                     cursor.execute('SELECT id FROM Media WHERE url = ?', (url,))
-#                     existing_media = cursor.fetchone()
-#
-#                     if existing_media:
-#                         media_id = existing_media[0]
-#                         cursor.execute('''
-#                         UPDATE Media
-#                         SET content = ?, transcription_model = ?, title = ?, type = ?, author = ?, ingestion_date = ?
-#                         WHERE id = ?
-#                         ''', (content, whisper_model, info_dict.get('title', 'Untitled'), media_type,
-#                               info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), media_id))
-#                     else:
-#                         cursor.execute('''
-#                         INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model)
-#                         VALUES (?, ?, ?, ?, ?, ?, ?)
-#                         ''', (url, info_dict.get('title', 'Untitled'), media_type, content,
-#                               info_dict.get('uploader', 'Unknown'), datetime.now().strftime('%Y-%m-%d'), whisper_model))
-#                         media_id = cursor.lastrowid
-#
-#                     # Add modification
-#                     cursor.execute('''
-#                     INSERT INTO MediaModifications (media_id, prompt, summary, modification_date)
-#                     VALUES (?, ?, ?, ?)
-#                     ''', (media_id, custom_prompt_input, summary, datetime.now().strftime('%Y-%m-%d')))
-#
-#                     # Process keywords
-#                     for keyword in keyword_list:
-#                         cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,))
-#                         cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
-#                         keyword_id = cursor.fetchone()[0]
-#                         cursor.execute('INSERT OR IGNORE INTO MediaKeywords (media_id, keyword_id) VALUES (?, ?)',
-#                                        (media_id, keyword_id))
-#
-#                     # Update full-text search index
-#                     cursor.execute('INSERT OR REPLACE INTO media_fts (rowid, title, content) VALUES (?, ?, ?)',
-#                                    (media_id, info_dict.get('title', 'Untitled'), content))
-#
-#                     # Add media version
-#                     cursor.execute('SELECT MAX(version) FROM MediaVersion WHERE media_id = ?', (media_id,))
-#                     current_version = cursor.fetchone()[0] or 0
-#                     cursor.execute('''
-#                     INSERT INTO MediaVersion (media_id, version, prompt, summary, created_at)
-#                     VALUES (?, ?, ?, ?, ?)
-#                     ''', (media_id, current_version + 1, custom_prompt_input, summary, datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
-#
-#                     # Create initial document version
-#                     create_document_version(media_id, content)
-#                     # Commit the transaction
-#                     conn.commit()
-#                     logging.info(
-#                         f"Media '{info_dict.get('title', 'Untitled')}' successfully added/updated with ID: {media_id}")
-#                     return f"Media '{info_dict.get('title', 'Untitled')}' added/updated successfully with keywords: {', '.join(keyword_list)}"
-#
-#                 except Exception as e:
-#                     conn.rollback()
-#                     raise e
-#
-#         except sqlite3.OperationalError as e:
-#             if 'database is locked' in str(e) and attempt < max_retries - 1:
-#                 delay = base_delay * (2 ** attempt)
-#                 logging.warning(f"Database is locked, retrying in {delay:.2f} seconds...")
-#                 time.sleep(delay)
-#             else:
-#                 logging.error(f"Database error after {attempt + 1} attempts: {e}")
-#                 raise DatabaseError(f"Error adding media with keywords: {e}")
-#         except Exception as e:
-#             logging.error(f"Unexpected error: {e}")
-#             raise DatabaseError(f"Unexpected error: {e}")
-#
-#     raise DatabaseError("Failed to add media to database after multiple attempts")
+
+# FIXME: This function is not complete and needs to be implemented
+def schedule_chunking(media_id: int, content: str, media_name: str):
+    try:
+        chunks = chunk_text(content, chunk_options['method'], chunk_options['max_size'], chunk_options['overlap'])
+        db = Database()
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            for i, chunk in enumerate(chunks):
+                cursor.execute('''
+                INSERT INTO MediaChunks (media_id, chunk_text, start_index, end_index, chunk_id)
+                VALUES (?, ?, ?, ?, ?)
+                ''', (media_id, chunk, i * chunk_options['max_size'],
+                      min((i + 1) * chunk_options['max_size'], len(content)),
+                      f"{media_id}_chunk_{i}"))
+            conn.commit()
+
+        # Update chunking status
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("UPDATE Media SET chunking_status = 'completed' WHERE id = ?", (media_id,))
+            conn.commit()
+
+    except Exception as e:
+        logging.error(f"Error scheduling chunking for media_id {media_id}: {str(e)}")
+        # You might want to update the chunking_status to 'failed' here
 
 #
 # End of ....
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Functions to manage prompts DB
@@ -1570,6 +1489,9 @@ def delete_prompt(prompt_id):
 
 #
 #
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Function to fetch/update media content
@@ -1834,6 +1756,9 @@ def search_and_display_items(query, search_type, page, entries_per_page,char_cou
 
 #
 # End of Functions to manage prompts DB / Fetch and update media content
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Obsidian-related Functions
@@ -1899,6 +1824,9 @@ def import_obsidian_note_to_db(note_data):
 
 #
 # End of Obsidian-related Functions
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Chat-related Functions
@@ -2501,6 +2429,9 @@ def get_paginated_files(page: int = 1, results_per_page: int = 50) -> Tuple[List
 
 #
 # End of Functions to handle deletion of media items
+#######################################################################################################################
+
+
 #######################################################################################################################
 #
 # Functions to manage document versions
@@ -2572,4 +2503,153 @@ def get_document_version(media_id: int, version_number: int = None) -> Dict[str,
 
 #
 # End of Functions to manage document versions
-#######################################################################################################################
+#######################################################################################################################
+
+
+#######################################################################################################################
+#
+# Functions to manage media chunks
+
+def process_chunks(database, chunks: List[Dict], media_id: int, batch_size: int = 100):
+    """
+    Process chunks in batches and insert them into the database.
+
+    :param database: Database instance to use for inserting chunks
+    :param chunks: List of chunk dictionaries
+    :param media_id: ID of the media these chunks belong to
+    :param batch_size: Number of chunks to process in each batch
+    """
+    total_chunks = len(chunks)
+    processed_chunks = 0
+
+    for i in range(0, total_chunks, batch_size):
+        batch = chunks[i:i + batch_size]
+        chunk_data = [
+            (media_id, chunk['text'], chunk['start_index'], chunk['end_index'])
+            for chunk in batch
+        ]
+
+        try:
+            database.execute_many(
+                "INSERT INTO MediaChunks (media_id, chunk_text, start_index, end_index) VALUES (?, ?, ?, ?)",
+                chunk_data
+            )
+            processed_chunks += len(batch)
+            logging.info(f"Processed {processed_chunks}/{total_chunks} chunks for media_id {media_id}")
+        except Exception as e:
+            logging.error(f"Error inserting chunk batch for media_id {media_id}: {e}")
+            # Optionally, you could raise an exception here to stop processing
+            # raise
+
+    logging.info(f"Finished processing all {total_chunks} chunks for media_id {media_id}")
+
+
+# Usage example:
+# chunks = [{'text': 'chunk1', 'start_index': 0, 'end_index': 10}, ...]
+# process_chunks(db, chunks, media_id=1, batch_size=100)
+
+def batch_insert_chunks(conn, chunks, media_id):
+    cursor = conn.cursor()
+    chunk_data = [(
+        media_id,
+        chunk['text'],
+        chunk['metadata']['start_index'],
+        chunk['metadata']['end_index'],
+        f"{media_id}_chunk_{i}"
+    ) for i, chunk in enumerate(chunks, 1)]
+
+    cursor.executemany('''
+    INSERT INTO MediaChunks (media_id, chunk_text, start_index, end_index, chunk_id)
+    VALUES (?, ?, ?, ?, ?)
+    ''', chunk_data)
+
+
+chunk_queue = queue.Queue()
+
+def chunk_processor():
+    while True:
+        chunk_batch = chunk_queue.get()
+        if chunk_batch is None:
+            break
+        try:
+            with db.get_connection() as conn:
+                conn.execute("BEGIN TRANSACTION")
+                try:
+                    batch_insert_chunks(conn, chunk_batch['chunks'], chunk_batch['media_id'])
+                    conn.commit()
+                except Exception as e:
+                    conn.rollback()
+                    logging.error(f"Error in batch insert: {str(e)}")
+        except Exception as e:
+            logging.error(f"Error processing chunk batch: {str(e)}")
+        finally:
+            chunk_queue.task_done()
+
+# Start the chunk processor thread
+#chunk_processor_thread = threading.Thread(target=chunk_processor)
+#chunk_processor_thread.start()
+
+# Make sure to properly shut down the chunk processor when your application exits
+# def shutdown_chunk_processor():
+#     chunk_queue.put(None)
+#     chunk_processor_thread.join()
+
+#FIXME - add into main db creation code
+def update_media_chunks_table():
+    with db.get_connection() as conn:
+        cursor = conn.cursor()
+        cursor.execute('''
+        CREATE TABLE IF NOT EXISTS MediaChunks_new (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            media_id INTEGER,
+            chunk_text TEXT,
+            start_index INTEGER,
+            end_index INTEGER,
+            chunk_id TEXT,
+            FOREIGN KEY (media_id) REFERENCES Media(id)
+        )
+        ''')
+        cursor.execute('''
+        INSERT INTO MediaChunks_new (media_id, chunk_text, start_index, end_index)
+        SELECT media_id, chunk_text, start_index, end_index FROM MediaChunks
+        ''')
+        cursor.execute('DROP TABLE MediaChunks')
+        cursor.execute('ALTER TABLE MediaChunks_new RENAME TO MediaChunks')
+
+    logger.info("Updated MediaChunks table schema")
+
+update_media_chunks_table()
+# Above function is a dirty hack that should be merged into the initial DB creation statement. This is a placeholder
+# FIXME
+
+
+# This is backwards compatibility for older setups.
+# Function to add a missing column to the Media table
+def add_missing_column_if_not_exists(db, table_name, column_name, column_definition):
+    try:
+        # Check if the column already exists in the table
+        cursor = db.cursor()
+        cursor.execute(f"PRAGMA table_info({table_name})")
+        columns = [column[1] for column in cursor.fetchall()]
+
+        # If the column is not found, add it
+        if column_name not in columns:
+            logging.info(f"Adding missing column '{column_name}' to table '{table_name}'")
+            cursor.execute(f"ALTER TABLE {table_name} ADD COLUMN {column_definition}")
+            db.commit()
+            logging.info(f"Column '{column_name}' added successfully.")
+        else:
+            logging.info(f"Column '{column_name}' already exists in table '{table_name}'")
+
+    except sqlite3.Error as e:
+        logging.error(f"Error checking or adding column '{column_name}' in table '{table_name}': {e}")
+        raise
+
+# Example usage of the function
+def update_media_table(db):
+    # Add chunking_status column if it doesn't exist
+    add_missing_column_if_not_exists(db, 'Media', 'chunking_status', "TEXT DEFAULT 'pending'")
+
+#
+# End of Functions to manage media chunks
+#######################################################################################################################