Syncing latest changes, lets see what breaks

App_Function_Libraries/Article_Summarization_Lib.py

@@ -1,292 +1,221 @@
-# Article_Summarization_Lib.py
-#########################################
-# Article Summarization Library
-# This library is used to handle summarization of articles.
-
-#
-####
-#
-####################
-# Function List
-#
-# 1.
-#
-####################
-#
-# Import necessary libraries
-import datetime
-from datetime import datetime
-import gradio as gr
-import json
-import os
-import logging
-import requests
-# 3rd-Party Imports
-from tqdm import tqdm
-
-from App_Function_Libraries.Utils import sanitize_filename
-# Local Imports
-from Article_Extractor_Lib import scrape_article
-from Local_Summarization_Lib import summarize_with_llama, summarize_with_oobabooga, summarize_with_tabbyapi, \
-    summarize_with_vllm, summarize_with_kobold, save_summary_to_file, summarize_with_local_llm
-from Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, summarize_with_cohere, \
-    summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, summarize_with_huggingface, \
-    summarize_with_mistral
-from SQLite_DB import Database, create_tables, add_media_with_keywords
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-def ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt):
-    try:
-        # Check if content is not empty or whitespace
-        if not content.strip():
-            raise ValueError("Content is empty.")
-
-        db = Database()
-        create_tables()
-        keyword_list = keywords.split(',') if keywords else ["default"]
-        keyword_str = ', '.join(keyword_list)
-
-        # Set default values for missing fields
-        url = url or 'Unknown'
-        title = title or 'Unknown'
-        author = author or 'Unknown'
-        keywords = keywords or 'default'
-        summary = summary or 'No summary available'
-        ingestion_date = ingestion_date or datetime.datetime.now().strftime('%Y-%m-%d')
-
-        # Log the values of all fields before calling add_media_with_keywords
-        logging.debug(f"URL: {url}")
-        logging.debug(f"Title: {title}")
-        logging.debug(f"Author: {author}")
-        logging.debug(f"Content: {content[:50]}... (length: {len(content)})")  # Log first 50 characters of content
-        logging.debug(f"Keywords: {keywords}")
-        logging.debug(f"Summary: {summary}")
-        logging.debug(f"Ingestion Date: {ingestion_date}")
-        logging.debug(f"Custom Prompt: {custom_prompt}")
-
-        # Check if any required field is empty and log the specific missing field
-        if not url:
-            logging.error("URL is missing.")
-            raise ValueError("URL is missing.")
-        if not title:
-            logging.error("Title is missing.")
-            raise ValueError("Title is missing.")
-        if not content:
-            logging.error("Content is missing.")
-            raise ValueError("Content is missing.")
-        if not keywords:
-            logging.error("Keywords are missing.")
-            raise ValueError("Keywords are missing.")
-        if not summary:
-            logging.error("Summary is missing.")
-            raise ValueError("Summary is missing.")
-        if not ingestion_date:
-            logging.error("Ingestion date is missing.")
-            raise ValueError("Ingestion date is missing.")
-        if not custom_prompt:
-            logging.error("Custom prompt is missing.")
-            raise ValueError("Custom prompt is missing.")
-
-        # Add media with keywords to the database
-        result = add_media_with_keywords(
-            url=url,
-            title=title,
-            media_type='article',
-            content=content,
-            keywords=keyword_str or "article_default",
-            prompt=custom_prompt or None,
-            summary=summary or "No summary generated",
-            transcription_model=None,  # or some default value if applicable
-            author=author or 'Unknown',
-            ingestion_date=ingestion_date
-        )
-        return result
-    except Exception as e:
-        logging.error(f"Failed to ingest article to the database: {e}")
-        return str(e)
-
-
-def scrape_and_summarize_multiple(urls, custom_prompt_arg, api_name, api_key, keywords, custom_article_titles, system_message=None):
-    urls = [url.strip() for url in urls.split('\n') if url.strip()]
-    custom_titles = custom_article_titles.split('\n') if custom_article_titles else []
-
-    results = []
-    errors = []
-
-    # Create a progress bar
-    progress = gr.Progress()
-
-    for i, url in tqdm(enumerate(urls), total=len(urls), desc="Processing URLs"):
-        custom_title = custom_titles[i] if i < len(custom_titles) else None
-        try:
-            result = scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_title, system_message)
-            results.append(f"Results for URL {i + 1}:\n{result}")
-        except Exception as e:
-            error_message = f"Error processing URL {i + 1} ({url}): {str(e)}"
-            errors.append(error_message)
-            results.append(f"Failed to process URL {i + 1}: {url}")
-
-        # Update progress
-        progress((i + 1) / len(urls), desc=f"Processed {i + 1}/{len(urls)} URLs")
-
-    # Combine results and errors
-    combined_output = "\n".join(results)
-    if errors:
-        combined_output += "\n\nErrors encountered:\n" + "\n".join(errors)
-
-    return combined_output
-
-
-def scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_article_title, system_message=None):
-    try:
-        # Step 1: Scrape the article
-        article_data = scrape_article(url)
-        print(f"Scraped Article Data: {article_data}")  # Debugging statement
-        if not article_data:
-            return "Failed to scrape the article."
-
-        # Use the custom title if provided, otherwise use the scraped title
-        title = custom_article_title.strip() if custom_article_title else article_data.get('title', 'Untitled')
-        author = article_data.get('author', 'Unknown')
-        content = article_data.get('content', '')
-        ingestion_date = datetime.now().strftime('%Y-%m-%d')
-
-        print(f"Title: {title}, Author: {author}, Content Length: {len(content)}")  # Debugging statement
-
-        # Custom system prompt for the article
-        system_message = system_message or "Act as a professional summarizer and summarize this article."
-        # Custom prompt for the article
-        article_custom_prompt = custom_prompt_arg or "Act as a professional summarizer and summarize this article."
-
-        # Step 2: Summarize the article
-        summary = None
-        if api_name:
-            logging.debug(f"Article_Summarizer: Summarization being performed by {api_name}")
-
-            # Sanitize filename for saving the JSON file
-            sanitized_title = sanitize_filename(title)
-            json_file_path = os.path.join("Results", f"{sanitized_title}_segments.json")
-
-            with open(json_file_path, 'w') as json_file:
-                json.dump([{'text': content}], json_file, indent=2)
-
-            # FIXME - Swap out this if/else to use the dedicated function....
-            try:
-                if api_name.lower() == 'openai':
-                    # def summarize_with_openai(api_key, input_data, custom_prompt_arg)
-                    summary = summarize_with_openai(api_key, json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "anthropic":
-                    # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5):
-                    summary = summarize_with_anthropic(api_key, json_file_path, article_custom_prompt, system_message)
-                elif api_name.lower() == "cohere":
-                    # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg)
-                    summary = summarize_with_cohere(api_key, json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "groq":
-                    logging.debug(f"MAIN: Trying to summarize with groq")
-                    # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg):
-                    summary = summarize_with_groq(api_key, json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "openrouter":
-                    logging.debug(f"MAIN: Trying to summarize with OpenRouter")
-                    # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg):
-                    summary = summarize_with_openrouter(api_key, json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "deepseek":
-                    logging.debug(f"MAIN: Trying to summarize with DeepSeek")
-                    # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg):
-                    summary = summarize_with_deepseek(api_key, json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "mistral":
-                    summary = summarize_with_mistral(api_key, json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "llama.cpp":
-                    logging.debug(f"MAIN: Trying to summarize with Llama.cpp")
-                    # def summarize_with_llama(api_url, file_path, token, custom_prompt)
-                    summary = summarize_with_llama(json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "kobold":
-                    logging.debug(f"MAIN: Trying to summarize with Kobold.cpp")
-                    # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url):
-                    summary = summarize_with_kobold(json_file_path, api_key, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "ooba":
-                    # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url):
-                    summary = summarize_with_oobabooga(json_file_path, api_key, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "tabbyapi":
-                    # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP):
-                    summary = summarize_with_tabbyapi(json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "vllm":
-                    logging.debug(f"MAIN: Trying to summarize with VLLM")
-                    # def summarize_with_vllm(api_key, input_data, custom_prompt_input):
-                    summary = summarize_with_vllm(json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "local-llm":
-                    logging.debug(f"MAIN: Trying to summarize with Local LLM")
-                    summary = summarize_with_local_llm(json_file_path, article_custom_prompt, system_message)
-
-                elif api_name.lower() == "huggingface":
-                    logging.debug(f"MAIN: Trying to summarize with huggingface")
-                    # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg):
-                    summarize_with_huggingface(api_key, json_file_path, article_custom_prompt, system_message)
-                # Add additional API handlers here...
-            except requests.exceptions.ConnectionError as e:
-                logging.error(f"Connection error while trying to summarize with {api_name}: {str(e)}")
-
-            if summary:
-                logging.info(f"Article_Summarizer: Summary generated using {api_name} API")
-                save_summary_to_file(summary, json_file_path)
-            else:
-                summary = "Summary not available"
-                logging.warning(f"Failed to generate summary using {api_name} API")
-
-        else:
-            summary = "Article Summarization: No API provided for summarization."
-
-        print(f"Summary: {summary}")  # Debugging statement
-
-        # Step 3: Ingest the article into the database
-        ingestion_result = ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date,
-                                                article_custom_prompt)
-
-        return f"Title: {title}\nAuthor: {author}\nIngestion Result: {ingestion_result}\n\nSummary: {summary}\n\nArticle Contents: {content}"
-    except Exception as e:
-        logging.error(f"Error processing URL {url}: {str(e)}")
-        return f"Failed to process URL {url}: {str(e)}"
-
-
-def ingest_unstructured_text(text, custom_prompt, api_name, api_key, keywords, custom_article_title, system_message=None):
-    title = custom_article_title.strip() if custom_article_title else "Unstructured Text"
-    author = "Unknown"
-    ingestion_date = datetime.now().strftime('%Y-%m-%d')
-
-    # Summarize the unstructured text
-    if api_name:
-        json_file_path = f"Results/{title.replace(' ', '_')}_segments.json"
-        with open(json_file_path, 'w') as json_file:
-            json.dump([{'text': text}], json_file, indent=2)
-
-        if api_name.lower() == 'openai':
-            summary = summarize_with_openai(api_key, json_file_path, custom_prompt, system_message)
-        # Add other APIs as needed
-        else:
-            summary = "Unsupported API."
-    else:
-        summary = "No API provided for summarization."
-
-    # Ingest the unstructured text into the database
-    ingestion_result = ingest_article_to_db('Unstructured Text', title, author, text, keywords, summary, ingestion_date,
-                                            custom_prompt)
-    return f"Title: {title}\nSummary: {summary}\nIngestion Result: {ingestion_result}"
-
-
-
-#
-#
+# Article_Summarization_Lib.py
+#########################################
+# Article Summarization Library
+# This library is used to handle summarization of articles.
+
+#
+####
+#
+####################
+# Function List
+#
+# 1.
+#
+####################
+#
+# Import necessary libraries
+import datetime
+from datetime import datetime
+import gradio as gr
+import json
+import os
+import logging
+import requests
+# 3rd-Party Imports
+from tqdm import tqdm
+
+from App_Function_Libraries.Utils import sanitize_filename
+# Local Imports
+from Article_Extractor_Lib import scrape_article
+from Local_Summarization_Lib import summarize_with_llama, summarize_with_oobabooga, summarize_with_tabbyapi, \
+    summarize_with_vllm, summarize_with_kobold, save_summary_to_file, summarize_with_local_llm
+from Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, summarize_with_cohere, \
+    summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, summarize_with_huggingface, \
+    summarize_with_mistral
+from App_Function_Libraries.DB_Manager import ingest_article_to_db
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+def scrape_and_summarize_multiple(urls, custom_prompt_arg, api_name, api_key, keywords, custom_article_titles, system_message=None):
+    urls = [url.strip() for url in urls.split('\n') if url.strip()]
+    custom_titles = custom_article_titles.split('\n') if custom_article_titles else []
+
+    results = []
+    errors = []
+
+    # Create a progress bar
+    progress = gr.Progress()
+
+    for i, url in tqdm(enumerate(urls), total=len(urls), desc="Processing URLs"):
+        custom_title = custom_titles[i] if i < len(custom_titles) else None
+        try:
+            result = scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_title, system_message)
+            results.append(f"Results for URL {i + 1}:\n{result}")
+        except Exception as e:
+            error_message = f"Error processing URL {i + 1} ({url}): {str(e)}"
+            errors.append(error_message)
+            results.append(f"Failed to process URL {i + 1}: {url}")
+
+        # Update progress
+        progress((i + 1) / len(urls), desc=f"Processed {i + 1}/{len(urls)} URLs")
+
+    # Combine results and errors
+    combined_output = "\n".join(results)
+    if errors:
+        combined_output += "\n\nErrors encountered:\n" + "\n".join(errors)
+
+    return combined_output
+
+
+def scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_article_title, system_message=None):
+    try:
+        # Step 1: Scrape the article
+        article_data = scrape_article(url)
+        print(f"Scraped Article Data: {article_data}")  # Debugging statement
+        if not article_data:
+            return "Failed to scrape the article."
+
+        # Use the custom title if provided, otherwise use the scraped title
+        title = custom_article_title.strip() if custom_article_title else article_data.get('title', 'Untitled')
+        author = article_data.get('author', 'Unknown')
+        content = article_data.get('content', '')
+        ingestion_date = datetime.now().strftime('%Y-%m-%d')
+
+        print(f"Title: {title}, Author: {author}, Content Length: {len(content)}")  # Debugging statement
+
+        # Custom system prompt for the article
+        system_message = system_message or "Act as a professional summarizer and summarize this article."
+        # Custom prompt for the article
+        article_custom_prompt = custom_prompt_arg or "Act as a professional summarizer and summarize this article."
+
+        # Step 2: Summarize the article
+        summary = None
+        if api_name:
+            logging.debug(f"Article_Summarizer: Summarization being performed by {api_name}")
+
+            # Sanitize filename for saving the JSON file
+            sanitized_title = sanitize_filename(title)
+            json_file_path = os.path.join("Results", f"{sanitized_title}_segments.json")
+
+            with open(json_file_path, 'w') as json_file:
+                json.dump([{'text': content}], json_file, indent=2)
+
+            # FIXME - Swap out this if/else to use the dedicated function....
+            try:
+                if api_name.lower() == 'openai':
+                    # def summarize_with_openai(api_key, input_data, custom_prompt_arg)
+                    summary = summarize_with_openai(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "anthropic":
+                    # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5):
+                    summary = summarize_with_anthropic(api_key, json_file_path, article_custom_prompt, system_message)
+                elif api_name.lower() == "cohere":
+                    # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg)
+                    summary = summarize_with_cohere(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "groq":
+                    logging.debug(f"MAIN: Trying to summarize with groq")
+                    # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg):
+                    summary = summarize_with_groq(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "openrouter":
+                    logging.debug(f"MAIN: Trying to summarize with OpenRouter")
+                    # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg):
+                    summary = summarize_with_openrouter(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "deepseek":
+                    logging.debug(f"MAIN: Trying to summarize with DeepSeek")
+                    # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg):
+                    summary = summarize_with_deepseek(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "mistral":
+                    summary = summarize_with_mistral(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "llama.cpp":
+                    logging.debug(f"MAIN: Trying to summarize with Llama.cpp")
+                    # def summarize_with_llama(api_url, file_path, token, custom_prompt)
+                    summary = summarize_with_llama(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "kobold":
+                    logging.debug(f"MAIN: Trying to summarize with Kobold.cpp")
+                    # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url):
+                    summary = summarize_with_kobold(json_file_path, api_key, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "ooba":
+                    # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url):
+                    summary = summarize_with_oobabooga(json_file_path, api_key, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "tabbyapi":
+                    # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP):
+                    summary = summarize_with_tabbyapi(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "vllm":
+                    logging.debug(f"MAIN: Trying to summarize with VLLM")
+                    # def summarize_with_vllm(api_key, input_data, custom_prompt_input):
+                    summary = summarize_with_vllm(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "local-llm":
+                    logging.debug(f"MAIN: Trying to summarize with Local LLM")
+                    summary = summarize_with_local_llm(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "huggingface":
+                    logging.debug(f"MAIN: Trying to summarize with huggingface")
+                    # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg):
+                    summarize_with_huggingface(api_key, json_file_path, article_custom_prompt, system_message)
+                # Add additional API handlers here...
+            except requests.exceptions.ConnectionError as e:
+                logging.error(f"Connection error while trying to summarize with {api_name}: {str(e)}")
+
+            if summary:
+                logging.info(f"Article_Summarizer: Summary generated using {api_name} API")
+                save_summary_to_file(summary, json_file_path)
+            else:
+                summary = "Summary not available"
+                logging.warning(f"Failed to generate summary using {api_name} API")
+
+        else:
+            summary = "Article Summarization: No API provided for summarization."
+
+        print(f"Summary: {summary}")  # Debugging statement
+
+        # Step 3: Ingest the article into the database
+        ingestion_result = ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date,
+                                                article_custom_prompt)
+
+        return f"Title: {title}\nAuthor: {author}\nIngestion Result: {ingestion_result}\n\nSummary: {summary}\n\nArticle Contents: {content}"
+    except Exception as e:
+        logging.error(f"Error processing URL {url}: {str(e)}")
+        return f"Failed to process URL {url}: {str(e)}"
+
+
+def ingest_unstructured_text(text, custom_prompt, api_name, api_key, keywords, custom_article_title, system_message=None):
+    title = custom_article_title.strip() if custom_article_title else "Unstructured Text"
+    author = "Unknown"
+    ingestion_date = datetime.now().strftime('%Y-%m-%d')
+
+    # Summarize the unstructured text
+    if api_name:
+        json_file_path = f"Results/{title.replace(' ', '_')}_segments.json"
+        with open(json_file_path, 'w') as json_file:
+            json.dump([{'text': text}], json_file, indent=2)
+
+        if api_name.lower() == 'openai':
+            summary = summarize_with_openai(api_key, json_file_path, custom_prompt, system_message)
+        # Add other APIs as needed
+        else:
+            summary = "Unsupported API."
+    else:
+        summary = "No API provided for summarization."
+
+    # Ingest the unstructured text into the database
+    ingestion_result = ingest_article_to_db('Unstructured Text', title, author, text, keywords, summary, ingestion_date,
+                                            custom_prompt)
+    return f"Title: {title}\nSummary: {summary}\nIngestion Result: {ingestion_result}"
+
+
+
+#
+#
 #######################################################################################################################

App_Function_Libraries/Audio_Files.py

@@ -1,692 +1,692 @@
-# Audio_Files.py
-#########################################
-# Audio Processing Library
-# This library is used to download or load audio files from a local directory.
-#
-####
-#
-# Functions:
-#
-# download_audio_file(url, save_path)
-# process_audio(
-# process_audio_file(audio_url, audio_file, whisper_model="small.en", api_name=None, api_key=None)
-#
-#
-#########################################
-# Imports
-import json
-import logging
-import os
-import subprocess
-import tempfile
-import uuid
-from datetime import datetime
-from pathlib import Path
-
-import requests
-import yt_dlp
-
-from App_Function_Libraries.Audio_Transcription_Lib import speech_to_text
-from App_Function_Libraries.Chunk_Lib import improved_chunking_process
-#
-# Local Imports
-from App_Function_Libraries.SQLite_DB import add_media_to_database, add_media_with_keywords, \
-    check_media_and_whisper_model
-from App_Function_Libraries.Summarization_General_Lib import save_transcription_and_summary, perform_transcription, \
-    perform_summarization
-from App_Function_Libraries.Utils import create_download_directory, save_segments_to_json, downloaded_files, \
-    sanitize_filename
-from App_Function_Libraries.Video_DL_Ingestion_Lib import extract_metadata
-
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-MAX_FILE_SIZE = 500 * 1024 * 1024
-
-
-def download_audio_file(url, current_whisper_model="", use_cookies=False, cookies=None):
-    try:
-        # Check if media already exists in the database and compare whisper models
-        should_download, reason = check_media_and_whisper_model(
-            url=url,
-            current_whisper_model=current_whisper_model
-        )
-
-        if not should_download:
-            logging.info(f"Skipping audio download: {reason}")
-            return None
-
-        logging.info(f"Proceeding with audio download: {reason}")
-
-        # Set up the request headers
-        headers = {}
-        if use_cookies and cookies:
-            try:
-                cookie_dict = json.loads(cookies)
-                headers['Cookie'] = '; '.join([f'{k}={v}' for k, v in cookie_dict.items()])
-            except json.JSONDecodeError:
-                logging.warning("Invalid cookie format. Proceeding without cookies.")
-
-        # Make the request
-        response = requests.get(url, headers=headers, stream=True)
-        # Raise an exception for bad status codes
-        response.raise_for_status()
-
-        # Get the file size
-        file_size = int(response.headers.get('content-length', 0))
-        if file_size > 500 * 1024 * 1024:  # 500 MB limit
-            raise ValueError("File size exceeds the 500MB limit.")
-
-        # Generate a unique filename
-        file_name = f"audio_{uuid.uuid4().hex[:8]}.mp3"
-        save_path = os.path.join('downloads', file_name)
-
-        # Ensure the downloads directory exists
-        os.makedirs('downloads', exist_ok=True)
-
-
-        # Download the file
-        with open(save_path, 'wb') as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                if chunk:
-                    f.write(chunk)
-
-        logging.info(f"Audio file downloaded successfully: {save_path}")
-        return save_path
-
-    except requests.RequestException as e:
-        logging.error(f"Error downloading audio file: {str(e)}")
-        raise
-    except ValueError as e:
-        logging.error(str(e))
-        raise
-    except Exception as e:
-        logging.error(f"Unexpected error downloading audio file: {str(e)}")
-        raise
-
-
-def process_audio(
-        audio_file_path,
-        num_speakers=2,
-        whisper_model="small.en",
-        custom_prompt_input=None,
-        offset=0,
-        api_name=None,
-        api_key=None,
-        vad_filter=False,
-        rolling_summarization=False,
-        detail_level=0.01,
-        keywords="default,no_keyword_set",
-        chunk_text_by_words=False,
-        max_words=0,
-        chunk_text_by_sentences=False,
-        max_sentences=0,
-        chunk_text_by_paragraphs=False,
-        max_paragraphs=0,
-        chunk_text_by_tokens=False,
-        max_tokens=0
-):
-    try:
-
-        # Perform transcription
-        audio_file_path, segments = perform_transcription(audio_file_path, offset, whisper_model, vad_filter)
-
-        if audio_file_path is None or segments is None:
-            logging.error("Process_Audio: Transcription failed or segments not available.")
-            return "Process_Audio: Transcription failed.", None, None, None, None, None
-
-        logging.debug(f"Process_Audio: Transcription audio_file: {audio_file_path}")
-        logging.debug(f"Process_Audio: Transcription segments: {segments}")
-
-        transcription_text = {'audio_file': audio_file_path, 'transcription': segments}
-        logging.debug(f"Process_Audio: Transcription text: {transcription_text}")
-
-        # Save segments to JSON
-        segments_json_path = save_segments_to_json(segments)
-
-        # Perform summarization
-        summary_text = None
-        if api_name:
-            if rolling_summarization is not None:
-                pass
-                # FIXME rolling summarization
-                # summary_text = rolling_summarize_function(
-                #     transcription_text,
-                #     detail=detail_level,
-                #     api_name=api_name,
-                #     api_key=api_key,
-                #     custom_prompt=custom_prompt_input,
-                #     chunk_by_words=chunk_text_by_words,
-                #     max_words=max_words,
-                #     chunk_by_sentences=chunk_text_by_sentences,
-                #     max_sentences=max_sentences,
-                #     chunk_by_paragraphs=chunk_text_by_paragraphs,
-                #     max_paragraphs=max_paragraphs,
-                #     chunk_by_tokens=chunk_text_by_tokens,
-                #     max_tokens=max_tokens
-                # )
-            else:
-                summary_text = perform_summarization(api_name, segments_json_path, custom_prompt_input, api_key)
-
-            if summary_text is None:
-                logging.error("Summary text is None. Check summarization function.")
-                summary_file_path = None
-        else:
-            summary_text = 'Summary not available'
-            summary_file_path = None
-
-        # Save transcription and summary
-        download_path = create_download_directory("Audio_Processing")
-        json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text,
-                                                                           download_path)
-
-        # Update function call to add_media_to_database so that it properly applies the title, author and file type
-        # Add to database
-        add_media_to_database(None, {'title': 'Audio File', 'author': 'Unknown'}, segments, summary_text, keywords,
-                              custom_prompt_input, whisper_model)
-
-        return transcription_text, summary_text, json_file_path, summary_file_path, None, None
-
-    except Exception as e:
-        logging.error(f"Error in process_audio: {str(e)}")
-        return str(e), None, None, None, None, None
-
-
-def process_single_audio(audio_file_path, whisper_model, api_name, api_key, keep_original,custom_keywords, source,
-                         custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
-                         use_multi_level_chunking, chunk_language):
-    progress = []
-    transcription = ""
-    summary = ""
-
-    def update_progress(message):
-        progress.append(message)
-        return "\n".join(progress)
-
-    try:
-        # Check file size before processing
-        file_size = os.path.getsize(audio_file_path)
-        if file_size > MAX_FILE_SIZE:
-            update_progress(f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f} MB. Skipping this file.")
-            return "\n".join(progress), "", ""
-
-        # Perform transcription
-        update_progress("Starting transcription...")
-        segments = speech_to_text(audio_file_path, whisper_model=whisper_model)
-        transcription = " ".join([segment['Text'] for segment in segments])
-        update_progress("Audio transcribed successfully.")
-
-        # Perform summarization if API is provided
-        if api_name and api_key:
-            update_progress("Starting summarization...")
-            summary = perform_summarization(api_name, transcription, "Summarize the following audio transcript",
-                                            api_key)
-            update_progress("Audio summarized successfully.")
-        else:
-            summary = "No summary available"
-
-        # Prepare keywords
-        keywords = "audio,transcription"
-        if custom_keywords:
-            keywords += f",{custom_keywords}"
-
-        # Add to database
-        add_media_with_keywords(
-            url=source,
-            title=os.path.basename(audio_file_path),
-            media_type='audio',
-            content=transcription,
-            keywords=keywords,
-            prompt="Summarize the following audio transcript",
-            summary=summary,
-            transcription_model=whisper_model,
-            author="Unknown",
-            ingestion_date=None  # This will use the current date
-        )
-        update_progress("Audio file added to database successfully.")
-
-        if not keep_original and source != "Uploaded File":
-            os.remove(audio_file_path)
-            update_progress(f"Temporary file {audio_file_path} removed.")
-        elif keep_original and source != "Uploaded File":
-            update_progress(f"Original audio file kept at: {audio_file_path}")
-
-    except Exception as e:
-        update_progress(f"Error processing {source}: {str(e)}")
-        transcription = f"Error: {str(e)}"
-        summary = "No summary due to error"
-
-    return "\n".join(progress), transcription, summary
-
-
-def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key, use_cookies, cookies, keep_original,
-                        custom_keywords, custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap,
-                        use_adaptive_chunking, use_multi_level_chunking, chunk_language, diarize):
-    progress = []
-    temp_files = []
-    all_transcriptions = []
-    all_summaries = []
-
-    def update_progress(message):
-        progress.append(message)
-        return "\n".join(progress)
-
-    def cleanup_files():
-        for file in temp_files:
-            try:
-                if os.path.exists(file):
-                    os.remove(file)
-                    update_progress(f"Temporary file {file} removed.")
-            except Exception as e:
-                update_progress(f"Failed to remove temporary file {file}: {str(e)}")
-
-    def reencode_mp3(mp3_file_path):
-        try:
-            reencoded_mp3_path = mp3_file_path.replace(".mp3", "_reencoded.mp3")
-            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, '-codec:a', 'libmp3lame', reencoded_mp3_path], check=True)
-            update_progress(f"Re-encoded {mp3_file_path} to {reencoded_mp3_path}.")
-            return reencoded_mp3_path
-        except subprocess.CalledProcessError as e:
-            update_progress(f"Error re-encoding {mp3_file_path}: {str(e)}")
-            raise
-
-    def convert_mp3_to_wav(mp3_file_path):
-        try:
-            wav_file_path = mp3_file_path.replace(".mp3", ".wav")
-            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, wav_file_path], check=True)
-            update_progress(f"Converted {mp3_file_path} to {wav_file_path}.")
-            return wav_file_path
-        except subprocess.CalledProcessError as e:
-            update_progress(f"Error converting {mp3_file_path} to WAV: {str(e)}")
-            raise
-
-    try:
-        # Check and set the ffmpeg command
-        global ffmpeg_cmd
-        if os.name == "nt":
-            logging.debug("Running on Windows")
-            ffmpeg_cmd = os.path.join(os.getcwd(), "Bin", "ffmpeg.exe")
-        else:
-            ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
-
-        # Ensure ffmpeg is accessible
-        if not os.path.exists(ffmpeg_cmd) and os.name == "nt":
-            raise FileNotFoundError(f"ffmpeg executable not found at path: {ffmpeg_cmd}")
-
-        # Define chunk options early to avoid undefined errors
-        chunk_options = {
-            'method': chunk_method,
-            'max_size': max_chunk_size,
-            'overlap': chunk_overlap,
-            'adaptive': use_adaptive_chunking,
-            'multi_level': use_multi_level_chunking,
-            'language': chunk_language
-        }
-
-        # Process multiple URLs
-        urls = [url.strip() for url in audio_urls.split('\n') if url.strip()]
-
-        for i, url in enumerate(urls):
-            update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}")
-
-            # Download and process audio file
-            audio_file_path = download_audio_file(url, use_cookies, cookies)
-            if not os.path.exists(audio_file_path):
-                update_progress(f"Downloaded file not found: {audio_file_path}")
-                continue
-
-            temp_files.append(audio_file_path)
-            update_progress("Audio file downloaded successfully.")
-
-            # Re-encode MP3 to fix potential issues
-            reencoded_mp3_path = reencode_mp3(audio_file_path)
-            if not os.path.exists(reencoded_mp3_path):
-                update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
-                continue
-
-            temp_files.append(reencoded_mp3_path)
-
-            # Convert re-encoded MP3 to WAV
-            wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
-            if not os.path.exists(wav_file_path):
-                update_progress(f"Converted WAV file not found: {wav_file_path}")
-                continue
-
-            temp_files.append(wav_file_path)
-
-            # Initialize transcription
-            transcription = ""
-
-            # Transcribe audio
-            if diarize:
-                segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
-            else:
-                segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
-
-            # Handle segments nested under 'segments' key
-            if isinstance(segments, dict) and 'segments' in segments:
-                segments = segments['segments']
-
-            if isinstance(segments, list):
-                transcription = " ".join([segment.get('Text', '') for segment in segments])
-                update_progress("Audio transcribed successfully.")
-            else:
-                update_progress("Unexpected segments format received from speech_to_text.")
-                logging.error(f"Unexpected segments format: {segments}")
-                continue
-
-            if not transcription.strip():
-                update_progress("Transcription is empty.")
-            else:
-                # Apply chunking
-                chunked_text = improved_chunking_process(transcription, chunk_options)
-
-                # Summarize
-                if api_name:
-                    try:
-                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
-                        update_progress("Audio summarized successfully.")
-                    except Exception as e:
-                        logging.error(f"Error during summarization: {str(e)}")
-                        summary = "Summary generation failed"
-                else:
-                    summary = "No summary available (API not provided)"
-
-                all_transcriptions.append(transcription)
-                all_summaries.append(summary)
-
-                # Add to database
-                add_media_with_keywords(
-                    url=url,
-                    title=os.path.basename(wav_file_path),
-                    media_type='audio',
-                    content=transcription,
-                    keywords=custom_keywords,
-                    prompt=custom_prompt_input,
-                    summary=summary,
-                    transcription_model=whisper_model,
-                    author="Unknown",
-                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
-                )
-                update_progress("Audio file processed and added to database.")
-
-        # Process uploaded file if provided
-        if audio_file:
-            if os.path.getsize(audio_file.name) > MAX_FILE_SIZE:
-                update_progress(
-                    f"Uploaded file size exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB. Skipping this file.")
-            else:
-                # Re-encode MP3 to fix potential issues
-                reencoded_mp3_path = reencode_mp3(audio_file.name)
-                if not os.path.exists(reencoded_mp3_path):
-                    update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
-                    return update_progress("Processing failed: Re-encoded file not found"), "", ""
-
-                temp_files.append(reencoded_mp3_path)
-
-                # Convert re-encoded MP3 to WAV
-                wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
-                if not os.path.exists(wav_file_path):
-                    update_progress(f"Converted WAV file not found: {wav_file_path}")
-                    return update_progress("Processing failed: Converted WAV file not found"), "", ""
-
-                temp_files.append(wav_file_path)
-
-                # Initialize transcription
-                transcription = ""
-
-                if diarize:
-                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
-                else:
-                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
-
-                # Handle segments nested under 'segments' key
-                if isinstance(segments, dict) and 'segments' in segments:
-                    segments = segments['segments']
-
-                if isinstance(segments, list):
-                    transcription = " ".join([segment.get('Text', '') for segment in segments])
-                else:
-                    update_progress("Unexpected segments format received from speech_to_text.")
-                    logging.error(f"Unexpected segments format: {segments}")
-
-                chunked_text = improved_chunking_process(transcription, chunk_options)
-
-                if api_name and api_key:
-                    try:
-                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
-                        update_progress("Audio summarized successfully.")
-                    except Exception as e:
-                        logging.error(f"Error during summarization: {str(e)}")
-                        summary = "Summary generation failed"
-                else:
-                    summary = "No summary available (API not provided)"
-
-                all_transcriptions.append(transcription)
-                all_summaries.append(summary)
-
-                add_media_with_keywords(
-                    url="Uploaded File",
-                    title=os.path.basename(wav_file_path),
-                    media_type='audio',
-                    content=transcription,
-                    keywords=custom_keywords,
-                    prompt=custom_prompt_input,
-                    summary=summary,
-                    transcription_model=whisper_model,
-                    author="Unknown",
-                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
-                )
-                update_progress("Uploaded file processed and added to database.")
-
-        # Final cleanup
-        if not keep_original:
-            cleanup_files()
-
-        final_progress = update_progress("All processing complete.")
-        final_transcriptions = "\n\n".join(all_transcriptions)
-        final_summaries = "\n\n".join(all_summaries)
-
-        return final_progress, final_transcriptions, final_summaries
-
-    except Exception as e:
-        logging.error(f"Error processing audio files: {str(e)}")
-        cleanup_files()
-        return update_progress(f"Processing failed: {str(e)}"), "", ""
-
-
-def download_youtube_audio(url):
-    try:
-        # Determine ffmpeg path based on the operating system.
-        ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg'
-
-        # Create a temporary directory
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Extract information about the video
-            with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
-                info_dict = ydl.extract_info(url, download=False)
-                sanitized_title = sanitize_filename(info_dict['title'])
-
-            # Setup the temporary filenames
-            temp_video_path = Path(temp_dir) / f"{sanitized_title}_temp.mp4"
-            temp_audio_path = Path(temp_dir) / f"{sanitized_title}.mp3"
-
-            # Initialize yt-dlp with options for downloading
-            ydl_opts = {
-                'format': 'bestaudio[ext=m4a]/best[height<=480]',  # Prefer best audio, or video up to 480p
-                'ffmpeg_location': ffmpeg_path,
-                'outtmpl': str(temp_video_path),
-                'noplaylist': True,
-                'quiet': True
-            }
-
-            # Execute yt-dlp to download the video/audio
-            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                ydl.download([url])
-
-            # Check if the file exists
-            if not temp_video_path.exists():
-                raise FileNotFoundError(f"Expected file was not found: {temp_video_path}")
-
-            # Use ffmpeg to extract audio
-            ffmpeg_command = [
-                ffmpeg_path,
-                '-i', str(temp_video_path),
-                '-vn',  # No video
-                '-acodec', 'libmp3lame',
-                '-b:a', '192k',
-                str(temp_audio_path)
-            ]
-            subprocess.run(ffmpeg_command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-
-            # Check if the audio file was created
-            if not temp_audio_path.exists():
-                raise FileNotFoundError(f"Expected audio file was not found: {temp_audio_path}")
-
-            # Create a persistent directory for the download if it doesn't exist
-            persistent_dir = Path("downloads")
-            persistent_dir.mkdir(exist_ok=True)
-
-            # Move the file from the temporary directory to the persistent directory
-            persistent_file_path = persistent_dir / f"{sanitized_title}.mp3"
-            os.replace(str(temp_audio_path), str(persistent_file_path))
-
-            # Add the file to the list of downloaded files
-            downloaded_files.append(str(persistent_file_path))
-
-            return str(persistent_file_path), f"Audio downloaded successfully: {sanitized_title}.mp3"
-    except Exception as e:
-        return None, f"Error downloading audio: {str(e)}"
-
-
-def process_podcast(url, title, author, keywords, custom_prompt, api_name, api_key, whisper_model,
-                    keep_original=False, enable_diarization=False, use_cookies=False, cookies=None,
-                    chunk_method=None, max_chunk_size=300, chunk_overlap=0, use_adaptive_chunking=False,
-                    use_multi_level_chunking=False, chunk_language='english'):
-    progress = []
-    error_message = ""
-    temp_files = []
-
-    def update_progress(message):
-        progress.append(message)
-        return "\n".join(progress)
-
-    def cleanup_files():
-        if not keep_original:
-            for file in temp_files:
-                try:
-                    if os.path.exists(file):
-                        os.remove(file)
-                        update_progress(f"Temporary file {file} removed.")
-                except Exception as e:
-                    update_progress(f"Failed to remove temporary file {file}: {str(e)}")
-
-    try:
-        # Download podcast
-        audio_file = download_audio_file(url, use_cookies, cookies)
-        temp_files.append(audio_file)
-        update_progress("Podcast downloaded successfully.")
-
-        # Extract metadata
-        metadata = extract_metadata(url)
-        title = title or metadata.get('title', 'Unknown Podcast')
-        author = author or metadata.get('uploader', 'Unknown Author')
-
-        # Format metadata for storage
-        metadata_text = f"""
-Metadata:
-Title: {title}
-Author: {author}
-Series: {metadata.get('series', 'N/A')}
-Episode: {metadata.get('episode', 'N/A')}
-Season: {metadata.get('season', 'N/A')}
-Upload Date: {metadata.get('upload_date', 'N/A')}
-Duration: {metadata.get('duration', 'N/A')} seconds
-Description: {metadata.get('description', 'N/A')}
-"""
-
-        # Update keywords
-        new_keywords = []
-        if metadata.get('series'):
-            new_keywords.append(f"series:{metadata['series']}")
-        if metadata.get('episode'):
-            new_keywords.append(f"episode:{metadata['episode']}")
-        if metadata.get('season'):
-            new_keywords.append(f"season:{metadata['season']}")
-
-        keywords = f"{keywords},{','.join(new_keywords)}" if keywords else ','.join(new_keywords)
-
-        update_progress(f"Metadata extracted - Title: {title}, Author: {author}, Keywords: {keywords}")
-
-        # Transcribe the podcast
-        try:
-            if enable_diarization:
-                segments = speech_to_text(audio_file, whisper_model=whisper_model, diarize=True)
-            else:
-                segments = speech_to_text(audio_file, whisper_model=whisper_model)
-            transcription = " ".join([segment['Text'] for segment in segments])
-            update_progress("Podcast transcribed successfully.")
-        except Exception as e:
-            error_message = f"Transcription failed: {str(e)}"
-            raise
-
-        # Apply chunking
-        chunk_options = {
-            'method': chunk_method,
-            'max_size': max_chunk_size,
-            'overlap': chunk_overlap,
-            'adaptive': use_adaptive_chunking,
-            'multi_level': use_multi_level_chunking,
-            'language': chunk_language
-        }
-        chunked_text = improved_chunking_process(transcription, chunk_options)
-
-        # Combine metadata and transcription
-        full_content = metadata_text + "\n\nTranscription:\n" + transcription
-
-        # Summarize if API is provided
-        summary = None
-        if api_name and api_key:
-            try:
-                summary = perform_summarization(api_name, chunked_text, custom_prompt, api_key)
-                update_progress("Podcast summarized successfully.")
-            except Exception as e:
-                error_message = f"Summarization failed: {str(e)}"
-                raise
-
-        # Add to database
-        try:
-            add_media_with_keywords(
-                url=url,
-                title=title,
-                media_type='podcast',
-                content=full_content,
-                keywords=keywords,
-                prompt=custom_prompt,
-                summary=summary or "No summary available",
-                transcription_model=whisper_model,
-                author=author,
-                ingestion_date=datetime.now().strftime('%Y-%m-%d')
-            )
-            update_progress("Podcast added to database successfully.")
-        except Exception as e:
-            error_message = f"Error adding podcast to database: {str(e)}"
-            raise
-
-        # Cleanup
-        cleanup_files()
-
-        return (update_progress("Processing complete."), full_content, summary or "No summary generated.",
-                title, author, keywords, error_message)
-
-    except Exception as e:
-        logging.error(f"Error processing podcast: {str(e)}")
-        cleanup_files()
-        return update_progress(f"Processing failed: {str(e)}"), "", "", "", "", "", str(e)
-
-
-#
-#
+# Audio_Files.py
+#########################################
+# Audio Processing Library
+# This library is used to download or load audio files from a local directory.
+#
+####
+#
+# Functions:
+#
+# download_audio_file(url, save_path)
+# process_audio(
+# process_audio_file(audio_url, audio_file, whisper_model="small.en", api_name=None, api_key=None)
+#
+#
+#########################################
+# Imports
+import json
+import logging
+import os
+import subprocess
+import tempfile
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+import requests
+import yt_dlp
+
+from App_Function_Libraries.Audio_Transcription_Lib import speech_to_text
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import add_media_to_database, add_media_with_keywords, \
+    check_media_and_whisper_model
+from App_Function_Libraries.Summarization_General_Lib import save_transcription_and_summary, perform_transcription, \
+    perform_summarization
+from App_Function_Libraries.Utils import create_download_directory, save_segments_to_json, downloaded_files, \
+    sanitize_filename
+from App_Function_Libraries.Video_DL_Ingestion_Lib import extract_metadata
+
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+MAX_FILE_SIZE = 500 * 1024 * 1024
+
+
+def download_audio_file(url, current_whisper_model="", use_cookies=False, cookies=None):
+    try:
+        # Check if media already exists in the database and compare whisper models
+        should_download, reason = check_media_and_whisper_model(
+            url=url,
+            current_whisper_model=current_whisper_model
+        )
+
+        if not should_download:
+            logging.info(f"Skipping audio download: {reason}")
+            return None
+
+        logging.info(f"Proceeding with audio download: {reason}")
+
+        # Set up the request headers
+        headers = {}
+        if use_cookies and cookies:
+            try:
+                cookie_dict = json.loads(cookies)
+                headers['Cookie'] = '; '.join([f'{k}={v}' for k, v in cookie_dict.items()])
+            except json.JSONDecodeError:
+                logging.warning("Invalid cookie format. Proceeding without cookies.")
+
+        # Make the request
+        response = requests.get(url, headers=headers, stream=True)
+        # Raise an exception for bad status codes
+        response.raise_for_status()
+
+        # Get the file size
+        file_size = int(response.headers.get('content-length', 0))
+        if file_size > 500 * 1024 * 1024:  # 500 MB limit
+            raise ValueError("File size exceeds the 500MB limit.")
+
+        # Generate a unique filename
+        file_name = f"audio_{uuid.uuid4().hex[:8]}.mp3"
+        save_path = os.path.join('downloads', file_name)
+
+        # Ensure the downloads directory exists
+        os.makedirs('downloads', exist_ok=True)
+
+
+        # Download the file
+        with open(save_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+
+        logging.info(f"Audio file downloaded successfully: {save_path}")
+        return save_path
+
+    except requests.RequestException as e:
+        logging.error(f"Error downloading audio file: {str(e)}")
+        raise
+    except ValueError as e:
+        logging.error(str(e))
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error downloading audio file: {str(e)}")
+        raise
+
+
+def process_audio(
+        audio_file_path,
+        num_speakers=2,
+        whisper_model="small.en",
+        custom_prompt_input=None,
+        offset=0,
+        api_name=None,
+        api_key=None,
+        vad_filter=False,
+        rolling_summarization=False,
+        detail_level=0.01,
+        keywords="default,no_keyword_set",
+        chunk_text_by_words=False,
+        max_words=0,
+        chunk_text_by_sentences=False,
+        max_sentences=0,
+        chunk_text_by_paragraphs=False,
+        max_paragraphs=0,
+        chunk_text_by_tokens=False,
+        max_tokens=0
+):
+    try:
+
+        # Perform transcription
+        audio_file_path, segments = perform_transcription(audio_file_path, offset, whisper_model, vad_filter)
+
+        if audio_file_path is None or segments is None:
+            logging.error("Process_Audio: Transcription failed or segments not available.")
+            return "Process_Audio: Transcription failed.", None, None, None, None, None
+
+        logging.debug(f"Process_Audio: Transcription audio_file: {audio_file_path}")
+        logging.debug(f"Process_Audio: Transcription segments: {segments}")
+
+        transcription_text = {'audio_file': audio_file_path, 'transcription': segments}
+        logging.debug(f"Process_Audio: Transcription text: {transcription_text}")
+
+        # Save segments to JSON
+        segments_json_path = save_segments_to_json(segments)
+
+        # Perform summarization
+        summary_text = None
+        if api_name:
+            if rolling_summarization is not None:
+                pass
+                # FIXME rolling summarization
+                # summary_text = rolling_summarize_function(
+                #     transcription_text,
+                #     detail=detail_level,
+                #     api_name=api_name,
+                #     api_key=api_key,
+                #     custom_prompt=custom_prompt_input,
+                #     chunk_by_words=chunk_text_by_words,
+                #     max_words=max_words,
+                #     chunk_by_sentences=chunk_text_by_sentences,
+                #     max_sentences=max_sentences,
+                #     chunk_by_paragraphs=chunk_text_by_paragraphs,
+                #     max_paragraphs=max_paragraphs,
+                #     chunk_by_tokens=chunk_text_by_tokens,
+                #     max_tokens=max_tokens
+                # )
+            else:
+                summary_text = perform_summarization(api_name, segments_json_path, custom_prompt_input, api_key)
+
+            if summary_text is None:
+                logging.error("Summary text is None. Check summarization function.")
+                summary_file_path = None
+        else:
+            summary_text = 'Summary not available'
+            summary_file_path = None
+
+        # Save transcription and summary
+        download_path = create_download_directory("Audio_Processing")
+        json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text,
+                                                                           download_path)
+
+        # Update function call to add_media_to_database so that it properly applies the title, author and file type
+        # Add to database
+        add_media_to_database(None, {'title': 'Audio File', 'author': 'Unknown'}, segments, summary_text, keywords,
+                              custom_prompt_input, whisper_model)
+
+        return transcription_text, summary_text, json_file_path, summary_file_path, None, None
+
+    except Exception as e:
+        logging.error(f"Error in process_audio: {str(e)}")
+        return str(e), None, None, None, None, None
+
+
+def process_single_audio(audio_file_path, whisper_model, api_name, api_key, keep_original,custom_keywords, source,
+                         custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                         use_multi_level_chunking, chunk_language):
+    progress = []
+    transcription = ""
+    summary = ""
+
+    def update_progress(message):
+        progress.append(message)
+        return "\n".join(progress)
+
+    try:
+        # Check file size before processing
+        file_size = os.path.getsize(audio_file_path)
+        if file_size > MAX_FILE_SIZE:
+            update_progress(f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f} MB. Skipping this file.")
+            return "\n".join(progress), "", ""
+
+        # Perform transcription
+        update_progress("Starting transcription...")
+        segments = speech_to_text(audio_file_path, whisper_model=whisper_model)
+        transcription = " ".join([segment['Text'] for segment in segments])
+        update_progress("Audio transcribed successfully.")
+
+        # Perform summarization if API is provided
+        if api_name and api_key:
+            update_progress("Starting summarization...")
+            summary = perform_summarization(api_name, transcription, "Summarize the following audio transcript",
+                                            api_key)
+            update_progress("Audio summarized successfully.")
+        else:
+            summary = "No summary available"
+
+        # Prepare keywords
+        keywords = "audio,transcription"
+        if custom_keywords:
+            keywords += f",{custom_keywords}"
+
+        # Add to database
+        add_media_with_keywords(
+            url=source,
+            title=os.path.basename(audio_file_path),
+            media_type='audio',
+            content=transcription,
+            keywords=keywords,
+            prompt="Summarize the following audio transcript",
+            summary=summary,
+            transcription_model=whisper_model,
+            author="Unknown",
+            ingestion_date=None  # This will use the current date
+        )
+        update_progress("Audio file added to database successfully.")
+
+        if not keep_original and source != "Uploaded File":
+            os.remove(audio_file_path)
+            update_progress(f"Temporary file {audio_file_path} removed.")
+        elif keep_original and source != "Uploaded File":
+            update_progress(f"Original audio file kept at: {audio_file_path}")
+
+    except Exception as e:
+        update_progress(f"Error processing {source}: {str(e)}")
+        transcription = f"Error: {str(e)}"
+        summary = "No summary due to error"
+
+    return "\n".join(progress), transcription, summary
+
+
+def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key, use_cookies, cookies, keep_original,
+                        custom_keywords, custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap,
+                        use_adaptive_chunking, use_multi_level_chunking, chunk_language, diarize):
+    progress = []
+    temp_files = []
+    all_transcriptions = []
+    all_summaries = []
+
+    def update_progress(message):
+        progress.append(message)
+        return "\n".join(progress)
+
+    def cleanup_files():
+        for file in temp_files:
+            try:
+                if os.path.exists(file):
+                    os.remove(file)
+                    update_progress(f"Temporary file {file} removed.")
+            except Exception as e:
+                update_progress(f"Failed to remove temporary file {file}: {str(e)}")
+
+    def reencode_mp3(mp3_file_path):
+        try:
+            reencoded_mp3_path = mp3_file_path.replace(".mp3", "_reencoded.mp3")
+            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, '-codec:a', 'libmp3lame', reencoded_mp3_path], check=True)
+            update_progress(f"Re-encoded {mp3_file_path} to {reencoded_mp3_path}.")
+            return reencoded_mp3_path
+        except subprocess.CalledProcessError as e:
+            update_progress(f"Error re-encoding {mp3_file_path}: {str(e)}")
+            raise
+
+    def convert_mp3_to_wav(mp3_file_path):
+        try:
+            wav_file_path = mp3_file_path.replace(".mp3", ".wav")
+            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, wav_file_path], check=True)
+            update_progress(f"Converted {mp3_file_path} to {wav_file_path}.")
+            return wav_file_path
+        except subprocess.CalledProcessError as e:
+            update_progress(f"Error converting {mp3_file_path} to WAV: {str(e)}")
+            raise
+
+    try:
+        # Check and set the ffmpeg command
+        global ffmpeg_cmd
+        if os.name == "nt":
+            logging.debug("Running on Windows")
+            ffmpeg_cmd = os.path.join(os.getcwd(), "Bin", "ffmpeg.exe")
+        else:
+            ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
+
+        # Ensure ffmpeg is accessible
+        if not os.path.exists(ffmpeg_cmd) and os.name == "nt":
+            raise FileNotFoundError(f"ffmpeg executable not found at path: {ffmpeg_cmd}")
+
+        # Define chunk options early to avoid undefined errors
+        chunk_options = {
+            'method': chunk_method,
+            'max_size': max_chunk_size,
+            'overlap': chunk_overlap,
+            'adaptive': use_adaptive_chunking,
+            'multi_level': use_multi_level_chunking,
+            'language': chunk_language
+        }
+
+        # Process multiple URLs
+        urls = [url.strip() for url in audio_urls.split('\n') if url.strip()]
+
+        for i, url in enumerate(urls):
+            update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}")
+
+            # Download and process audio file
+            audio_file_path = download_audio_file(url, use_cookies, cookies)
+            if not os.path.exists(audio_file_path):
+                update_progress(f"Downloaded file not found: {audio_file_path}")
+                continue
+
+            temp_files.append(audio_file_path)
+            update_progress("Audio file downloaded successfully.")
+
+            # Re-encode MP3 to fix potential issues
+            reencoded_mp3_path = reencode_mp3(audio_file_path)
+            if not os.path.exists(reencoded_mp3_path):
+                update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
+                continue
+
+            temp_files.append(reencoded_mp3_path)
+
+            # Convert re-encoded MP3 to WAV
+            wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
+            if not os.path.exists(wav_file_path):
+                update_progress(f"Converted WAV file not found: {wav_file_path}")
+                continue
+
+            temp_files.append(wav_file_path)
+
+            # Initialize transcription
+            transcription = ""
+
+            # Transcribe audio
+            if diarize:
+                segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
+            else:
+                segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
+
+            # Handle segments nested under 'segments' key
+            if isinstance(segments, dict) and 'segments' in segments:
+                segments = segments['segments']
+
+            if isinstance(segments, list):
+                transcription = " ".join([segment.get('Text', '') for segment in segments])
+                update_progress("Audio transcribed successfully.")
+            else:
+                update_progress("Unexpected segments format received from speech_to_text.")
+                logging.error(f"Unexpected segments format: {segments}")
+                continue
+
+            if not transcription.strip():
+                update_progress("Transcription is empty.")
+            else:
+                # Apply chunking
+                chunked_text = improved_chunking_process(transcription, chunk_options)
+
+                # Summarize
+                if api_name:
+                    try:
+                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                        update_progress("Audio summarized successfully.")
+                    except Exception as e:
+                        logging.error(f"Error during summarization: {str(e)}")
+                        summary = "Summary generation failed"
+                else:
+                    summary = "No summary available (API not provided)"
+
+                all_transcriptions.append(transcription)
+                all_summaries.append(summary)
+
+                # Add to database
+                add_media_with_keywords(
+                    url=url,
+                    title=os.path.basename(wav_file_path),
+                    media_type='audio',
+                    content=transcription,
+                    keywords=custom_keywords,
+                    prompt=custom_prompt_input,
+                    summary=summary,
+                    transcription_model=whisper_model,
+                    author="Unknown",
+                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
+                )
+                update_progress("Audio file processed and added to database.")
+
+        # Process uploaded file if provided
+        if audio_file:
+            if os.path.getsize(audio_file.name) > MAX_FILE_SIZE:
+                update_progress(
+                    f"Uploaded file size exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB. Skipping this file.")
+            else:
+                # Re-encode MP3 to fix potential issues
+                reencoded_mp3_path = reencode_mp3(audio_file.name)
+                if not os.path.exists(reencoded_mp3_path):
+                    update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
+                    return update_progress("Processing failed: Re-encoded file not found"), "", ""
+
+                temp_files.append(reencoded_mp3_path)
+
+                # Convert re-encoded MP3 to WAV
+                wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
+                if not os.path.exists(wav_file_path):
+                    update_progress(f"Converted WAV file not found: {wav_file_path}")
+                    return update_progress("Processing failed: Converted WAV file not found"), "", ""
+
+                temp_files.append(wav_file_path)
+
+                # Initialize transcription
+                transcription = ""
+
+                if diarize:
+                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
+                else:
+                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
+
+                # Handle segments nested under 'segments' key
+                if isinstance(segments, dict) and 'segments' in segments:
+                    segments = segments['segments']
+
+                if isinstance(segments, list):
+                    transcription = " ".join([segment.get('Text', '') for segment in segments])
+                else:
+                    update_progress("Unexpected segments format received from speech_to_text.")
+                    logging.error(f"Unexpected segments format: {segments}")
+
+                chunked_text = improved_chunking_process(transcription, chunk_options)
+
+                if api_name and api_key:
+                    try:
+                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                        update_progress("Audio summarized successfully.")
+                    except Exception as e:
+                        logging.error(f"Error during summarization: {str(e)}")
+                        summary = "Summary generation failed"
+                else:
+                    summary = "No summary available (API not provided)"
+
+                all_transcriptions.append(transcription)
+                all_summaries.append(summary)
+
+                add_media_with_keywords(
+                    url="Uploaded File",
+                    title=os.path.basename(wav_file_path),
+                    media_type='audio',
+                    content=transcription,
+                    keywords=custom_keywords,
+                    prompt=custom_prompt_input,
+                    summary=summary,
+                    transcription_model=whisper_model,
+                    author="Unknown",
+                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
+                )
+                update_progress("Uploaded file processed and added to database.")
+
+        # Final cleanup
+        if not keep_original:
+            cleanup_files()
+
+        final_progress = update_progress("All processing complete.")
+        final_transcriptions = "\n\n".join(all_transcriptions)
+        final_summaries = "\n\n".join(all_summaries)
+
+        return final_progress, final_transcriptions, final_summaries
+
+    except Exception as e:
+        logging.error(f"Error processing audio files: {str(e)}")
+        cleanup_files()
+        return update_progress(f"Processing failed: {str(e)}"), "", ""
+
+
+def download_youtube_audio(url):
+    try:
+        # Determine ffmpeg path based on the operating system.
+        ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg'
+
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Extract information about the video
+            with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
+                info_dict = ydl.extract_info(url, download=False)
+                sanitized_title = sanitize_filename(info_dict['title'])
+
+            # Setup the temporary filenames
+            temp_video_path = Path(temp_dir) / f"{sanitized_title}_temp.mp4"
+            temp_audio_path = Path(temp_dir) / f"{sanitized_title}.mp3"
+
+            # Initialize yt-dlp with options for downloading
+            ydl_opts = {
+                'format': 'bestaudio[ext=m4a]/best[height<=480]',  # Prefer best audio, or video up to 480p
+                'ffmpeg_location': ffmpeg_path,
+                'outtmpl': str(temp_video_path),
+                'noplaylist': True,
+                'quiet': True
+            }
+
+            # Execute yt-dlp to download the video/audio
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                ydl.download([url])
+
+            # Check if the file exists
+            if not temp_video_path.exists():
+                raise FileNotFoundError(f"Expected file was not found: {temp_video_path}")
+
+            # Use ffmpeg to extract audio
+            ffmpeg_command = [
+                ffmpeg_path,
+                '-i', str(temp_video_path),
+                '-vn',  # No video
+                '-acodec', 'libmp3lame',
+                '-b:a', '192k',
+                str(temp_audio_path)
+            ]
+            subprocess.run(ffmpeg_command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+            # Check if the audio file was created
+            if not temp_audio_path.exists():
+                raise FileNotFoundError(f"Expected audio file was not found: {temp_audio_path}")
+
+            # Create a persistent directory for the download if it doesn't exist
+            persistent_dir = Path("downloads")
+            persistent_dir.mkdir(exist_ok=True)
+
+            # Move the file from the temporary directory to the persistent directory
+            persistent_file_path = persistent_dir / f"{sanitized_title}.mp3"
+            os.replace(str(temp_audio_path), str(persistent_file_path))
+
+            # Add the file to the list of downloaded files
+            downloaded_files.append(str(persistent_file_path))
+
+            return str(persistent_file_path), f"Audio downloaded successfully: {sanitized_title}.mp3"
+    except Exception as e:
+        return None, f"Error downloading audio: {str(e)}"
+
+
+def process_podcast(url, title, author, keywords, custom_prompt, api_name, api_key, whisper_model,
+                    keep_original=False, enable_diarization=False, use_cookies=False, cookies=None,
+                    chunk_method=None, max_chunk_size=300, chunk_overlap=0, use_adaptive_chunking=False,
+                    use_multi_level_chunking=False, chunk_language='english'):
+    progress = []
+    error_message = ""
+    temp_files = []
+
+    def update_progress(message):
+        progress.append(message)
+        return "\n".join(progress)
+
+    def cleanup_files():
+        if not keep_original:
+            for file in temp_files:
+                try:
+                    if os.path.exists(file):
+                        os.remove(file)
+                        update_progress(f"Temporary file {file} removed.")
+                except Exception as e:
+                    update_progress(f"Failed to remove temporary file {file}: {str(e)}")
+
+    try:
+        # Download podcast
+        audio_file = download_audio_file(url, use_cookies, cookies)
+        temp_files.append(audio_file)
+        update_progress("Podcast downloaded successfully.")
+
+        # Extract metadata
+        metadata = extract_metadata(url)
+        title = title or metadata.get('title', 'Unknown Podcast')
+        author = author or metadata.get('uploader', 'Unknown Author')
+
+        # Format metadata for storage
+        metadata_text = f"""
+Metadata:
+Title: {title}
+Author: {author}
+Series: {metadata.get('series', 'N/A')}
+Episode: {metadata.get('episode', 'N/A')}
+Season: {metadata.get('season', 'N/A')}
+Upload Date: {metadata.get('upload_date', 'N/A')}
+Duration: {metadata.get('duration', 'N/A')} seconds
+Description: {metadata.get('description', 'N/A')}
+"""
+
+        # Update keywords
+        new_keywords = []
+        if metadata.get('series'):
+            new_keywords.append(f"series:{metadata['series']}")
+        if metadata.get('episode'):
+            new_keywords.append(f"episode:{metadata['episode']}")
+        if metadata.get('season'):
+            new_keywords.append(f"season:{metadata['season']}")
+
+        keywords = f"{keywords},{','.join(new_keywords)}" if keywords else ','.join(new_keywords)
+
+        update_progress(f"Metadata extracted - Title: {title}, Author: {author}, Keywords: {keywords}")
+
+        # Transcribe the podcast
+        try:
+            if enable_diarization:
+                segments = speech_to_text(audio_file, whisper_model=whisper_model, diarize=True)
+            else:
+                segments = speech_to_text(audio_file, whisper_model=whisper_model)
+            transcription = " ".join([segment['Text'] for segment in segments])
+            update_progress("Podcast transcribed successfully.")
+        except Exception as e:
+            error_message = f"Transcription failed: {str(e)}"
+            raise
+
+        # Apply chunking
+        chunk_options = {
+            'method': chunk_method,
+            'max_size': max_chunk_size,
+            'overlap': chunk_overlap,
+            'adaptive': use_adaptive_chunking,
+            'multi_level': use_multi_level_chunking,
+            'language': chunk_language
+        }
+        chunked_text = improved_chunking_process(transcription, chunk_options)
+
+        # Combine metadata and transcription
+        full_content = metadata_text + "\n\nTranscription:\n" + transcription
+
+        # Summarize if API is provided
+        summary = None
+        if api_name and api_key:
+            try:
+                summary = perform_summarization(api_name, chunked_text, custom_prompt, api_key)
+                update_progress("Podcast summarized successfully.")
+            except Exception as e:
+                error_message = f"Summarization failed: {str(e)}"
+                raise
+
+        # Add to database
+        try:
+            add_media_with_keywords(
+                url=url,
+                title=title,
+                media_type='podcast',
+                content=full_content,
+                keywords=keywords,
+                prompt=custom_prompt,
+                summary=summary or "No summary available",
+                transcription_model=whisper_model,
+                author=author,
+                ingestion_date=datetime.now().strftime('%Y-%m-%d')
+            )
+            update_progress("Podcast added to database successfully.")
+        except Exception as e:
+            error_message = f"Error adding podcast to database: {str(e)}"
+            raise
+
+        # Cleanup
+        cleanup_files()
+
+        return (update_progress("Processing complete."), full_content, summary or "No summary generated.",
+                title, author, keywords, error_message)
+
+    except Exception as e:
+        logging.error(f"Error processing podcast: {str(e)}")
+        cleanup_files()
+        return update_progress(f"Processing failed: {str(e)}"), "", "", "", "", "", str(e)
+
+
+#
+#
 #######################################################################################################################

App_Function_Libraries/Audio_Transcription_Lib.py

@@ -1,192 +1,192 @@
-# Audio_Transcription_Lib.py
-#########################################
-# Transcription Library
-# This library is used to perform transcription of audio files.
-# Currently, uses faster_whisper for transcription.
-#
-####
-import configparser
-####################
-# Function List
-#
-# 1. convert_to_wav(video_file_path, offset=0, overwrite=False)
-# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False)
-#
-####################
-#
-# Import necessary libraries to run solo for testing
-import gc
-import json
-import logging
-import os
-import sys
-import subprocess
-import time
-
-# DEBUG Imports
-#from memory_profiler import profile
-
-# Import Local
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-# Convert video .m4a into .wav using ffmpeg
-#   ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav"
-#       https://www.gyan.dev/ffmpeg/builds/
-#
-
-
-whisper_model_instance = None
-# Retrieve processing choice from the configuration file
-config = configparser.ConfigParser()
-config.read('config.txt')
-processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
-
-
-# FIXME: This is a temporary solution.
-# This doesn't clear older models, which means potentially a lot of memory is being used...
-def get_whisper_model(model_name, device):
-    global whisper_model_instance
-    if whisper_model_instance is None:
-        from faster_whisper import WhisperModel
-        logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
-        whisper_model_instance = WhisperModel(model_name, device=device)
-    return whisper_model_instance
-
-
-# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
-#DEBUG
-#@profile
-def convert_to_wav(video_file_path, offset=0, overwrite=False):
-    out_path = os.path.splitext(video_file_path)[0] + ".wav"
-
-    if os.path.exists(out_path) and not overwrite:
-        print(f"File '{out_path}' already exists. Skipping conversion.")
-        logging.info(f"Skipping conversion as file already exists: {out_path}")
-        return out_path
-    print("Starting conversion process of .m4a to .WAV")
-    out_path = os.path.splitext(video_file_path)[0] + ".wav"
-
-    try:
-        if os.name == "nt":
-            logging.debug("ffmpeg being ran on windows")
-
-            if sys.platform.startswith('win'):
-                ffmpeg_cmd = ".\\Bin\\ffmpeg.exe"
-                logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}")
-            else:
-                ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
-
-            command = [
-                ffmpeg_cmd,  # Assuming the working directory is correctly set where .\Bin exists
-                "-ss", "00:00:00",  # Start at the beginning of the video
-                "-i", video_file_path,
-                "-ar", "16000",  # Audio sample rate
-                "-ac", "1",  # Number of audio channels
-                "-c:a", "pcm_s16le",  # Audio codec
-                out_path
-            ]
-            try:
-                # Redirect stdin from null device to prevent ffmpeg from waiting for input
-                with open(os.devnull, 'rb') as null_file:
-                    result = subprocess.run(command, stdin=null_file, text=True, capture_output=True)
-                if result.returncode == 0:
-                    logging.info("FFmpeg executed successfully")
-                    logging.debug("FFmpeg output: %s", result.stdout)
-                else:
-                    logging.error("Error in running FFmpeg")
-                    logging.error("FFmpeg stderr: %s", result.stderr)
-                    raise RuntimeError(f"FFmpeg error: {result.stderr}")
-            except Exception as e:
-                logging.error("Error occurred - ffmpeg doesn't like windows")
-                raise RuntimeError("ffmpeg failed")
-        elif os.name == "posix":
-            os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
-        else:
-            raise RuntimeError("Unsupported operating system")
-        logging.info("Conversion to WAV completed: %s", out_path)
-    except subprocess.CalledProcessError as e:
-        logging.error("Error executing FFmpeg command: %s", str(e))
-        raise RuntimeError("Error converting video file to WAV")
-    except Exception as e:
-        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
-        return {"error": str(e)}
-    gc.collect()
-    return out_path
-
-
-# Transcribe .wav into .segments.json
-#DEBUG
-#@profile
-def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False):
-    global whisper_model_instance, processing_choice
-    logging.info('speech-to-text: Loading faster_whisper model: %s', whisper_model)
-
-    time_start = time.time()
-    if audio_file_path is None:
-        raise ValueError("speech-to-text: No audio file provided")
-    logging.info("speech-to-text: Audio file path: %s", audio_file_path)
-
-    try:
-        _, file_ending = os.path.splitext(audio_file_path)
-        out_file = audio_file_path.replace(file_ending, ".segments.json")
-        prettified_out_file = audio_file_path.replace(file_ending, ".segments_pretty.json")
-        if os.path.exists(out_file):
-            logging.info("speech-to-text: Segments file already exists: %s", out_file)
-            with open(out_file) as f:
-                global segments
-                segments = json.load(f)
-            return segments
-
-        logging.info('speech-to-text: Starting transcription...')
-        options = dict(language=selected_source_lang, beam_size=5, best_of=5, vad_filter=vad_filter)
-        transcribe_options = dict(task="transcribe", **options)
-        # use function and config at top of file
-        whisper_model_instance = get_whisper_model(whisper_model, processing_choice)
-        segments_raw, info = whisper_model_instance.transcribe(audio_file_path, **transcribe_options)
-
-        segments = []
-        for segment_chunk in segments_raw:
-            chunk = {
-                "Time_Start": segment_chunk.start,
-                "Time_End": segment_chunk.end,
-                "Text": segment_chunk.text
-            }
-            logging.debug("Segment: %s", chunk)
-            segments.append(chunk)
-
-        if segments:
-            segments[0]["Text"] = f"This text was transcribed using whisper model: {whisper_model}\n\n" + segments[0]["Text"]
-
-        if not segments:
-            raise RuntimeError("No transcription produced. The audio file may be invalid or empty.")
-        logging.info("speech-to-text: Transcription completed in %.2f seconds", time.time() - time_start)
-
-        # Save the segments to a JSON file - prettified and non-prettified
-        # FIXME so this is an optional flag to save either the prettified json file or the normal one
-        save_json = True
-        if save_json:
-            logging.info("speech-to-text: Saving segments to JSON file")
-            output_data = {'segments': segments}
-
-            logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file)
-            with open(prettified_out_file, 'w') as f:
-                json.dump(output_data, f, indent=2)
-
-            logging.info("speech-to-text: Saving JSON to %s", out_file)
-            with open(out_file, 'w') as f:
-                json.dump(output_data, f)
-
-        logging.debug(f"speech-to-text: returning {segments[:500]}")
-        gc.collect()
-        return segments
-
-    except Exception as e:
-        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
-        raise RuntimeError("speech-to-text: Error transcribing audio")
-
-#
-#
+# Audio_Transcription_Lib.py
+#########################################
+# Transcription Library
+# This library is used to perform transcription of audio files.
+# Currently, uses faster_whisper for transcription.
+#
+####
+import configparser
+####################
+# Function List
+#
+# 1. convert_to_wav(video_file_path, offset=0, overwrite=False)
+# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False)
+#
+####################
+#
+# Import necessary libraries to run solo for testing
+import gc
+import json
+import logging
+import os
+import sys
+import subprocess
+import time
+
+# DEBUG Imports
+#from memory_profiler import profile
+
+# Import Local
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+# Convert video .m4a into .wav using ffmpeg
+#   ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav"
+#       https://www.gyan.dev/ffmpeg/builds/
+#
+
+
+whisper_model_instance = None
+# Retrieve processing choice from the configuration file
+config = configparser.ConfigParser()
+config.read('config.txt')
+processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
+
+
+# FIXME: This is a temporary solution.
+# This doesn't clear older models, which means potentially a lot of memory is being used...
+def get_whisper_model(model_name, device):
+    global whisper_model_instance
+    if whisper_model_instance is None:
+        from faster_whisper import WhisperModel
+        logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
+        whisper_model_instance = WhisperModel(model_name, device=device)
+    return whisper_model_instance
+
+
+# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
+#DEBUG
+#@profile
+def convert_to_wav(video_file_path, offset=0, overwrite=False):
+    out_path = os.path.splitext(video_file_path)[0] + ".wav"
+
+    if os.path.exists(out_path) and not overwrite:
+        print(f"File '{out_path}' already exists. Skipping conversion.")
+        logging.info(f"Skipping conversion as file already exists: {out_path}")
+        return out_path
+    print("Starting conversion process of .m4a to .WAV")
+    out_path = os.path.splitext(video_file_path)[0] + ".wav"
+
+    try:
+        if os.name == "nt":
+            logging.debug("ffmpeg being ran on windows")
+
+            if sys.platform.startswith('win'):
+                ffmpeg_cmd = ".\\Bin\\ffmpeg.exe"
+                logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}")
+            else:
+                ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
+
+            command = [
+                ffmpeg_cmd,  # Assuming the working directory is correctly set where .\Bin exists
+                "-ss", "00:00:00",  # Start at the beginning of the video
+                "-i", video_file_path,
+                "-ar", "16000",  # Audio sample rate
+                "-ac", "1",  # Number of audio channels
+                "-c:a", "pcm_s16le",  # Audio codec
+                out_path
+            ]
+            try:
+                # Redirect stdin from null device to prevent ffmpeg from waiting for input
+                with open(os.devnull, 'rb') as null_file:
+                    result = subprocess.run(command, stdin=null_file, text=True, capture_output=True)
+                if result.returncode == 0:
+                    logging.info("FFmpeg executed successfully")
+                    logging.debug("FFmpeg output: %s", result.stdout)
+                else:
+                    logging.error("Error in running FFmpeg")
+                    logging.error("FFmpeg stderr: %s", result.stderr)
+                    raise RuntimeError(f"FFmpeg error: {result.stderr}")
+            except Exception as e:
+                logging.error("Error occurred - ffmpeg doesn't like windows")
+                raise RuntimeError("ffmpeg failed")
+        elif os.name == "posix":
+            os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
+        else:
+            raise RuntimeError("Unsupported operating system")
+        logging.info("Conversion to WAV completed: %s", out_path)
+    except subprocess.CalledProcessError as e:
+        logging.error("Error executing FFmpeg command: %s", str(e))
+        raise RuntimeError("Error converting video file to WAV")
+    except Exception as e:
+        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
+        return {"error": str(e)}
+    gc.collect()
+    return out_path
+
+
+# Transcribe .wav into .segments.json
+#DEBUG
+#@profile
+def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False):
+    global whisper_model_instance, processing_choice
+    logging.info('speech-to-text: Loading faster_whisper model: %s', whisper_model)
+
+    time_start = time.time()
+    if audio_file_path is None:
+        raise ValueError("speech-to-text: No audio file provided")
+    logging.info("speech-to-text: Audio file path: %s", audio_file_path)
+
+    try:
+        _, file_ending = os.path.splitext(audio_file_path)
+        out_file = audio_file_path.replace(file_ending, ".segments.json")
+        prettified_out_file = audio_file_path.replace(file_ending, ".segments_pretty.json")
+        if os.path.exists(out_file):
+            logging.info("speech-to-text: Segments file already exists: %s", out_file)
+            with open(out_file) as f:
+                global segments
+                segments = json.load(f)
+            return segments
+
+        logging.info('speech-to-text: Starting transcription...')
+        options = dict(language=selected_source_lang, beam_size=5, best_of=5, vad_filter=vad_filter)
+        transcribe_options = dict(task="transcribe", **options)
+        # use function and config at top of file
+        whisper_model_instance = get_whisper_model(whisper_model, processing_choice)
+        segments_raw, info = whisper_model_instance.transcribe(audio_file_path, **transcribe_options)
+
+        segments = []
+        for segment_chunk in segments_raw:
+            chunk = {
+                "Time_Start": segment_chunk.start,
+                "Time_End": segment_chunk.end,
+                "Text": segment_chunk.text
+            }
+            logging.debug("Segment: %s", chunk)
+            segments.append(chunk)
+
+        if segments:
+            segments[0]["Text"] = f"This text was transcribed using whisper model: {whisper_model}\n\n" + segments[0]["Text"]
+
+        if not segments:
+            raise RuntimeError("No transcription produced. The audio file may be invalid or empty.")
+        logging.info("speech-to-text: Transcription completed in %.2f seconds", time.time() - time_start)
+
+        # Save the segments to a JSON file - prettified and non-prettified
+        # FIXME so this is an optional flag to save either the prettified json file or the normal one
+        save_json = True
+        if save_json:
+            logging.info("speech-to-text: Saving segments to JSON file")
+            output_data = {'segments': segments}
+
+            logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file)
+            with open(prettified_out_file, 'w') as f:
+                json.dump(output_data, f, indent=2)
+
+            logging.info("speech-to-text: Saving JSON to %s", out_file)
+            with open(out_file, 'w') as f:
+                json.dump(output_data, f)
+
+        logging.debug(f"speech-to-text: returning {segments[:500]}")
+        gc.collect()
+        return segments
+
+    except Exception as e:
+        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
+        raise RuntimeError("speech-to-text: Error transcribing audio")
+
+#
+#
 #######################################################################################################################

App_Function_Libraries/Chat.py

@@ -0,0 +1,273 @@
+# Chat.py
+# Chat functions for interacting with the LLMs as chatbots
+
+# Imports
+import json
+import logging
+import os
+import re
+import tempfile
+from datetime import datetime
+
+from App_Function_Libraries.DB_Manager import get_conversation_name, save_chat_history_to_database
+from App_Function_Libraries.LLM_API_Calls import chat_with_openai, chat_with_anthropic, chat_with_cohere, \
+    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface, chat_with_vllm
+from App_Function_Libraries.LLM_API_Calls_Local import chat_with_aphrodite, chat_with_local_llm, chat_with_ollama, \
+    chat_with_kobold, chat_with_llama, chat_with_oobabooga, chat_with_tabbyapi
+from App_Function_Libraries.SQLite_DB import load_media_content
+from App_Function_Libraries.Utils import generate_unique_filename
+
+
+#
+# External Imports
+#
+# Local Imports
+#
+
+####################################################################################################
+def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt, temperature,
+         system_message=None):
+    try:
+        logging.info(f"Debug - Chat Function - Message: {message}")
+        logging.info(f"Debug - Chat Function - Media Content: {media_content}")
+        logging.info(f"Debug - Chat Function - Selected Parts: {selected_parts}")
+        logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
+        # logging.info(f"Debug - Chat Function - Prompt: {prompt}")
+
+        # Ensure selected_parts is a list
+        if not isinstance(selected_parts, (list, tuple)):
+            selected_parts = [selected_parts] if selected_parts else []
+
+        # logging.debug(f"Debug - Chat Function - Selected Parts (after check): {selected_parts}")
+
+        # Combine the selected parts of the media content
+        combined_content = "\n\n".join(
+            [f"{part.capitalize()}: {media_content.get(part, '')}" for part in selected_parts if part in media_content])
+        # Print first 500 chars
+        # logging.debug(f"Debug - Chat Function - Combined Content: {combined_content[:500]}...")
+
+        # Prepare the input for the API
+        if not history:
+            input_data = f"{combined_content}\n\nUser: {message}\n"
+        else:
+            input_data = f"User: {message}\n"
+        # Print first 500 chars
+        # logging.info(f"Debug - Chat Function - Input Data: {input_data[:500]}...")
+
+        if system_message:
+            print(f"System message: {system_message}")
+            logging.debug(f"Debug - Chat Function - System Message: {system_message}")
+        temperature = float(temperature) if temperature else 0.7
+        temp = temperature
+
+        logging.debug("Debug - Chat Function - Temperature: {temperature}")
+        logging.debug(f"Debug - Chat Function - API Key: {api_key[:10]}")
+        logging.debug(f"Debug - Chat Function - Prompt: {prompt}")
+
+        # Use the existing API request code based on the selected endpoint
+        logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
+        if api_endpoint.lower() == 'openai':
+            response = chat_with_openai(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "anthropic":
+            response = chat_with_anthropic(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "cohere":
+            response = chat_with_cohere(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "groq":
+            response = chat_with_groq(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "openrouter":
+            response = chat_with_openrouter(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "deepseek":
+            response = chat_with_deepseek(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "mistral":
+            response = chat_with_mistral(api_key, input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "llama.cpp":
+            response = chat_with_llama(input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "kobold":
+            response = chat_with_kobold(input_data, api_key, prompt, temp, system_message)
+        elif api_endpoint.lower() == "ooba":
+            response = chat_with_oobabooga(input_data, api_key, prompt, temp, system_message)
+        elif api_endpoint.lower() == "tabbyapi":
+            response = chat_with_tabbyapi(input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "vllm":
+            response = chat_with_vllm(input_data, prompt, system_message)
+        elif api_endpoint.lower() == "local-llm":
+            response = chat_with_local_llm(input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "huggingface":
+            response = chat_with_huggingface(api_key, input_data, prompt, temp)  # , system_message)
+        elif api_endpoint.lower() == "ollama":
+            response = chat_with_ollama(input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "aphrodite":
+            response = chat_with_aphrodite(input_data, prompt, temp, system_message)
+        else:
+            raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
+
+        return response
+
+    except Exception as e:
+        logging.error(f"Error in chat function: {str(e)}")
+        return f"An error occurred: {str(e)}"
+
+
+def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content):
+    logging.info(f"Attempting to save chat history. Media content type: {type(media_content)}")
+    try:
+        # Extract the media_id and media_name from the media_content
+        media_id = None
+        media_name = None
+        if isinstance(media_content, dict):
+            logging.debug(f"Media content keys: {media_content.keys()}")
+            if 'content' in media_content:
+                try:
+                    content = media_content['content']
+                    if isinstance(content, str):
+                        content_json = json.loads(content)
+                    elif isinstance(content, dict):
+                        content_json = content
+                    else:
+                        raise ValueError(f"Unexpected content type: {type(content)}")
+
+                    # Use the webpage_url as the media_id
+                    media_id = content_json.get('webpage_url')
+                    # Use the title as the media_name
+                    media_name = content_json.get('title')
+
+                    logging.info(f"Extracted media_id: {media_id}, media_name: {media_name}")
+                except json.JSONDecodeError:
+                    logging.error("Failed to decode JSON from media_content['content']")
+                except Exception as e:
+                    logging.error(f"Error processing media_content: {str(e)}")
+            else:
+                logging.warning("'content' key not found in media_content")
+        else:
+            logging.warning(f"media_content is not a dictionary. Type: {type(media_content)}")
+
+        if media_id is None:
+            # If we couldn't find a media_id, we'll use a placeholder
+            media_id = "unknown_media"
+            logging.warning(f"Unable to extract media_id from media_content. Using placeholder: {media_id}")
+
+        if media_name is None:
+            media_name = "Unnamed Media"
+            logging.warning(f"Unable to extract media_name from media_content. Using placeholder: {media_name}")
+
+        # Generate a unique conversation name using media_id and current timestamp
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        conversation_name = f"Chat_{media_id}_{timestamp}"
+
+        new_conversation_id = save_chat_history_to_database(chatbot, conversation_id, media_id, media_name,
+                                                            conversation_name)
+        return new_conversation_id, f"Chat history saved successfully as {conversation_name}!"
+    except Exception as e:
+        error_message = f"Failed to save chat history: {str(e)}"
+        logging.error(error_message, exc_info=True)
+        return conversation_id, error_message
+
+
+def save_chat_history(history, conversation_id, media_content):
+    try:
+        content, conversation_name = generate_chat_history_content(history, conversation_id, media_content)
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        safe_conversation_name = re.sub(r'[^a-zA-Z0-9_-]', '_', conversation_name)
+        base_filename = f"{safe_conversation_name}_{timestamp}.json"
+
+        # Create a temporary file
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as temp_file:
+            temp_file.write(content)
+            temp_file_path = temp_file.name
+
+        # Generate a unique filename
+        unique_filename = generate_unique_filename(os.path.dirname(temp_file_path), base_filename)
+        final_path = os.path.join(os.path.dirname(temp_file_path), unique_filename)
+
+        # Rename the temporary file to the unique filename
+        os.rename(temp_file_path, final_path)
+
+        return final_path
+    except Exception as e:
+        logging.error(f"Error saving chat history: {str(e)}")
+        return None
+
+
+def generate_chat_history_content(history, conversation_id, media_content):
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+    conversation_name = get_conversation_name(conversation_id)
+
+    if not conversation_name:
+        media_name = extract_media_name(media_content)
+        if media_name:
+            conversation_name = f"{media_name}-chat"
+        else:
+            conversation_name = f"chat-{timestamp}"  # Fallback name
+
+    chat_data = {
+        "conversation_id": conversation_id,
+        "conversation_name": conversation_name,
+        "timestamp": timestamp,
+        "history": [
+            {
+                "role": "user" if i % 2 == 0 else "bot",
+                "content": msg[0] if isinstance(msg, tuple) else msg
+            }
+            for i, msg in enumerate(history)
+        ]
+    }
+
+    return json.dumps(chat_data, indent=2), conversation_name
+
+
+def extract_media_name(media_content):
+    if isinstance(media_content, dict):
+        content = media_content.get('content', {})
+        if isinstance(content, str):
+            try:
+                content = json.loads(content)
+            except json.JSONDecodeError:
+                logging.warning("Failed to parse media_content JSON string")
+                return None
+
+        # Try to extract title from the content
+        if isinstance(content, dict):
+            return content.get('title') or content.get('name')
+
+    logging.warning(f"Unexpected media_content format: {type(media_content)}")
+    return None
+
+
+def update_chat_content(selected_item, use_content, use_summary, use_prompt, item_mapping):
+    logging.debug(f"Debug - Update Chat Content - Selected Item: {selected_item}\n")
+    logging.debug(f"Debug - Update Chat Content - Use Content: {use_content}\n\n\n\n")
+    logging.debug(f"Debug - Update Chat Content - Use Summary: {use_summary}\n\n")
+    logging.debug(f"Debug - Update Chat Content - Use Prompt: {use_prompt}\n\n")
+    logging.debug(f"Debug - Update Chat Content - Item Mapping: {item_mapping}\n\n")
+
+    if selected_item and selected_item in item_mapping:
+        media_id = item_mapping[selected_item]
+        content = load_media_content(media_id)
+        selected_parts = []
+        if use_content and "content" in content:
+            selected_parts.append("content")
+        if use_summary and "summary" in content:
+            selected_parts.append("summary")
+        if use_prompt and "prompt" in content:
+            selected_parts.append("prompt")
+
+        # Modified debug print
+        if isinstance(content, dict):
+            print(f"Debug - Update Chat Content - Content keys: {list(content.keys())}")
+            for key, value in content.items():
+                print(f"Debug - Update Chat Content - {key} (first 500 char): {str(value)[:500]}\n\n\n\n")
+        else:
+            print(f"Debug - Update Chat Content - Content(first 500 char): {str(content)[:500]}\n\n\n\n")
+
+        print(f"Debug - Update Chat Content - Selected Parts: {selected_parts}")
+        return content, selected_parts
+    else:
+        print(f"Debug - Update Chat Content - No item selected or item not in mapping")
+        return {}, []
+
+
+#
+# End of Chat.py
+##########################################################################################################################

App_Function_Libraries/Chat_related_functions.py

@@ -0,0 +1,41 @@
+# Chat_related_functions.py
+# Contains functions related to chat
+# WIP.
+#
+# Importing required libraries
+import json
+import os
+from pathlib import Path
+import json
+#
+########################################################################################################################
+# Set globals
+CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+
+def save_character(character_data):
+    if CHARACTERS_FILE.exists():
+        with CHARACTERS_FILE.open('r') as f:
+            characters = json.load(f)
+    else:
+        characters = {}
+
+    characters[character_data['name']] = character_data
+
+    with CHARACTERS_FILE.open('w') as f:
+        json.dump(characters, f, indent=2)
+
+
+def load_characters():
+    if os.path.exists(CHARACTERS_FILE):
+        with open(CHARACTERS_FILE, 'r') as f:
+            return json.load(f)
+    return {}
+
+
+def get_character_names():
+    characters = load_characters()
+    return list(characters.keys())
+
+
+
+

App_Function_Libraries/ChromaDB_Library.py

@@ -0,0 +1,225 @@
+import configparser
+import logging
+import sqlite3
+from typing import List, Dict, Any
+
+import chromadb
+import requests
+
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process
+
+#######################################################################################################################
+#
+# Functions for ChromaDB
+
+# Get ChromaDB settings
+# Load configuration
+config = configparser.ConfigParser()
+config.read('config.txt')
+chroma_db_path = config.get('Database', 'chroma_db_path', fallback='chroma_db')
+chroma_client = chromadb.PersistentClient(path=chroma_db_path)
+
+# Get embedding settings
+embedding_provider = config.get('Embeddings', 'provider', fallback='openai')
+embedding_model = config.get('Embeddings', 'model', fallback='text-embedding-3-small')
+embedding_api_key = config.get('Embeddings', 'api_key', fallback='')
+embedding_api_url = config.get('Embeddings', 'api_url', fallback='')
+
+# Get chunking options
+chunk_options = {
+    'method': config.get('Chunking', 'method', fallback='words'),
+    'max_size': config.getint('Chunking', 'max_size', fallback=400),
+    'overlap': config.getint('Chunking', 'overlap', fallback=200),
+    'adaptive': config.getboolean('Chunking', 'adaptive', fallback=False),
+    'multi_level': config.getboolean('Chunking', 'multi_level', fallback=False),
+    'language': config.get('Chunking', 'language', fallback='english')
+}
+
+
+def auto_update_chroma_embeddings(media_id: int, content: str):
+    """
+    Automatically update ChromaDB embeddings when a new item is ingested into the SQLite database.
+
+    :param media_id: The ID of the newly ingested media item
+    :param content: The content of the newly ingested media item
+    """
+    collection_name = f"media_{media_id}"
+
+    # Initialize or get the ChromaDB collection
+    collection = chroma_client.get_or_create_collection(name=collection_name)
+
+    # Check if embeddings already exist for this media_id
+    existing_embeddings = collection.get(ids=[f"{media_id}_chunk_{i}" for i in range(len(content))])
+
+    if existing_embeddings and len(existing_embeddings) > 0:
+        logging.info(f"Embeddings already exist for media ID {media_id}, skipping...")
+    else:
+        # Process and store content if embeddings do not already exist
+        process_and_store_content(content, collection_name, media_id)
+        logging.info(f"Updated ChromaDB embeddings for media ID: {media_id}")
+
+
+# Function to process content, create chunks, embeddings, and store in ChromaDB and SQLite
+def process_and_store_content(content: str, collection_name: str, media_id: int):
+    # Process the content into chunks
+    chunks = improved_chunking_process(content, chunk_options)
+    texts = [chunk['text'] for chunk in chunks]
+
+    # Generate embeddings for each chunk
+    embeddings = [create_embedding(text) for text in texts]
+
+    # Create unique IDs for each chunk using the media_id and chunk index
+    ids = [f"{media_id}_chunk_{i}" for i in range(len(texts))]
+
+    # Store the texts, embeddings, and IDs in ChromaDB
+    store_in_chroma(collection_name, texts, embeddings, ids)
+
+    # Store the chunks in SQLite FTS as well
+    from App_Function_Libraries.DB_Manager import db
+    with db.get_connection() as conn:
+        cursor = conn.cursor()
+        for text in texts:
+            cursor.execute("INSERT INTO media_fts (content) VALUES (?)", (text,))
+        conn.commit()
+
+
+# Function to store documents and their embeddings in ChromaDB
+def store_in_chroma(collection_name: str, texts: List[str], embeddings: List[List[float]], ids: List[str]):
+    collection = chroma_client.get_or_create_collection(name=collection_name)
+    collection.add(
+        documents=texts,
+        embeddings=embeddings,
+        ids=ids
+    )
+
+# Function to perform vector search using ChromaDB
+def vector_search(collection_name: str, query: str, k: int = 10) -> List[str]:
+    query_embedding = create_embedding(query)
+    collection = chroma_client.get_collection(name=collection_name)
+    results = collection.query(
+        query_embeddings=[query_embedding],
+        n_results=k
+    )
+    return results['documents'][0]
+
+
+def create_embedding(text: str) -> List[float]:
+    if embedding_provider == 'openai':
+        import openai
+        openai.api_key = embedding_api_key
+        response = openai.Embedding.create(input=text, model=embedding_model)
+        return response['data'][0]['embedding']
+    elif embedding_provider == 'local':
+        # FIXME - This is a placeholder for API calls to a local embedding model
+        response = requests.post(
+            embedding_api_url,
+            json={"text": text, "model": embedding_model},
+            headers={"Authorization": f"Bearer {embedding_api_key}"}
+        )
+        return response.json()['embedding']
+    # FIXME - this seems correct, but idk....
+    elif embedding_provider == 'huggingface':
+        from transformers import AutoTokenizer, AutoModel
+        import torch
+
+        tokenizer = AutoTokenizer.from_pretrained(embedding_model)
+        model = AutoModel.from_pretrained(embedding_model)
+
+        inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True, max_length=512)
+        with torch.no_grad():
+            outputs = model(**inputs)
+
+        # Use the mean of the last hidden state as the sentence embedding
+        embeddings = outputs.last_hidden_state.mean(dim=1)
+        return embeddings[0].tolist()  # Convert to list for consistency
+    else:
+        raise ValueError(f"Unsupported embedding provider: {embedding_provider}")
+
+
+def create_all_embeddings(api_choice: str) -> str:
+    try:
+        global embedding_provider
+        embedding_provider = api_choice
+
+        all_content = get_all_content_from_database()
+
+        if not all_content:
+            return "No content found in the database."
+
+        texts_to_embed = []
+        embeddings_to_store = []
+        ids_to_store = []
+        collection_name = "all_content_embeddings"
+
+        # Initialize or get the ChromaDB collection
+        collection = chroma_client.get_or_create_collection(name=collection_name)
+
+        for content_item in all_content:
+            media_id = content_item['id']
+            text = content_item['content']
+
+            # Check if the embedding already exists in ChromaDB
+            embedding_exists = collection.get(ids=[f"doc_{media_id}"])
+
+            if embedding_exists:
+                logging.info(f"Embedding already exists for media ID {media_id}, skipping...")
+                continue  # Skip if embedding already exists
+
+            # Create the embedding
+            embedding = create_embedding(text)
+
+            # Collect the text, embedding, and ID for batch storage
+            texts_to_embed.append(text)
+            embeddings_to_store.append(embedding)
+            ids_to_store.append(f"doc_{media_id}")
+
+        # Store all new embeddings in ChromaDB
+        if texts_to_embed and embeddings_to_store:
+            store_in_chroma(collection_name, texts_to_embed, embeddings_to_store, ids_to_store)
+
+        return "Embeddings created and stored successfully for all new content."
+    except Exception as e:
+        logging.error(f"Error during embedding creation: {str(e)}")
+        return f"Error: {str(e)}"
+
+
+def get_all_content_from_database() -> List[Dict[str, Any]]:
+    """
+    Retrieve all media content from the database that requires embedding.
+
+    Returns:
+        List[Dict[str, Any]]: A list of dictionaries, each containing the media ID, content, title, and other relevant fields.
+    """
+    try:
+        from App_Function_Libraries.DB_Manager import db
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT id, content, title, author, type
+                FROM Media
+                WHERE is_trash = 0  -- Exclude items marked as trash
+            """)
+            media_items = cursor.fetchall()
+
+            # Convert the results into a list of dictionaries
+            all_content = [
+                {
+                    'id': item[0],
+                    'content': item[1],
+                    'title': item[2],
+                    'author': item[3],
+                    'type': item[4]
+                }
+                for item in media_items
+            ]
+
+        return all_content
+
+    except sqlite3.Error as e:
+        logging.error(f"Error retrieving all content from database: {e}")
+        from App_Function_Libraries.SQLite_DB import DatabaseError
+        raise DatabaseError(f"Error retrieving all content from database: {e}")
+
+#
+# End of Functions for ChromaDB
+#######################################################################################################################

App_Function_Libraries/Chunk_Lib.py

@@ -1,583 +1,587 @@
-# Chunk_Lib.py
-#########################################
-# Chunking Library
-# This library is used to perform chunking of input files.
-# Currently, uses naive approaches. Nothing fancy.
-#
-####
-# Import necessary libraries
-import logging
-import re
-
-from typing import List, Optional, Tuple, Dict, Any
-
-from openai import OpenAI
-from tqdm import tqdm
-#
-# Import 3rd party
-from transformers import GPT2Tokenizer
-import nltk
-from nltk.tokenize import sent_tokenize, word_tokenize
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-#
-# Import Local
-from App_Function_Libraries.Tokenization_Methods_Lib import openai_tokenize
-from App_Function_Libraries.Utils import load_comprehensive_config
-
-
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-# FIXME - Make sure it only downloads if it already exists, and does a check first.
-# Ensure NLTK data is downloaded
-def ntlk_prep():
-    nltk.download('punkt')
-
-# Load GPT2 tokenizer
-tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-
-# Load Config file for API keys
-config = load_comprehensive_config()
-openai_api_key = config.get('API', 'openai_api_key', fallback=None)
-
-def load_document(file_path):
-    with open(file_path, 'r') as file:
-        text = file.read()
-    return re.sub('\\s+', ' ', text).strip()
-
-
-def improved_chunking_process(text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]:
-    chunk_method = chunk_options.get('method', 'words')
-    max_chunk_size = chunk_options.get('max_size', 300)
-    overlap = chunk_options.get('overlap', 0)
-    language = chunk_options.get('language', 'english')
-    adaptive = chunk_options.get('adaptive', False)
-    multi_level = chunk_options.get('multi_level', False)
-
-    if adaptive:
-        max_chunk_size = adaptive_chunk_size(text, max_chunk_size)
-
-    if multi_level:
-        chunks = multi_level_chunking(text, chunk_method, max_chunk_size, overlap, language)
-    else:
-        if chunk_method == 'words':
-            chunks = chunk_text_by_words(text, max_chunk_size, overlap)
-        elif chunk_method == 'sentences':
-            chunks = chunk_text_by_sentences(text, max_chunk_size, overlap, language)
-        elif chunk_method == 'paragraphs':
-            chunks = chunk_text_by_paragraphs(text, max_chunk_size, overlap)
-        elif chunk_method == 'tokens':
-            chunks = chunk_text_by_tokens(text, max_chunk_size, overlap)
-        elif chunk_method == 'chapters':
-            return chunk_ebook_by_chapters(text, chunk_options)
-        else:
-            # No chunking applied
-            chunks = [text]
-
-    return [{'text': chunk, 'metadata': get_chunk_metadata(chunk, text)} for chunk in chunks]
-
-
-def adaptive_chunk_size(text: str, base_size: int) -> int:
-    # Simple adaptive logic: adjust chunk size based on text complexity
-    avg_word_length = sum(len(word) for word in text.split()) / len(text.split())
-    if avg_word_length > 6:  # Arbitrary threshold for "complex" text
-        return int(base_size * 0.8)  # Reduce chunk size for complex text
-    return base_size
-
-
-def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]:
-    # First level: chunk by paragraphs
-    paragraphs = chunk_text_by_paragraphs(text, max_size * 2, overlap)
-
-    # Second level: chunk each paragraph further
-    chunks = []
-    for para in paragraphs:
-        if method == 'words':
-            chunks.extend(chunk_text_by_words(para, max_size, overlap))
-        elif method == 'sentences':
-            chunks.extend(chunk_text_by_sentences(para, max_size, overlap, language))
-        else:
-            chunks.append(para)
-
-    return chunks
-
-
-def chunk_text_by_words(text: str, max_words: int = 300, overlap: int = 0) -> List[str]:
-    words = text.split()
-    chunks = []
-    for i in range(0, len(words), max_words - overlap):
-        chunk = ' '.join(words[i:i + max_words])
-        chunks.append(chunk)
-    return post_process_chunks(chunks)
-
-
-def chunk_text_by_sentences(text: str, max_sentences: int = 10, overlap: int = 0, language: str = 'english') -> List[
-    str]:
-    nltk.download('punkt', quiet=True)
-    sentences = nltk.sent_tokenize(text, language=language)
-    chunks = []
-    for i in range(0, len(sentences), max_sentences - overlap):
-        chunk = ' '.join(sentences[i:i + max_sentences])
-        chunks.append(chunk)
-    return post_process_chunks(chunks)
-
-
-def chunk_text_by_paragraphs(text: str, max_paragraphs: int = 5, overlap: int = 0) -> List[str]:
-    paragraphs = re.split(r'\n\s*\n', text)
-    chunks = []
-    for i in range(0, len(paragraphs), max_paragraphs - overlap):
-        chunk = '\n\n'.join(paragraphs[i:i + max_paragraphs])
-        chunks.append(chunk)
-    return post_process_chunks(chunks)
-
-
-def chunk_text_by_tokens(text: str, max_tokens: int = 1000, overlap: int = 0) -> List[str]:
-    # This is a simplified token-based chunking. For more accurate tokenization,
-    # consider using a proper tokenizer like GPT-2 TokenizerFast
-    words = text.split()
-    chunks = []
-    current_chunk = []
-    current_token_count = 0
-
-    for word in words:
-        word_token_count = len(word) // 4 + 1  # Rough estimate of token count
-        if current_token_count + word_token_count > max_tokens and current_chunk:
-            chunks.append(' '.join(current_chunk))
-            current_chunk = current_chunk[-overlap:] if overlap > 0 else []
-            current_token_count = sum(len(w) // 4 + 1 for w in current_chunk)
-
-        current_chunk.append(word)
-        current_token_count += word_token_count
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return post_process_chunks(chunks)
-
-
-def post_process_chunks(chunks: List[str]) -> List[str]:
-    return [chunk.strip() for chunk in chunks if chunk.strip()]
-
-
-def get_chunk_metadata(chunk: str, full_text: str, chunk_type: str = "generic", chapter_number: Optional[int] = None, chapter_pattern: Optional[str] = None) -> Dict[str, Any]:
-    start_index = full_text.index(chunk)
-    metadata = {
-        'start_index': start_index,
-        'end_index': start_index + len(chunk),
-        'word_count': len(chunk.split()),
-        'char_count': len(chunk),
-        'chunk_type': chunk_type
-    }
-    if chunk_type == "chapter":
-        metadata['chapter_number'] = chapter_number
-        metadata['chapter_pattern'] = chapter_pattern
-    return metadata
-
-
-# Hybrid approach, chunk each sentence while ensuring total token size does not exceed a maximum number
-def chunk_text_hybrid(text, max_tokens=1000):
-    sentences = nltk.tokenize.sent_tokenize(text)
-    chunks = []
-    current_chunk = []
-    current_length = 0
-
-    for sentence in sentences:
-        tokens = tokenizer.encode(sentence)
-        if current_length + len(tokens) <= max_tokens:
-            current_chunk.append(sentence)
-            current_length += len(tokens)
-        else:
-            chunks.append(' '.join(current_chunk))
-            current_chunk = [sentence]
-            current_length = len(tokens)
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return chunks
-
-# Thanks openai
-def chunk_on_delimiter(input_string: str,
-                       max_tokens: int,
-                       delimiter: str) -> List[str]:
-    chunks = input_string.split(delimiter)
-    combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum(
-        chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True)
-    if dropped_chunk_count > 0:
-        print(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.")
-    combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks]
-    return combined_chunks
-
-# ????FIXME
-def recursive_summarize_chunks(chunks, summarize_func, custom_prompt, temp=None, system_prompt=None):
-    summarized_chunks = []
-    current_summary = ""
-
-    logging.debug(f"recursive_summarize_chunks: Summarizing {len(chunks)} chunks recursively...")
-    logging.debug(f"recursive_summarize_chunks:  temperature is @ {temp}")
-    for i, chunk in enumerate(chunks):
-        if i == 0:
-            current_summary = summarize_func(chunk, custom_prompt, temp, system_prompt)
-        else:
-            combined_text = current_summary + "\n\n" + chunk
-            current_summary = summarize_func(combined_text, custom_prompt, temp, system_prompt)
-
-        summarized_chunks.append(current_summary)
-
-    return summarized_chunks
-
-
-# Sample text for testing
-sample_text = """
-Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence 
-concerned with the interactions between computers and human language, in particular how to program computers 
-to process and analyze large amounts of natural language data. The result is a computer capable of "understanding" 
-the contents of documents, including the contextual nuances of the language within them. The technology can then 
-accurately extract information and insights contained in the documents as well as categorize and organize the documents themselves.
-
-Challenges in natural language processing frequently involve speech recognition, natural language understanding, 
-and natural language generation.
-
-Natural language processing has its roots in the 1950s. Already in 1950, Alan Turing published an article titled 
-"Computing Machinery and Intelligence" which proposed what is now called the Turing test as a criterion of intelligence.
-"""
-
-# Example usage of different chunking methods
-# print("Chunking by words:")
-# print(chunk_text_by_words(sample_text, max_words=50))
-#
-# print("\nChunking by sentences:")
-# print(chunk_text_by_sentences(sample_text, max_sentences=2))
-#
-# print("\nChunking by paragraphs:")
-# print(chunk_text_by_paragraphs(sample_text, max_paragraphs=1))
-#
-# print("\nChunking by tokens:")
-# print(chunk_text_by_tokens(sample_text, max_tokens=50))
-#
-# print("\nHybrid chunking:")
-# print(chunk_text_hybrid(sample_text, max_tokens=50))
-
-
-
-#######################################################################################################################
-#
-# Experimental Semantic Chunking
-#
-
-# Chunk text into segments based on semantic similarity
-def count_units(text, unit='tokens'):
-    if unit == 'words':
-        return len(text.split())
-    elif unit == 'tokens':
-        return len(word_tokenize(text))
-    elif unit == 'characters':
-        return len(text)
-    else:
-        raise ValueError("Invalid unit. Choose 'words', 'tokens', or 'characters'.")
-
-
-def semantic_chunking(text, max_chunk_size=2000, unit='words'):
-    nltk.download('punkt', quiet=True)
-    sentences = sent_tokenize(text)
-    vectorizer = TfidfVectorizer()
-    sentence_vectors = vectorizer.fit_transform(sentences)
-
-    chunks = []
-    current_chunk = []
-    current_size = 0
-
-    for i, sentence in enumerate(sentences):
-        sentence_size = count_units(sentence, unit)
-        if current_size + sentence_size > max_chunk_size and current_chunk:
-            chunks.append(' '.join(current_chunk))
-            overlap_size = count_units(' '.join(current_chunk[-3:]), unit)  # Use last 3 sentences for overlap
-            current_chunk = current_chunk[-3:]  # Keep last 3 sentences for overlap
-            current_size = overlap_size
-
-        current_chunk.append(sentence)
-        current_size += sentence_size
-
-        if i + 1 < len(sentences):
-            current_vector = sentence_vectors[i]
-            next_vector = sentence_vectors[i + 1]
-            similarity = cosine_similarity(current_vector, next_vector)[0][0]
-            if similarity < 0.5 and current_size >= max_chunk_size // 2:
-                chunks.append(' '.join(current_chunk))
-                overlap_size = count_units(' '.join(current_chunk[-3:]), unit)
-                current_chunk = current_chunk[-3:]
-                current_size = overlap_size
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return chunks
-
-
-def semantic_chunk_long_file(file_path, max_chunk_size=1000, overlap=100):
-    try:
-        with open(file_path, 'r', encoding='utf-8') as file:
-            content = file.read()
-
-        chunks = semantic_chunking(content, max_chunk_size, overlap)
-        return chunks
-    except Exception as e:
-        logging.error(f"Error chunking text file: {str(e)}")
-        return None
-#######################################################################################################################
-
-
-
-
-
-
-#######################################################################################################################
-#
-# OpenAI Rolling Summarization
-#
-
-client = OpenAI(api_key=openai_api_key)
-def get_chat_completion(messages, model='gpt-4-turbo'):
-    response = client.chat.completions.create(
-        model=model,
-        messages=messages,
-        temperature=0,
-    )
-    return response.choices[0].message.content
-
-
-# This function combines text chunks into larger blocks without exceeding a specified token count.
-#   It returns the combined chunks, their original indices, and the number of dropped chunks due to overflow.
-def combine_chunks_with_no_minimum(
-        chunks: List[str],
-        max_tokens: int,
-        chunk_delimiter="\n\n",
-        header: Optional[str] = None,
-        add_ellipsis_for_overflow=False,
-) -> Tuple[List[str], List[int]]:
-    dropped_chunk_count = 0
-    output = []  # list to hold the final combined chunks
-    output_indices = []  # list to hold the indices of the final combined chunks
-    candidate = (
-        [] if header is None else [header]
-    )  # list to hold the current combined chunk candidate
-    candidate_indices = []
-    for chunk_i, chunk in enumerate(chunks):
-        chunk_with_header = [chunk] if header is None else [header, chunk]
-        # FIXME MAKE NOT OPENAI SPECIFIC
-        if len(openai_tokenize(chunk_delimiter.join(chunk_with_header))) > max_tokens:
-            print(f"warning: chunk overflow")
-            if (
-                    add_ellipsis_for_overflow
-                    # FIXME MAKE NOT OPENAI SPECIFIC
-                    and len(openai_tokenize(chunk_delimiter.join(candidate + ["..."]))) <= max_tokens
-            ):
-                candidate.append("...")
-                dropped_chunk_count += 1
-            continue  # this case would break downstream assumptions
-        # estimate token count with the current chunk added
-        # FIXME MAKE NOT OPENAI SPECIFIC
-        extended_candidate_token_count = len(openai_tokenize(chunk_delimiter.join(candidate + [chunk])))
-        # If the token count exceeds max_tokens, add the current candidate to output and start a new candidate
-        if extended_candidate_token_count > max_tokens:
-            output.append(chunk_delimiter.join(candidate))
-            output_indices.append(candidate_indices)
-            candidate = chunk_with_header  # re-initialize candidate
-            candidate_indices = [chunk_i]
-        # otherwise keep extending the candidate
-        else:
-            candidate.append(chunk)
-            candidate_indices.append(chunk_i)
-    # add the remaining candidate to output if it's not empty
-    if (header is not None and len(candidate) > 1) or (header is None and len(candidate) > 0):
-        output.append(chunk_delimiter.join(candidate))
-        output_indices.append(candidate_indices)
-    return output, output_indices, dropped_chunk_count
-
-
-def rolling_summarize(text: str,
-                      detail: float = 0,
-                      model: str = 'gpt-4-turbo',
-                      additional_instructions: Optional[str] = None,
-                      minimum_chunk_size: Optional[int] = 500,
-                      chunk_delimiter: str = ".",
-                      summarize_recursively=False,
-                      verbose=False):
-    """
-    Summarizes a given text by splitting it into chunks, each of which is summarized individually.
-    The level of detail in the summary can be adjusted, and the process can optionally be made recursive.
-
-    Parameters:
-        - text (str): The text to be summarized.
-        - detail (float, optional): A value between 0 and 1
-            indicating the desired level of detail in the summary. 0 leads to a higher level summary, and 1 results in a more
-            detailed summary. Defaults to 0.
-        - additional_instructions (Optional[str], optional): Additional instructions to provide to the
-            model for customizing summaries. - minimum_chunk_size (Optional[int], optional): The minimum size for text
-            chunks. Defaults to 500.
-        - chunk_delimiter (str, optional): The delimiter used to split the text into chunks. Defaults to ".".
-        - summarize_recursively (bool, optional): If True, summaries are generated recursively, using previous summaries for context.
-        - verbose (bool, optional): If True, prints detailed information about the chunking process.
-    Returns:
-    - str: The final compiled summary of the text.
-
-    The function first determines the number of chunks by interpolating between a minimum and a maximum chunk count
-    based on the `detail` parameter. It then splits the text into chunks and summarizes each chunk. If
-    `summarize_recursively` is True, each summary is based on the previous summaries, adding more context to the
-    summarization process. The function returns a compiled summary of all chunks.
-    """
-
-    # check detail is set correctly
-    assert 0 <= detail <= 1
-
-    # interpolate the number of chunks based to get specified level of detail
-    max_chunks = len(chunk_on_delimiter(text, minimum_chunk_size, chunk_delimiter))
-    min_chunks = 1
-    num_chunks = int(min_chunks + detail * (max_chunks - min_chunks))
-
-    # adjust chunk_size based on interpolated number of chunks
-    # FIXME MAKE NOT OPENAI SPECIFIC
-    document_length = len(openai_tokenize(text))
-    chunk_size = max(minimum_chunk_size, document_length // num_chunks)
-    text_chunks = chunk_on_delimiter(text, chunk_size, chunk_delimiter)
-    if verbose:
-        print(f"Splitting the text into {len(text_chunks)} chunks to be summarized.")
-        # FIXME MAKE NOT OPENAI SPECIFIC
-        print(f"Chunk lengths are {[len(openai_tokenize(x)) for x in text_chunks]}")
-
-    # set system message - FIXME
-    system_message_content = "Rewrite this text in summarized form."
-    if additional_instructions is not None:
-        system_message_content += f"\n\n{additional_instructions}"
-
-    accumulated_summaries = []
-    for i, chunk in enumerate(tqdm(text_chunks)):
-        if summarize_recursively and accumulated_summaries:
-            # Combine previous summary with current chunk for recursive summarization
-            combined_text = accumulated_summaries[-1] + "\n\n" + chunk
-            user_message_content = f"Previous summary and new content to summarize:\n\n{combined_text}"
-        else:
-            user_message_content = chunk
-
-        messages = [
-            {"role": "system", "content": system_message_content},
-            {"role": "user", "content": user_message_content}
-        ]
-
-        response = get_chat_completion(messages, model=model)
-        accumulated_summaries.append(response)
-
-    final_summary = '\n\n'.join(accumulated_summaries)
-    return final_summary
-
-#
-#
-#######################################################################################################################
-#
-# Ebook Chapter Chunking
-
-
-def chunk_ebook_by_chapters(text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]:
-    max_chunk_size = chunk_options.get('max_size', 300)
-    overlap = chunk_options.get('overlap', 0)
-    custom_pattern = chunk_options.get('custom_chapter_pattern', None)
-
-    # List of chapter heading patterns to try, in order
-    chapter_patterns = [
-        custom_pattern,
-        r'^#{1,2}\s+',  # Markdown style: '# ' or '## '
-        r'^Chapter\s+\d+',  # 'Chapter ' followed by numbers
-        r'^\d+\.\s+',  # Numbered chapters: '1. ', '2. ', etc.
-        r'^[A-Z\s]+$'  # All caps headings
-    ]
-
-    chapter_positions = []
-    used_pattern = None
-
-    for pattern in chapter_patterns:
-        if pattern is None:
-            continue
-        chapter_regex = re.compile(pattern, re.MULTILINE | re.IGNORECASE)
-        chapter_positions = [match.start() for match in chapter_regex.finditer(text)]
-        if chapter_positions:
-            used_pattern = pattern
-            break
-
-    # If no chapters found, return the entire content as one chunk
-    if not chapter_positions:
-        return [{'text': text, 'metadata': get_chunk_metadata(text, text, chunk_type="whole_document")}]
-
-    # Split content into chapters
-    chunks = []
-    for i in range(len(chapter_positions)):
-        start = chapter_positions[i]
-        end = chapter_positions[i + 1] if i + 1 < len(chapter_positions) else None
-        chapter = text[start:end]
-
-        # Apply overlap if specified
-        if overlap > 0 and i > 0:
-            overlap_start = max(0, start - overlap)
-            chapter = text[overlap_start:end]
-
-        chunks.append(chapter)
-
-    # Post-process chunks
-    processed_chunks = post_process_chunks(chunks)
-
-    # Add metadata to chunks
-    return [{'text': chunk, 'metadata': get_chunk_metadata(chunk, text, chunk_type="chapter", chapter_number=i + 1,
-                                                           chapter_pattern=used_pattern)}
-            for i, chunk in enumerate(processed_chunks)]
-
-
-# # Example usage
-# if __name__ == "__main__":
-#     sample_ebook_content = """
-# # Chapter 1: Introduction
-#
-# This is the introduction.
-#
-# ## Section 1.1
-#
-# Some content here.
-#
-# # Chapter 2: Main Content
-#
-# This is the main content.
-#
-# ## Section 2.1
-#
-# More content here.
-#
-# CHAPTER THREE
-#
-# This is the third chapter.
-#
-# 4. Fourth Chapter
-#
-# This is the fourth chapter.
-# """
-#
-#     chunk_options = {
-#         'method': 'chapters',
-#         'max_size': 500,
-#         'overlap': 50,
-#         'custom_chapter_pattern': r'^CHAPTER\s+[A-Z]+'  # Custom pattern for 'CHAPTER THREE' style
-#     }
-#
-#     chunked_chapters = improved_chunking_process(sample_ebook_content, chunk_options)
-#
-#     for i, chunk in enumerate(chunked_chapters, 1):
-#         print(f"Chunk {i}:")
-#         print(chunk['text'])
-#         print(f"Metadata: {chunk['metadata']}\n")
-
-
-
-
-#
-# End of Chunking Library
+# Chunk_Lib.py
+#########################################
+# Chunking Library
+# This library is used to perform chunking of input files.
+# Currently, uses naive approaches. Nothing fancy.
+#
+####
+# Import necessary libraries
+import logging
+import re
+
+from typing import List, Optional, Tuple, Dict, Any
+
+from openai import OpenAI
+from tqdm import tqdm
+#
+# Import 3rd party
+from transformers import GPT2Tokenizer
+import nltk
+from nltk.tokenize import sent_tokenize, word_tokenize
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.metrics.pairwise import cosine_similarity
+#
+# Import Local
+from App_Function_Libraries.Tokenization_Methods_Lib import openai_tokenize
+from App_Function_Libraries.Utils import load_comprehensive_config
+
+
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+# FIXME - Make sure it only downloads if it already exists, and does a check first.
+# Ensure NLTK data is downloaded
+def ntlk_prep():
+    nltk.download('punkt')
+
+# Load GPT2 tokenizer
+tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+
+# Load Config file for API keys
+config = load_comprehensive_config()
+openai_api_key = config.get('API', 'openai_api_key', fallback=None)
+
+def load_document(file_path):
+    with open(file_path, 'r') as file:
+        text = file.read()
+    return re.sub('\\s+', ' ', text).strip()
+
+
+def improved_chunking_process(text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]:
+    chunk_method = chunk_options.get('method', 'words')
+    max_chunk_size = chunk_options.get('max_size', 300)
+    overlap = chunk_options.get('overlap', 0)
+    language = chunk_options.get('language', 'english')
+    adaptive = chunk_options.get('adaptive', False)
+    multi_level = chunk_options.get('multi_level', False)
+
+    if adaptive:
+        max_chunk_size = adaptive_chunk_size(text, max_chunk_size)
+
+    if multi_level:
+        chunks = multi_level_chunking(text, chunk_method, max_chunk_size, overlap, language)
+    else:
+        if chunk_method == 'words':
+            chunks = chunk_text_by_words(text, max_chunk_size, overlap)
+        elif chunk_method == 'sentences':
+            chunks = chunk_text_by_sentences(text, max_chunk_size, overlap, language)
+        elif chunk_method == 'paragraphs':
+            chunks = chunk_text_by_paragraphs(text, max_chunk_size, overlap)
+        elif chunk_method == 'tokens':
+            chunks = chunk_text_by_tokens(text, max_chunk_size, overlap)
+        elif chunk_method == 'chapters':
+            return chunk_ebook_by_chapters(text, chunk_options)
+        else:
+            # No chunking applied
+            chunks = [text]
+
+    return [{'text': chunk, 'metadata': get_chunk_metadata(chunk, text)} for chunk in chunks]
+
+
+def adaptive_chunk_size(text: str, base_size: int) -> int:
+    # Simple adaptive logic: adjust chunk size based on text complexity
+    avg_word_length = sum(len(word) for word in text.split()) / len(text.split())
+    if avg_word_length > 6:  # Arbitrary threshold for "complex" text
+        return int(base_size * 0.8)  # Reduce chunk size for complex text
+    return base_size
+
+
+def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]:
+    # First level: chunk by paragraphs
+    paragraphs = chunk_text_by_paragraphs(text, max_size * 2, overlap)
+
+    # Second level: chunk each paragraph further
+    chunks = []
+    for para in paragraphs:
+        if method == 'words':
+            chunks.extend(chunk_text_by_words(para, max_size, overlap))
+        elif method == 'sentences':
+            chunks.extend(chunk_text_by_sentences(para, max_size, overlap, language))
+        else:
+            chunks.append(para)
+
+    return chunks
+
+
+def chunk_text_by_words(text: str, max_words: int = 300, overlap: int = 0) -> List[str]:
+    words = text.split()
+    chunks = []
+    for i in range(0, len(words), max_words - overlap):
+        chunk = ' '.join(words[i:i + max_words])
+        chunks.append(chunk)
+    return post_process_chunks(chunks)
+
+
+def chunk_text_by_sentences(text: str, max_sentences: int = 10, overlap: int = 0, language: str = 'english') -> List[
+    str]:
+    nltk.download('punkt', quiet=True)
+    sentences = nltk.sent_tokenize(text, language=language)
+    chunks = []
+    for i in range(0, len(sentences), max_sentences - overlap):
+        chunk = ' '.join(sentences[i:i + max_sentences])
+        chunks.append(chunk)
+    return post_process_chunks(chunks)
+
+
+def chunk_text_by_paragraphs(text: str, max_paragraphs: int = 5, overlap: int = 0) -> List[str]:
+    paragraphs = re.split(r'\n\s*\n', text)
+    chunks = []
+    for i in range(0, len(paragraphs), max_paragraphs - overlap):
+        chunk = '\n\n'.join(paragraphs[i:i + max_paragraphs])
+        chunks.append(chunk)
+    return post_process_chunks(chunks)
+
+
+def chunk_text_by_tokens(text: str, max_tokens: int = 1000, overlap: int = 0) -> List[str]:
+    # This is a simplified token-based chunking. For more accurate tokenization,
+    # consider using a proper tokenizer like GPT-2 TokenizerFast
+    words = text.split()
+    chunks = []
+    current_chunk = []
+    current_token_count = 0
+
+    for word in words:
+        word_token_count = len(word) // 4 + 1  # Rough estimate of token count
+        if current_token_count + word_token_count > max_tokens and current_chunk:
+            chunks.append(' '.join(current_chunk))
+            current_chunk = current_chunk[-overlap:] if overlap > 0 else []
+            current_token_count = sum(len(w) // 4 + 1 for w in current_chunk)
+
+        current_chunk.append(word)
+        current_token_count += word_token_count
+
+    if current_chunk:
+        chunks.append(' '.join(current_chunk))
+
+    return post_process_chunks(chunks)
+
+
+def post_process_chunks(chunks: List[str]) -> List[str]:
+    return [chunk.strip() for chunk in chunks if chunk.strip()]
+
+
+def get_chunk_metadata(chunk: str, full_text: str, chunk_type: str = "generic", chapter_number: Optional[int] = None, chapter_pattern: Optional[str] = None) -> Dict[str, Any]:
+    try:
+        start_index = full_text.index(chunk)
+        metadata = {
+            'start_index': start_index,
+            'end_index': start_index + len(chunk),
+            'word_count': len(chunk.split()),
+            'char_count': len(chunk),
+            'chunk_type': chunk_type
+        }
+        if chunk_type == "chapter":
+            metadata['chapter_number'] = chapter_number
+            metadata['chapter_pattern'] = chapter_pattern
+        return metadata
+    except ValueError as e:
+        logging.error(f"Chunk not found in full_text: {chunk[:50]}... Full text length: {len(full_text)}")
+        raise
+
+
+# Hybrid approach, chunk each sentence while ensuring total token size does not exceed a maximum number
+def chunk_text_hybrid(text, max_tokens=1000):
+    sentences = nltk.tokenize.sent_tokenize(text)
+    chunks = []
+    current_chunk = []
+    current_length = 0
+
+    for sentence in sentences:
+        tokens = tokenizer.encode(sentence)
+        if current_length + len(tokens) <= max_tokens:
+            current_chunk.append(sentence)
+            current_length += len(tokens)
+        else:
+            chunks.append(' '.join(current_chunk))
+            current_chunk = [sentence]
+            current_length = len(tokens)
+
+    if current_chunk:
+        chunks.append(' '.join(current_chunk))
+
+    return chunks
+
+# Thanks openai
+def chunk_on_delimiter(input_string: str,
+                       max_tokens: int,
+                       delimiter: str) -> List[str]:
+    chunks = input_string.split(delimiter)
+    combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum(
+        chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True)
+    if dropped_chunk_count > 0:
+        print(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.")
+    combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks]
+    return combined_chunks
+
+# ????FIXME
+def recursive_summarize_chunks(chunks, summarize_func, custom_prompt, temp=None, system_prompt=None):
+    summarized_chunks = []
+    current_summary = ""
+
+    logging.debug(f"recursive_summarize_chunks: Summarizing {len(chunks)} chunks recursively...")
+    logging.debug(f"recursive_summarize_chunks:  temperature is @ {temp}")
+    for i, chunk in enumerate(chunks):
+        if i == 0:
+            current_summary = summarize_func(chunk, custom_prompt, temp, system_prompt)
+        else:
+            combined_text = current_summary + "\n\n" + chunk
+            current_summary = summarize_func(combined_text, custom_prompt, temp, system_prompt)
+
+        summarized_chunks.append(current_summary)
+
+    return summarized_chunks
+
+
+# Sample text for testing
+sample_text = """
+Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence 
+concerned with the interactions between computers and human language, in particular how to program computers 
+to process and analyze large amounts of natural language data. The result is a computer capable of "understanding" 
+the contents of documents, including the contextual nuances of the language within them. The technology can then 
+accurately extract information and insights contained in the documents as well as categorize and organize the documents themselves.
+
+Challenges in natural language processing frequently involve speech recognition, natural language understanding, 
+and natural language generation.
+
+Natural language processing has its roots in the 1950s. Already in 1950, Alan Turing published an article titled 
+"Computing Machinery and Intelligence" which proposed what is now called the Turing test as a criterion of intelligence.
+"""
+
+# Example usage of different chunking methods
+# print("Chunking by words:")
+# print(chunk_text_by_words(sample_text, max_words=50))
+#
+# print("\nChunking by sentences:")
+# print(chunk_text_by_sentences(sample_text, max_sentences=2))
+#
+# print("\nChunking by paragraphs:")
+# print(chunk_text_by_paragraphs(sample_text, max_paragraphs=1))
+#
+# print("\nChunking by tokens:")
+# print(chunk_text_by_tokens(sample_text, max_tokens=50))
+#
+# print("\nHybrid chunking:")
+# print(chunk_text_hybrid(sample_text, max_tokens=50))
+
+
+
+#######################################################################################################################
+#
+# Experimental Semantic Chunking
+#
+
+# Chunk text into segments based on semantic similarity
+def count_units(text, unit='tokens'):
+    if unit == 'words':
+        return len(text.split())
+    elif unit == 'tokens':
+        return len(word_tokenize(text))
+    elif unit == 'characters':
+        return len(text)
+    else:
+        raise ValueError("Invalid unit. Choose 'words', 'tokens', or 'characters'.")
+
+
+def semantic_chunking(text, max_chunk_size=2000, unit='words'):
+    nltk.download('punkt', quiet=True)
+    sentences = sent_tokenize(text)
+    vectorizer = TfidfVectorizer()
+    sentence_vectors = vectorizer.fit_transform(sentences)
+
+    chunks = []
+    current_chunk = []
+    current_size = 0
+
+    for i, sentence in enumerate(sentences):
+        sentence_size = count_units(sentence, unit)
+        if current_size + sentence_size > max_chunk_size and current_chunk:
+            chunks.append(' '.join(current_chunk))
+            overlap_size = count_units(' '.join(current_chunk[-3:]), unit)  # Use last 3 sentences for overlap
+            current_chunk = current_chunk[-3:]  # Keep last 3 sentences for overlap
+            current_size = overlap_size
+
+        current_chunk.append(sentence)
+        current_size += sentence_size
+
+        if i + 1 < len(sentences):
+            current_vector = sentence_vectors[i]
+            next_vector = sentence_vectors[i + 1]
+            similarity = cosine_similarity(current_vector, next_vector)[0][0]
+            if similarity < 0.5 and current_size >= max_chunk_size // 2:
+                chunks.append(' '.join(current_chunk))
+                overlap_size = count_units(' '.join(current_chunk[-3:]), unit)
+                current_chunk = current_chunk[-3:]
+                current_size = overlap_size
+
+    if current_chunk:
+        chunks.append(' '.join(current_chunk))
+
+    return chunks
+
+
+def semantic_chunk_long_file(file_path, max_chunk_size=1000, overlap=100):
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            content = file.read()
+
+        chunks = semantic_chunking(content, max_chunk_size, overlap)
+        return chunks
+    except Exception as e:
+        logging.error(f"Error chunking text file: {str(e)}")
+        return None
+#######################################################################################################################
+
+
+
+
+
+
+#######################################################################################################################
+#
+# OpenAI Rolling Summarization
+#
+
+client = OpenAI(api_key=openai_api_key)
+def get_chat_completion(messages, model='gpt-4-turbo'):
+    response = client.chat.completions.create(
+        model=model,
+        messages=messages,
+        temperature=0,
+    )
+    return response.choices[0].message.content
+
+
+# This function combines text chunks into larger blocks without exceeding a specified token count.
+#   It returns the combined chunks, their original indices, and the number of dropped chunks due to overflow.
+def combine_chunks_with_no_minimum(
+        chunks: List[str],
+        max_tokens: int,
+        chunk_delimiter="\n\n",
+        header: Optional[str] = None,
+        add_ellipsis_for_overflow=False,
+) -> Tuple[List[str], List[int]]:
+    dropped_chunk_count = 0
+    output = []  # list to hold the final combined chunks
+    output_indices = []  # list to hold the indices of the final combined chunks
+    candidate = (
+        [] if header is None else [header]
+    )  # list to hold the current combined chunk candidate
+    candidate_indices = []
+    for chunk_i, chunk in enumerate(chunks):
+        chunk_with_header = [chunk] if header is None else [header, chunk]
+        # FIXME MAKE NOT OPENAI SPECIFIC
+        if len(openai_tokenize(chunk_delimiter.join(chunk_with_header))) > max_tokens:
+            print(f"warning: chunk overflow")
+            if (
+                    add_ellipsis_for_overflow
+                    # FIXME MAKE NOT OPENAI SPECIFIC
+                    and len(openai_tokenize(chunk_delimiter.join(candidate + ["..."]))) <= max_tokens
+            ):
+                candidate.append("...")
+                dropped_chunk_count += 1
+            continue  # this case would break downstream assumptions
+        # estimate token count with the current chunk added
+        # FIXME MAKE NOT OPENAI SPECIFIC
+        extended_candidate_token_count = len(openai_tokenize(chunk_delimiter.join(candidate + [chunk])))
+        # If the token count exceeds max_tokens, add the current candidate to output and start a new candidate
+        if extended_candidate_token_count > max_tokens:
+            output.append(chunk_delimiter.join(candidate))
+            output_indices.append(candidate_indices)
+            candidate = chunk_with_header  # re-initialize candidate
+            candidate_indices = [chunk_i]
+        # otherwise keep extending the candidate
+        else:
+            candidate.append(chunk)
+            candidate_indices.append(chunk_i)
+    # add the remaining candidate to output if it's not empty
+    if (header is not None and len(candidate) > 1) or (header is None and len(candidate) > 0):
+        output.append(chunk_delimiter.join(candidate))
+        output_indices.append(candidate_indices)
+    return output, output_indices, dropped_chunk_count
+
+
+def rolling_summarize(text: str,
+                      detail: float = 0,
+                      model: str = 'gpt-4-turbo',
+                      additional_instructions: Optional[str] = None,
+                      minimum_chunk_size: Optional[int] = 500,
+                      chunk_delimiter: str = ".",
+                      summarize_recursively=False,
+                      verbose=False):
+    """
+    Summarizes a given text by splitting it into chunks, each of which is summarized individually.
+    The level of detail in the summary can be adjusted, and the process can optionally be made recursive.
+
+    Parameters:
+        - text (str): The text to be summarized.
+        - detail (float, optional): A value between 0 and 1
+            indicating the desired level of detail in the summary. 0 leads to a higher level summary, and 1 results in a more
+            detailed summary. Defaults to 0.
+        - additional_instructions (Optional[str], optional): Additional instructions to provide to the
+            model for customizing summaries. - minimum_chunk_size (Optional[int], optional): The minimum size for text
+            chunks. Defaults to 500.
+        - chunk_delimiter (str, optional): The delimiter used to split the text into chunks. Defaults to ".".
+        - summarize_recursively (bool, optional): If True, summaries are generated recursively, using previous summaries for context.
+        - verbose (bool, optional): If True, prints detailed information about the chunking process.
+    Returns:
+    - str: The final compiled summary of the text.
+
+    The function first determines the number of chunks by interpolating between a minimum and a maximum chunk count
+    based on the `detail` parameter. It then splits the text into chunks and summarizes each chunk. If
+    `summarize_recursively` is True, each summary is based on the previous summaries, adding more context to the
+    summarization process. The function returns a compiled summary of all chunks.
+    """
+
+    # check detail is set correctly
+    assert 0 <= detail <= 1
+
+    # interpolate the number of chunks based to get specified level of detail
+    max_chunks = len(chunk_on_delimiter(text, minimum_chunk_size, chunk_delimiter))
+    min_chunks = 1
+    num_chunks = int(min_chunks + detail * (max_chunks - min_chunks))
+
+    # adjust chunk_size based on interpolated number of chunks
+    # FIXME MAKE NOT OPENAI SPECIFIC
+    document_length = len(openai_tokenize(text))
+    chunk_size = max(minimum_chunk_size, document_length // num_chunks)
+    text_chunks = chunk_on_delimiter(text, chunk_size, chunk_delimiter)
+    if verbose:
+        print(f"Splitting the text into {len(text_chunks)} chunks to be summarized.")
+        # FIXME MAKE NOT OPENAI SPECIFIC
+        print(f"Chunk lengths are {[len(openai_tokenize(x)) for x in text_chunks]}")
+
+    # set system message - FIXME
+    system_message_content = "Rewrite this text in summarized form."
+    if additional_instructions is not None:
+        system_message_content += f"\n\n{additional_instructions}"
+
+    accumulated_summaries = []
+    for i, chunk in enumerate(tqdm(text_chunks)):
+        if summarize_recursively and accumulated_summaries:
+            # Combine previous summary with current chunk for recursive summarization
+            combined_text = accumulated_summaries[-1] + "\n\n" + chunk
+            user_message_content = f"Previous summary and new content to summarize:\n\n{combined_text}"
+        else:
+            user_message_content = chunk
+
+        messages = [
+            {"role": "system", "content": system_message_content},
+            {"role": "user", "content": user_message_content}
+        ]
+
+        response = get_chat_completion(messages, model=model)
+        accumulated_summaries.append(response)
+
+    final_summary = '\n\n'.join(accumulated_summaries)
+    return final_summary
+
+#
+#
+#######################################################################################################################
+#
+# Ebook Chapter Chunking
+
+
+def chunk_ebook_by_chapters(text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]:
+    max_chunk_size = chunk_options.get('max_size', 300)
+    overlap = chunk_options.get('overlap', 0)
+    custom_pattern = chunk_options.get('custom_chapter_pattern', None)
+
+    # List of chapter heading patterns to try, in order
+    chapter_patterns = [
+        custom_pattern,
+        r'^#{1,2}\s+',  # Markdown style: '# ' or '## '
+        r'^Chapter\s+\d+',  # 'Chapter ' followed by numbers
+        r'^\d+\.\s+',  # Numbered chapters: '1. ', '2. ', etc.
+        r'^[A-Z\s]+$'  # All caps headings
+    ]
+
+    chapter_positions = []
+    used_pattern = None
+
+    for pattern in chapter_patterns:
+        if pattern is None:
+            continue
+        chapter_regex = re.compile(pattern, re.MULTILINE | re.IGNORECASE)
+        chapter_positions = [match.start() for match in chapter_regex.finditer(text)]
+        if chapter_positions:
+            used_pattern = pattern
+            break
+
+    # If no chapters found, return the entire content as one chunk
+    if not chapter_positions:
+        return [{'text': text, 'metadata': get_chunk_metadata(text, text, chunk_type="whole_document")}]
+
+    # Split content into chapters
+    chunks = []
+    for i in range(len(chapter_positions)):
+        start = chapter_positions[i]
+        end = chapter_positions[i + 1] if i + 1 < len(chapter_positions) else None
+        chapter = text[start:end]
+
+        # Apply overlap if specified
+        if overlap > 0 and i > 0:
+            overlap_start = max(0, start - overlap)
+            chapter = text[overlap_start:end]
+
+        chunks.append(chapter)
+
+    # Post-process chunks
+    processed_chunks = post_process_chunks(chunks)
+
+    # Add metadata to chunks
+    return [{'text': chunk, 'metadata': get_chunk_metadata(chunk, text, chunk_type="chapter", chapter_number=i + 1,
+                                                           chapter_pattern=used_pattern)}
+            for i, chunk in enumerate(processed_chunks)]
+
+
+# # Example usage
+# if __name__ == "__main__":
+#     sample_ebook_content = """
+# # Chapter 1: Introduction
+#
+# This is the introduction.
+#
+# ## Section 1.1
+#
+# Some content here.
+#
+# # Chapter 2: Main Content
+#
+# This is the main content.
+#
+# ## Section 2.1
+#
+# More content here.
+#
+# CHAPTER THREE
+#
+# This is the third chapter.
+#
+# 4. Fourth Chapter
+#
+# This is the fourth chapter.
+# """
+#
+#     chunk_options = {
+#         'method': 'chapters',
+#         'max_size': 500,
+#         'overlap': 50,
+#         'custom_chapter_pattern': r'^CHAPTER\s+[A-Z]+'  # Custom pattern for 'CHAPTER THREE' style
+#     }
+#
+#     chunked_chapters = improved_chunking_process(sample_ebook_content, chunk_options)
+#
+#     for i, chunk in enumerate(chunked_chapters, 1):
+#         print(f"Chunk {i}:")
+#         print(chunk['text'])
+#         print(f"Metadata: {chunk['metadata']}\n")
+
+
+
+
+#
+# End of Chunking Library
 #######################################################################################################################

App_Function_Libraries/DB_Manager.py

@@ -0,0 +1,472 @@
+import configparser
+import logging
+import os
+from contextlib import contextmanager
+from time import sleep
+from typing import Tuple
+import sqlite3
+# 3rd-Party Libraries
+from elasticsearch import Elasticsearch
+
+############################################################################################################
+#
+# This file contains the DatabaseManager class, which is responsible for managing the database connection, i.e. either SQLite or Elasticsearch.
+
+####
+# The DatabaseManager class provides the following methods:
+# - add_media: Add a new media item to the database
+# - fetch_items_by_keyword: Fetch media items from the database based on a keyword
+# - fetch_item_details: Fetch details of a specific media item from the database
+# - update_media_content: Update the content of a specific media item in the database
+# - search_and_display_items: Search for media items in the database and display the results
+# - close_connection: Close the database connection
+####
+
+# Import your existing SQLite functions
+from SQLite_DB import (
+    update_media_content as sqlite_update_media_content,
+    list_prompts as sqlite_list_prompts,
+    search_and_display as sqlite_search_and_display,
+    fetch_prompt_details as sqlite_fetch_prompt_details,
+    keywords_browser_interface as sqlite_keywords_browser_interface,
+    add_keyword as sqlite_add_keyword,
+    delete_keyword as sqlite_delete_keyword,
+    export_keywords_to_csv as sqlite_export_keywords_to_csv,
+    ingest_article_to_db as sqlite_ingest_article_to_db,
+    add_media_to_database as sqlite_add_media_to_database,
+    import_obsidian_note_to_db as sqlite_import_obsidian_note_to_db,
+    add_prompt as sqlite_add_prompt,
+    delete_chat_message as sqlite_delete_chat_message,
+    update_chat_message as sqlite_update_chat_message,
+    add_chat_message as sqlite_add_chat_message,
+    get_chat_messages as sqlite_get_chat_messages,
+    search_chat_conversations as sqlite_search_chat_conversations,
+    create_chat_conversation as sqlite_create_chat_conversation,
+    save_chat_history_to_database as sqlite_save_chat_history_to_database,
+    view_database as sqlite_view_database,
+    get_transcripts as sqlite_get_transcripts,
+    get_trashed_items as sqlite_get_trashed_items,
+    user_delete_item as sqlite_user_delete_item,
+    empty_trash as sqlite_empty_trash,
+    create_automated_backup as sqlite_create_automated_backup,
+    add_or_update_prompt as sqlite_add_or_update_prompt,
+    load_prompt_details as sqlite_load_prompt_details,
+    load_preset_prompts as sqlite_load_preset_prompts,
+    insert_prompt_to_db as sqlite_insert_prompt_to_db,
+    delete_prompt as sqlite_delete_prompt,
+    search_and_display_items as sqlite_search_and_display_items,
+    get_conversation_name as sqlite_get_conversation_name,
+    add_media_with_keywords as sqlite_add_media_with_keywords,
+    check_media_and_whisper_model as sqlite_check_media_and_whisper_model,
+    DatabaseError
+)
+
+class Database:
+    def __init__(self, db_path=None):
+        self.db_path = db_path or os.getenv('DB_NAME', 'media_summary.db')
+        self.pool = []
+        self.pool_size = 10
+
+    @contextmanager
+    def get_connection(self):
+        retry_count = 5
+        retry_delay = 1
+        conn = None
+        while retry_count > 0:
+            try:
+                conn = self.pool.pop() if self.pool else sqlite3.connect(self.db_path, check_same_thread=False)
+                yield conn
+                self.pool.append(conn)
+                return
+            except sqlite3.OperationalError as e:
+                if 'database is locked' in str(e):
+                    logging.warning(f"Database is locked, retrying in {retry_delay} seconds...")
+                    retry_count -= 1
+                    sleep(retry_delay)
+                else:
+                    raise DatabaseError(f"Database error: {e}")
+            except Exception as e:
+                raise DatabaseError(f"Unexpected error: {e}")
+            finally:
+                # Ensure the connection is returned to the pool even on failure
+                if conn and conn not in self.pool:
+                    self.pool.append(conn)
+        raise DatabaseError("Database is locked and retries have been exhausted")
+
+    def execute_query(self, query: str, params: Tuple = ()) -> None:
+        with self.get_connection() as conn:
+            try:
+                cursor = conn.cursor()
+                cursor.execute(query, params)
+                conn.commit()
+            except sqlite3.Error as e:
+                raise DatabaseError(f"Database error: {e}, Query: {query}")
+
+    def close_all_connections(self):
+        for conn in self.pool:
+            conn.close()
+        self.pool.clear()
+
+def get_db_config():
+    config = configparser.ConfigParser()
+    config.read('config.txt')
+    return {
+        'type': config['Database']['type'],
+        'sqlite_path': config.get('Database', 'sqlite_path', fallback='media_summary.db'),
+        'elasticsearch_host': config.get('Database', 'elasticsearch_host', fallback='localhost'),
+        'elasticsearch_port': config.getint('Database', 'elasticsearch_port', fallback=9200)
+    }
+
+db_config = get_db_config()
+db_type = db_config['type']
+
+if db_type == 'sqlite':
+    # Use the config path if provided, otherwise fall back to default
+    db = Database(db_config.get('sqlite_path'))
+elif db_type == 'elasticsearch':
+    es = Elasticsearch([{
+        'host': db_config['elasticsearch_host'],
+        'port': db_config['elasticsearch_port']
+    }])
+else:
+    raise ValueError(f"Unsupported database type: {db_type}")
+
+db_path = db_config['sqlite_path']
+
+# Update this path to the directory where you want to store the database backups
+backup_dir = os.environ.get('DB_BACKUP_DIR', 'path/to/backup/directory')
+
+
+
+
+if db_type == 'sqlite':
+    conn = sqlite3.connect(db_config['sqlite_path'])
+    cursor = conn.cursor()
+elif db_type == 'elasticsearch':
+    es = Elasticsearch([{
+        'host': db_config['elasticsearch_host'],
+        'port': db_config['elasticsearch_port']
+    }])
+else:
+    raise ValueError(f"Unsupported database type: {db_type}")
+
+############################################################################################################
+#
+# DB-Searching functions
+
+def view_database(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_view_database(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def search_and_display_items(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_search_and_display_items(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def search_and_display(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_search_and_display(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of DB-Searching functions
+############################################################################################################
+
+############################################################################################################
+#
+# Transcript-related Functions
+
+def get_transcripts(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_transcripts(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of Transcript-related Functions
+############################################################################################################
+
+############################################################################################################
+#
+# DB-Ingestion functions
+
+def add_media_to_database(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_media_to_database(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+
+def import_obsidian_note_to_db(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_import_obsidian_note_to_db(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def update_media_content(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_update_media_content(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def add_media_with_keywords(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_media_with_keywords(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def check_media_and_whisper_model(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_check_media_and_whisper_model(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        raise NotImplementedError("Elasticsearch version of check_media_and_whisper_model not yet implemented")
+
+def ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt):
+    if db_type == 'sqlite':
+        return sqlite_ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of ingest_article_to_db not yet implemented")
+    else:
+        raise ValueError(f"Unsupported database type: {db_type}")
+
+#
+# End of DB-Ingestion functions
+############################################################################################################
+
+
+############################################################################################################
+#
+# Prompt-related functions
+
+def list_prompts(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_list_prompts(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+
+def fetch_prompt_details(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_fetch_prompt_details(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def add_prompt(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_prompt(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+
+def add_or_update_prompt(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_or_update_prompt(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def load_prompt_details(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_load_prompt_details(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def load_preset_prompts(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_load_preset_prompts(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def insert_prompt_to_db(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_insert_prompt_to_db(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def delete_prompt(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_delete_prompt(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of Prompt-related functions
+############################################################################################################
+
+############################################################################################################
+#
+# Keywords-related Functions
+
+def keywords_browser_interface(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_keywords_browser_interface(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def add_keyword(*args, **kwargs):
+    if db_type == 'sqlite':
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+        return sqlite_add_keyword(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def delete_keyword(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_delete_keyword(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def export_keywords_to_csv(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_export_keywords_to_csv(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of Keywords-related Functions
+############################################################################################################
+
+############################################################################################################
+#
+# Chat-related Functions
+
+def delete_chat_message(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_delete_chat_message(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def update_chat_message(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_update_chat_message(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def add_chat_message(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_chat_message(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def get_chat_messages(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_chat_messages(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def search_chat_conversations(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_search_chat_conversations(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def create_chat_conversation(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_create_chat_conversation(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def save_chat_history_to_database(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_save_chat_history_to_database(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def get_conversation_name(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_conversation_name(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of Chat-related Functions
+############################################################################################################
+
+############################################################################################################
+#
+# Trash-related Functions
+
+def get_trashed_items(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_trashed_items(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def user_delete_item(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_user_delete_item(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def empty_trash(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_empty_trash(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of Trash-related Functions
+############################################################################################################
+
+############################################################################################################
+#
+# DB-Backup Functions
+
+def create_automated_backup(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_create_automated_backup(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+#
+# End of DB-Backup Functions
+############################################################################################################
+
+############################################################################################################
+#
+# Function to close the database connection for SQLite
+
+def close_connection():
+    if db_type == 'sqlite':
+        db.close_all_connections()
+    # Elasticsearch doesn't need explicit closing
+
+#
+# End of file
+############################################################################################################

App_Function_Libraries/Diarization_Lib.py

@@ -33,7 +33,7 @@ import yaml
 
 def load_pipeline_from_pretrained(path_to_config: str | Path) -> SpeakerDiarization:
     path_to_config = Path(path_to_config).resolve()
-    print(f"Loading pyannote pipeline from {path_to_config}...")
+    logging.debug(f"Loading pyannote pipeline from {path_to_config}...")
 
     if not path_to_config.exists():
         raise FileNotFoundError(f"Config file not found: {path_to_config}")
@@ -45,11 +45,6 @@ def load_pipeline_from_pretrained(path_to_config: str | Path) -> SpeakerDiarizat
     # Store current working directory
     cwd = Path.cwd().resolve()
 
-    # Change to the directory containing the config file
-    cd_to = path_to_config.parent.resolve()
-    print(f"Changing working directory to {cd_to}")
-    os.chdir(cd_to)
-
     try:
         # Create a SpeakerDiarization pipeline
         pipeline = SpeakerDiarization()

App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py

@@ -0,0 +1,152 @@
+# Audio_ingestion_tab.py
+# Description: Gradio UI for ingesting audio files into the database
+#
+# Imports
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Audio_Files import process_audio_files
+from App_Function_Libraries.DB_Manager import load_preset_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models
+#
+#######################################################################################################################
+# Functions:
+
+def create_audio_processing_tab():
+    with gr.TabItem("Audio File Transcription + Summarization"):
+        gr.Markdown("# Transcribe & Summarize Audio Files from URLs or Local Files!")
+        with gr.Row():
+            with gr.Column():
+                audio_url_input = gr.Textbox(label="Audio File URL(s)", placeholder="Enter the URL(s) of the audio file(s), one per line")
+                audio_file_input = gr.File(label="Upload Audio File", file_types=["audio/*"])
+
+                use_cookies_input = gr.Checkbox(label="Use cookies for authenticated download", value=False)
+                cookies_input = gr.Textbox(
+                    label="Audio Download Cookies",
+                    placeholder="Paste your cookies here (JSON format)",
+                    lines=3,
+                    visible=False
+                )
+
+                use_cookies_input.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[use_cookies_input],
+                    outputs=[cookies_input]
+                )
+
+                diarize_input = gr.Checkbox(label="Enable Speaker Diarization", value=False)
+                whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
+
+                with gr.Row():
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=load_preset_prompts(),
+                                                visible=False)
+                with gr.Row():
+                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+""",
+                                                     lines=3,
+                                                     visible=False)
+
+                custom_prompt_checkbox.change(
+                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    inputs=[custom_prompt_checkbox],
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+                preset_prompt_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[preset_prompt_checkbox],
+                    outputs=[preset_prompt]
+                )
+
+                def update_prompts(preset_name):
+                    prompts = update_user_prompt(preset_name)
+                    return (
+                        gr.update(value=prompts["user_prompt"], visible=True),
+                        gr.update(value=prompts["system_prompt"], visible=True)
+                    )
+
+                preset_prompt.change(
+                    update_prompts,
+                    inputs=preset_prompt,
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                    value=None,
+                    label="API for Summarization (Optional)"
+                )
+                api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here", type="password")
+                custom_keywords_input = gr.Textbox(label="Custom Keywords", placeholder="Enter custom keywords, comma-separated")
+                keep_original_input = gr.Checkbox(label="Keep original audio file", value=False)
+
+                chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False)
+                with gr.Row(visible=False) as chunking_options_box:
+                    gr.Markdown("### Chunking Options")
+                    with gr.Column():
+                        chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], label="Chunking Method")
+                        max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size")
+                        chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
+                        use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking")
+                        use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking")
+                        chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], label="Chunking Language")
+
+                chunking_options_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[chunking_options_checkbox],
+                    outputs=[chunking_options_box]
+                )
+
+                process_audio_button = gr.Button("Process Audio File(s)")
+
+            with gr.Column():
+                audio_progress_output = gr.Textbox(label="Progress")
+                audio_transcription_output = gr.Textbox(label="Transcription")
+                audio_summary_output = gr.Textbox(label="Summary")
+                download_transcription = gr.File(label="Download All Transcriptions as JSON")
+                download_summary = gr.File(label="Download All Summaries as Text")
+
+        process_audio_button.click(
+            fn=process_audio_files,
+            inputs=[audio_url_input, audio_file_input, whisper_model_input, api_name_input, api_key_input,
+                    use_cookies_input, cookies_input, keep_original_input, custom_keywords_input, custom_prompt_input,
+                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, use_multi_level_chunking,
+                    chunk_language, diarize_input],
+            outputs=[audio_progress_output, audio_transcription_output, audio_summary_output]
+        )
+
+#
+# End of Audio_ingestion_tab.py
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Chat_ui.py

@@ -0,0 +1,1017 @@
+# Chat_ui.py
+# Description: Chat interface functions for Gradio
+#
+# Imports
+import html
+import json
+import logging
+import os
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Chat import chat, save_chat_history, update_chat_content, save_chat_history_to_db_wrapper
+from App_Function_Libraries.DB_Manager import add_chat_message, search_chat_conversations, create_chat_conversation, \
+    get_chat_messages, update_chat_message, delete_chat_message, load_preset_prompts, db
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_user_prompt
+
+
+#
+#
+########################################################################################################################
+#
+# Functions:
+
+
+def show_edit_message(selected):
+    if selected:
+        return gr.update(value=selected[0], visible=True), gr.update(value=selected[1], visible=True), gr.update(
+            visible=True)
+    return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
+
+
+def show_delete_message(selected):
+    if selected:
+        return gr.update(value=selected[1], visible=True), gr.update(visible=True)
+    return gr.update(visible=False), gr.update(visible=False)
+
+
+def debug_output(media_content, selected_parts):
+    print(f"Debug - Media Content: {media_content}")
+    print(f"Debug - Selected Parts: {selected_parts}")
+    return ""
+
+
+def update_selected_parts(use_content, use_summary, use_prompt):
+    selected_parts = []
+    if use_content:
+        selected_parts.append("content")
+    if use_summary:
+        selected_parts.append("summary")
+    if use_prompt:
+        selected_parts.append("prompt")
+    print(f"Debug - Update Selected Parts: {selected_parts}")
+    return selected_parts
+
+
+# Old update_user_prompt shim for backwards compatibility
+def get_system_prompt(preset_name):
+    # For backwards compatibility
+    prompts = update_user_prompt(preset_name)
+    return prompts["system_prompt"]
+
+def clear_chat():
+    # Return empty list for chatbot and None for conversation_id
+    return gr.update(value=[]), None
+
+
+# FIXME - add additional features....
+def chat_wrapper(message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, conversation_id,
+                 save_conversation, temperature, system_prompt, max_tokens=None, top_p=None, frequency_penalty=None,
+                 presence_penalty=None, stop_sequence=None):
+    try:
+        if save_conversation:
+            if conversation_id is None:
+                # Create a new conversation
+                media_id = media_content.get('id', None)
+                conversation_name = f"Chat about {media_content.get('title', 'Unknown Media')} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+                conversation_id = create_chat_conversation(media_id, conversation_name)
+
+            # Add user message to the database
+            user_message_id = add_chat_message(conversation_id, "user", message)
+
+        # Include the selected parts and custom_prompt only for the first message
+        if not history and selected_parts:
+            message_body = "\n".join(selected_parts)
+            full_message = f"{custom_prompt}\n\n{message}\n\n{message_body}"
+        elif custom_prompt:
+            full_message = f"{custom_prompt}\n\n{message}"
+        else:
+            full_message = message
+
+        # Generate bot response
+        bot_message = chat(full_message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt,
+                           temperature, system_prompt)
+
+        if save_conversation:
+            # Add assistant message to the database
+            add_chat_message(conversation_id, "assistant", bot_message)
+
+        # Update history
+        history.append((message, bot_message))
+
+        return bot_message, history, conversation_id
+    except Exception as e:
+        logging.error(f"Error in chat wrapper: {str(e)}")
+        return "An error occurred.", history, conversation_id
+
+
+def search_conversations(query):
+    try:
+        conversations = search_chat_conversations(query)
+        if not conversations:
+            print(f"Debug - Search Conversations - No results found for query: {query}")
+            return gr.update(choices=[])
+
+        conversation_options = [
+            (f"{c['conversation_name']} (Media: {c['media_title']}, ID: {c['id']})", c['id'])
+            for c in conversations
+        ]
+        print(f"Debug - Search Conversations - Options: {conversation_options}")
+        return gr.update(choices=conversation_options)
+    except Exception as e:
+        print(f"Debug - Search Conversations - Error: {str(e)}")
+        return gr.update(choices=[])
+
+
+def load_conversation(conversation_id):
+    if not conversation_id:
+        return [], None
+
+    messages = get_chat_messages(conversation_id)
+    history = [
+        (msg['message'], None) if msg['sender'] == 'user' else (None, msg['message'])
+        for msg in messages
+    ]
+    return history, conversation_id
+
+
+def update_message_in_chat(message_id, new_text, history):
+    update_chat_message(message_id, new_text)
+    updated_history = [(msg1, msg2) if msg1[1] != message_id and msg2[1] != message_id
+                       else ((new_text, msg1[1]) if msg1[1] == message_id else (new_text, msg2[1]))
+                       for msg1, msg2 in history]
+    return updated_history
+
+
+def delete_message_from_chat(message_id, history):
+    delete_chat_message(message_id)
+    updated_history = [(msg1, msg2) for msg1, msg2 in history if msg1[1] != message_id and msg2[1] != message_id]
+    return updated_history
+
+
+def create_chat_interface():
+    custom_css = """
+    .chatbot-container .message-wrap .message {
+        font-size: 14px !important;
+    }
+    """
+    with gr.TabItem("Remote LLM Chat (Horizontal)"):
+        gr.Markdown("# Chat with a designated LLM Endpoint, using your selected item as starting context")
+        chat_history = gr.State([])
+        media_content = gr.State({})
+        selected_parts = gr.State([])
+        conversation_id = gr.State(None)
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
+                                             label="Search By")
+                search_button = gr.Button("Search")
+                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+                item_mapping = gr.State({})
+                with gr.Row():
+                    use_content = gr.Checkbox(label="Use Content")
+                    use_summary = gr.Checkbox(label="Use Summary")
+                    use_prompt = gr.Checkbox(label="Use Prompt")
+                    save_conversation = gr.Checkbox(label="Save Conversation", value=False, visible=True)
+                with gr.Row():
+                    temperature = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
+                with gr.Row():
+                    conversation_search = gr.Textbox(label="Search Conversations")
+                with gr.Row():
+                    search_conversations_btn = gr.Button("Search Conversations")
+                with gr.Row():
+                    previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
+                with gr.Row():
+                    load_conversations_btn = gr.Button("Load Selected Conversation")
+
+                api_endpoint = gr.Dropdown(label="Select API Endpoint",
+                                           choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
+                                                    "Mistral", "OpenRouter",
+                                                    "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama",
+                                                    "HuggingFace"])
+                api_key = gr.Textbox(label="API Key (if required)", type="password")
+                custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                            choices=load_preset_prompts(),
+                                            visible=False)
+                user_prompt = gr.Textbox(label="Custom Prompt",
+                                         placeholder="Enter custom prompt here",
+                                         lines=3,
+                                         visible=False)
+                system_prompt_input = gr.Textbox(label="System Prompt",
+                                                 value="You are a helpful AI assitant",
+                                                 lines=3,
+                                                 visible=False)
+            with gr.Column():
+                chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
+                msg = gr.Textbox(label="Enter your message")
+                submit = gr.Button("Submit")
+                clear_chat_button = gr.Button("Clear Chat")
+
+                edit_message_id = gr.Number(label="Message ID to Edit", visible=False)
+                edit_message_text = gr.Textbox(label="Edit Message", visible=False)
+                update_message_button = gr.Button("Update Message", visible=False)
+
+                delete_message_id = gr.Number(label="Message ID to Delete", visible=False)
+                delete_message_button = gr.Button("Delete Message", visible=False)
+
+                save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
+                save_chat_history_as_file = gr.Button("Save Chat History as File")
+                download_file = gr.File(label="Download Chat History")
+                save_status = gr.Textbox(label="Save Status", interactive=False)
+
+        # Restore original functionality
+        search_button.click(
+            fn=update_dropdown,
+            inputs=[search_query_input, search_type_input],
+            outputs=[items_output, item_mapping]
+        )
+
+        def save_chat_wrapper(history, conversation_id, media_content):
+            file_path = save_chat_history(history, conversation_id, media_content)
+            if file_path:
+                return file_path, f"Chat history saved successfully as {os.path.basename(file_path)}!"
+            else:
+                return None, "Error saving chat history. Please check the logs and try again."
+
+        save_chat_history_as_file.click(
+            save_chat_wrapper,
+            inputs=[chatbot, conversation_id, media_content],
+            outputs=[download_file, save_status]
+        )
+
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        def clear_chat():
+            return [], None  # Return empty list for chatbot and None for conversation_id
+
+        clear_chat_button.click(
+            clear_chat,
+            outputs=[chatbot, conversation_id]
+        )
+        preset_prompt.change(
+            update_prompts,
+            inputs=preset_prompt,
+            outputs=[user_prompt, system_prompt_input]
+        )
+        custom_prompt_checkbox.change(
+            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+            inputs=[custom_prompt_checkbox],
+            outputs=[user_prompt, system_prompt_input]
+        )
+        preset_prompt_checkbox.change(
+            fn=lambda x: gr.update(visible=x),
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt]
+        )
+
+        submit.click(
+            chat_wrapper,
+            inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt,
+                    conversation_id, save_conversation, temperature, system_prompt_input],
+            outputs=[msg, chatbot, conversation_id]
+        ).then(  # Clear the message box after submission
+            lambda x: gr.update(value=""),
+            inputs=[chatbot],
+            outputs=[msg]
+        ).then(  # Clear the user prompt after the first message
+            lambda: (gr.update(value=""), gr.update(value="")),
+            outputs=[user_prompt, system_prompt_input]
+        )
+
+        items_output.change(
+            update_chat_content,
+            inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
+            outputs=[media_content, selected_parts]
+        )
+        use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                           outputs=[selected_parts])
+        use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                           outputs=[selected_parts])
+        use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                          outputs=[selected_parts])
+        items_output.change(debug_output, inputs=[media_content, selected_parts], outputs=[])
+
+        search_conversations_btn.click(
+            search_conversations,
+            inputs=[conversation_search],
+            outputs=[previous_conversations]
+        )
+
+        load_conversations_btn.click(
+            clear_chat,
+            outputs=[chatbot, chat_history]
+        ).then(
+            load_conversation,
+            inputs=[previous_conversations],
+            outputs=[chatbot, conversation_id]
+        )
+
+        previous_conversations.change(
+            load_conversation,
+            inputs=[previous_conversations],
+            outputs=[chat_history]
+        )
+
+        update_message_button.click(
+            update_message_in_chat,
+            inputs=[edit_message_id, edit_message_text, chat_history],
+            outputs=[chatbot]
+        )
+
+        delete_message_button.click(
+            delete_message_from_chat,
+            inputs=[delete_message_id, chat_history],
+            outputs=[chatbot]
+        )
+
+        save_chat_history_as_file.click(
+            save_chat_history,
+            inputs=[chatbot, conversation_id],
+            outputs=[download_file]
+        )
+
+        save_chat_history_to_db.click(
+            save_chat_history_to_db_wrapper,
+            inputs=[chatbot, conversation_id, media_content],
+            outputs=[conversation_id, gr.Textbox(label="Save Status")]
+        )
+
+        chatbot.select(show_edit_message, None, [edit_message_text, edit_message_id, update_message_button])
+        chatbot.select(show_delete_message, None, [delete_message_id, delete_message_button])
+
+
+def create_chat_interface_stacked():
+    custom_css = """
+    .chatbot-container .message-wrap .message {
+        font-size: 14px !important;
+    }
+    """
+    with gr.TabItem("Remote LLM Chat - Stacked"):
+        gr.Markdown("# Stacked Chat")
+        chat_history = gr.State([])
+        media_content = gr.State({})
+        selected_parts = gr.State([])
+        conversation_id = gr.State(None)
+
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
+                                             label="Search By")
+                search_button = gr.Button("Search")
+                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+                item_mapping = gr.State({})
+                with gr.Row():
+                    use_content = gr.Checkbox(label="Use Content")
+                    use_summary = gr.Checkbox(label="Use Summary")
+                    use_prompt = gr.Checkbox(label="Use Prompt")
+                    save_conversation = gr.Checkbox(label="Save Conversation", value=False, visible=True)
+                    temp = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
+                with gr.Row():
+                    conversation_search = gr.Textbox(label="Search Conversations")
+                with gr.Row():
+                    previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
+                with gr.Row():
+                    search_conversations_btn = gr.Button("Search Conversations")
+                    load_conversations_btn = gr.Button("Load Selected Conversation")
+            with gr.Column():
+                api_endpoint = gr.Dropdown(label="Select API Endpoint",
+                                           choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
+                                                    "OpenRouter", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi",
+                                                    "VLLM", "ollama", "HuggingFace"])
+                api_key = gr.Textbox(label="API Key (if required)", type="password")
+                preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                            choices=load_preset_prompts(),
+                                            visible=True)
+                system_prompt = gr.Textbox(label="System Prompt",
+                                           value="You are a helpful AI assistant.",
+                                           lines=3,
+                                           visible=True)
+                user_prompt = gr.Textbox(label="Custom User Prompt",
+                                         placeholder="Enter custom prompt here",
+                                         lines=3,
+                                         visible=True)
+                gr.Markdown("Scroll down for the chat window...")
+        with gr.Row():
+            with gr.Column(scale=1):
+                chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
+                msg = gr.Textbox(label="Enter your message")
+        with gr.Row():
+            with gr.Column():
+                submit = gr.Button("Submit")
+                clear_chat_button = gr.Button("Clear Chat")
+
+                edit_message_id = gr.Number(label="Message ID to Edit", visible=False)
+                edit_message_text = gr.Textbox(label="Edit Message", visible=False)
+                update_message_button = gr.Button("Update Message", visible=False)
+
+                delete_message_id = gr.Number(label="Message ID to Delete", visible=False)
+                delete_message_button = gr.Button("Delete Message", visible=False)
+                save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
+                save_chat_history_as_file = gr.Button("Save Chat History as File")
+            with gr.Column():
+                download_file = gr.File(label="Download Chat History")
+
+        # Restore original functionality
+        search_button.click(
+            fn=update_dropdown,
+            inputs=[search_query_input, search_type_input],
+            outputs=[items_output, item_mapping]
+        )
+
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        clear_chat_button.click(
+            clear_chat,
+            outputs=[chatbot, conversation_id]
+        )
+        preset_prompt.change(
+            update_prompts,
+            inputs=preset_prompt,
+            outputs=[user_prompt, system_prompt]
+        )
+
+        submit.click(
+            chat_wrapper,
+            inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt,
+                    conversation_id, save_conversation, temp, system_prompt],
+            outputs=[msg, chatbot, conversation_id]
+        ).then(  # Clear the message box after submission
+            lambda x: gr.update(value=""),
+            inputs=[chatbot],
+            outputs=[msg]
+        ).then(  # Clear the user prompt after the first message
+            lambda: gr.update(value=""),
+            outputs=[user_prompt, system_prompt]
+        )
+
+        items_output.change(
+            update_chat_content,
+            inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
+            outputs=[media_content, selected_parts]
+        )
+        use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                           outputs=[selected_parts])
+        use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                           outputs=[selected_parts])
+        use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                          outputs=[selected_parts])
+        items_output.change(debug_output, inputs=[media_content, selected_parts], outputs=[])
+
+        search_conversations_btn.click(
+            search_conversations,
+            inputs=[conversation_search],
+            outputs=[previous_conversations]
+        )
+
+        load_conversations_btn.click(
+            clear_chat,
+            outputs=[chatbot, chat_history]
+        ).then(
+            load_conversation,
+            inputs=[previous_conversations],
+            outputs=[chatbot, conversation_id]
+        )
+
+        previous_conversations.change(
+            load_conversation,
+            inputs=[previous_conversations],
+            outputs=[chat_history]
+        )
+
+        update_message_button.click(
+            update_message_in_chat,
+            inputs=[edit_message_id, edit_message_text, chat_history],
+            outputs=[chatbot]
+        )
+
+        delete_message_button.click(
+            delete_message_from_chat,
+            inputs=[delete_message_id, chat_history],
+            outputs=[chatbot]
+        )
+
+        save_chat_history_as_file.click(
+            save_chat_history,
+            inputs=[chatbot, conversation_id],
+            outputs=[download_file]
+        )
+
+        save_chat_history_to_db.click(
+            save_chat_history_to_db_wrapper,
+            inputs=[chatbot, conversation_id, media_content],
+            outputs=[conversation_id, gr.Textbox(label="Save Status")]
+        )
+
+        chatbot.select(show_edit_message, None, [edit_message_text, edit_message_id, update_message_button])
+        chatbot.select(show_delete_message, None, [delete_message_id, delete_message_button])
+
+
+# FIXME - System prompts
+def create_chat_interface_multi_api():
+    custom_css = """
+    .chatbot-container .message-wrap .message {
+        font-size: 14px !important;
+    }
+    .chat-window {
+        height: 400px;
+        overflow-y: auto;
+    }
+    """
+    with gr.TabItem("One Prompt - Multiple APIs"):
+        gr.Markdown("# One Prompt but Multiple API Chat Interface")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
+                                             label="Search By")
+                search_button = gr.Button("Search")
+                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+                item_mapping = gr.State({})
+                with gr.Row():
+                    use_content = gr.Checkbox(label="Use Content")
+                    use_summary = gr.Checkbox(label="Use Summary")
+                    use_prompt = gr.Checkbox(label="Use Prompt")
+            with gr.Column():
+                preset_prompt = gr.Dropdown(label="Select Preset Prompt", choices=load_preset_prompts(), visible=True)
+                system_prompt = gr.Textbox(label="System Prompt", value="You are a helpful AI assistant.", lines=5)
+                user_prompt = gr.Textbox(label="Modify Prompt", lines=5, value=".")
+
+        with gr.Row():
+            chatbots = []
+            api_endpoints = []
+            api_keys = []
+            temperatures = []
+            for i in range(3):
+                with gr.Column():
+                    gr.Markdown(f"### Chat Window {i + 1}")
+                    api_endpoint = gr.Dropdown(label=f"API Endpoint {i + 1}",
+                                               choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
+                                                        "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
+                                                        "Ooba",
+                                                        "Tabbyapi", "VLLM", "ollama", "HuggingFace"])
+                    api_key = gr.Textbox(label=f"API Key {i + 1} (if required)", type="password")
+                    temperature = gr.Slider(label=f"Temperature {i + 1}", minimum=0.0, maximum=1.0, step=0.05,
+                                            value=0.7)
+                    chatbot = gr.Chatbot(height=800, elem_classes="chat-window")
+                    chatbots.append(chatbot)
+                    api_endpoints.append(api_endpoint)
+                    api_keys.append(api_key)
+                    temperatures.append(temperature)
+
+        with gr.Row():
+            msg = gr.Textbox(label="Enter your message", scale=4)
+            submit = gr.Button("Submit", scale=1)
+            # FIXME - clear chat
+        #     clear_chat_button = gr.Button("Clear Chat")
+        #
+        # clear_chat_button.click(
+        #     clear_chat,
+        #     outputs=[chatbot]
+        # )
+
+        # State variables
+        chat_history = [gr.State([]) for _ in range(3)]
+        media_content = gr.State({})
+        selected_parts = gr.State([])
+        conversation_id = gr.State(None)
+
+        # Event handlers
+        search_button.click(
+            fn=update_dropdown,
+            inputs=[search_query_input, search_type_input],
+            outputs=[items_output, item_mapping]
+        )
+
+        preset_prompt.change(update_user_prompt, inputs=preset_prompt, outputs=user_prompt)
+
+        def chat_wrapper_multi(message, custom_prompt, system_prompt, *args):
+            chat_histories = args[:3]
+            chatbots = args[3:6]
+            api_endpoints = args[6:9]
+            api_keys = args[9:12]
+            temperatures = args[12:15]
+            media_content = args[15]
+            selected_parts = args[16]
+
+            new_chat_histories = []
+            new_chatbots = []
+
+            for i in range(3):
+                # Call chat_wrapper with dummy values for conversation_id and save_conversation
+                bot_message, new_history, _ = chat_wrapper(
+                    message, chat_histories[i], media_content, selected_parts,
+                    api_endpoints[i], api_keys[i], custom_prompt, None,  # None for conversation_id
+                    False,  # False for save_conversation
+                    temperature=temperatures[i],
+                    system_prompt=system_prompt
+                )
+
+                new_chatbot = chatbots[i] + [(message, bot_message)]
+
+                new_chat_histories.append(new_history)
+                new_chatbots.append(new_chatbot)
+
+            return [gr.update(value="")] + new_chatbots + new_chat_histories
+
+        # In the create_chat_interface_multi_api function:
+        submit.click(
+            chat_wrapper_multi,
+            inputs=[msg, user_prompt,
+                    system_prompt] + chat_history + chatbots + api_endpoints + api_keys + temperatures +
+                   [media_content, selected_parts],
+            outputs=[msg] + chatbots + chat_history
+        ).then(
+            lambda: (gr.update(value=""), gr.update(value="")),
+            outputs=[msg, user_prompt]
+        )
+
+        items_output.change(
+            update_chat_content,
+            inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
+            outputs=[media_content, selected_parts]
+        )
+
+        for checkbox in [use_content, use_summary, use_prompt]:
+            checkbox.change(
+                update_selected_parts,
+                inputs=[use_content, use_summary, use_prompt],
+                outputs=[selected_parts]
+            )
+
+
+def create_chat_interface_four():
+    custom_css = """
+    .chatbot-container .message-wrap .message {
+        font-size: 14px !important;
+    }
+    .chat-window {
+        height: 400px;
+        overflow-y: auto;
+    }
+    """
+    with gr.TabItem("Four Independent API Chats"):
+        gr.Markdown("# Four Independent API Chat Interfaces")
+
+        with gr.Row():
+            with gr.Column():
+                preset_prompt = gr.Dropdown(label="Select Preset Prompt", choices=load_preset_prompts(), visible=True)
+                user_prompt = gr.Textbox(label="Modify Prompt", lines=3, value=".")
+            with gr.Column():
+                gr.Markdown("Scroll down for the chat windows...")
+        chat_interfaces = []
+        for row in range(2):
+            with gr.Row():
+                for col in range(2):
+                    i = row * 2 + col
+                    with gr.Column():
+                        gr.Markdown(f"### Chat Window {i + 1}")
+                        api_endpoint = gr.Dropdown(label=f"API Endpoint {i + 1}",
+                                                   choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
+                                                            "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
+                                                            "Ooba",
+                                                            "Tabbyapi", "VLLM", "ollama", "HuggingFace"])
+                        api_key = gr.Textbox(label=f"API Key {i + 1} (if required)", type="password")
+                        temperature = gr.Slider(label=f"Temperature {i + 1}", minimum=0.0, maximum=1.0, step=0.05,
+                                                value=0.7)
+                        chatbot = gr.Chatbot(height=400, elem_classes="chat-window")
+                        msg = gr.Textbox(label=f"Enter your message for Chat {i + 1}")
+                        submit = gr.Button(f"Submit to Chat {i + 1}")
+
+                        chat_interfaces.append({
+                            'api_endpoint': api_endpoint,
+                            'api_key': api_key,
+                            'temperature': temperature,
+                            'chatbot': chatbot,
+                            'msg': msg,
+                            'submit': submit,
+                            'chat_history': gr.State([])
+                        })
+
+        preset_prompt.change(update_user_prompt, inputs=preset_prompt, outputs=user_prompt)
+
+        def chat_wrapper_single(message, chat_history, api_endpoint, api_key, temperature, user_prompt):
+            logging.debug(f"Chat Wrapper Single - Message: {message}, Chat History: {chat_history}")
+            new_msg, new_history, _ = chat_wrapper(
+                message, chat_history, {}, [],  # Empty media_content and selected_parts
+                api_endpoint, api_key, user_prompt, None,  # No conversation_id
+                False,  # Not saving conversation
+                temperature=temperature, system_prompt=""
+            )
+            chat_history.append((message, new_msg))
+            return "", chat_history, chat_history
+
+        for interface in chat_interfaces:
+            logging.debug(f"Chat Interface - Clicked Submit for Chat {interface['chatbot']}"),
+            interface['submit'].click(
+                chat_wrapper_single,
+                inputs=[
+                    interface['msg'],
+                    interface['chat_history'],
+                    interface['api_endpoint'],
+                    interface['api_key'],
+                    interface['temperature'],
+                    user_prompt
+                ],
+                outputs=[
+                    interface['msg'],
+                    interface['chatbot'],
+                    interface['chat_history']
+                ]
+            )
+
+
+def chat_wrapper_single(message, chat_history, chatbot, api_endpoint, api_key, temperature, media_content,
+                        selected_parts, conversation_id, save_conversation, user_prompt):
+    new_msg, new_history, new_conv_id = chat_wrapper(
+        message, chat_history, media_content, selected_parts,
+        api_endpoint, api_key, user_prompt, conversation_id,
+        save_conversation, temperature, system_prompt=""
+    )
+
+    if new_msg:
+        updated_chatbot = chatbot + [(message, new_msg)]
+    else:
+        updated_chatbot = chatbot
+
+    return new_msg, updated_chatbot, new_history, new_conv_id
+
+
+# FIXME - Finish implementing functions + testing/valdidation
+def create_chat_management_tab():
+    with gr.TabItem("Chat Management"):
+        gr.Markdown("# Chat Management")
+
+        with gr.Row():
+            search_query = gr.Textbox(label="Search Conversations")
+            search_button = gr.Button("Search")
+
+        conversation_list = gr.Dropdown(label="Select Conversation", choices=[])
+        conversation_mapping = gr.State({})
+
+        with gr.Tabs():
+            with gr.TabItem("Edit"):
+                chat_content = gr.TextArea(label="Chat Content (JSON)", lines=20, max_lines=50)
+                save_button = gr.Button("Save Changes")
+
+            with gr.TabItem("Preview"):
+                chat_preview = gr.HTML(label="Chat Preview")
+        result_message = gr.Markdown("")
+
+        def search_conversations(query):
+            conversations = search_chat_conversations(query)
+            choices = [f"{conv['conversation_name']} (Media: {conv['media_title']}, ID: {conv['id']})" for conv in
+                       conversations]
+            mapping = {choice: conv['id'] for choice, conv in zip(choices, conversations)}
+            return gr.update(choices=choices), mapping
+
+        def load_conversations(selected, conversation_mapping):
+            logging.info(f"Selected: {selected}")
+            logging.info(f"Conversation mapping: {conversation_mapping}")
+
+            try:
+                if selected and selected in conversation_mapping:
+                    conversation_id = conversation_mapping[selected]
+                    messages = get_chat_messages(conversation_id)
+                    conversation_data = {
+                        "conversation_id": conversation_id,
+                        "messages": messages
+                    }
+                    json_content = json.dumps(conversation_data, indent=2)
+
+                    # Create HTML preview
+                    html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
+                    for msg in messages:
+                        sender_style = "background-color: #e6f3ff;" if msg[
+                                                                           'sender'] == 'user' else "background-color: #f0f0f0;"
+                        html_preview += f"<div style='margin-bottom: 10px; padding: 10px; border-radius: 5px; {sender_style}'>"
+                        html_preview += f"<strong>{msg['sender']}:</strong> {html.escape(msg['message'])}<br>"
+                        html_preview += f"<small>Timestamp: {msg['timestamp']}</small>"
+                        html_preview += "</div>"
+                    html_preview += "</div>"
+
+                    logging.info("Returning json_content and html_preview")
+                    return json_content, html_preview
+                else:
+                    logging.warning("No conversation selected or not in mapping")
+                    return "", "<p>No conversation selected</p>"
+            except Exception as e:
+                logging.error(f"Error in load_conversations: {str(e)}")
+                return f"Error: {str(e)}", "<p>Error loading conversation</p>"
+
+        def validate_conversation_json(content):
+            try:
+                data = json.loads(content)
+                if not isinstance(data, dict):
+                    return False, "Invalid JSON structure: root should be an object"
+                if "conversation_id" not in data or not isinstance(data["conversation_id"], int):
+                    return False, "Missing or invalid conversation_id"
+                if "messages" not in data or not isinstance(data["messages"], list):
+                    return False, "Missing or invalid messages array"
+                for msg in data["messages"]:
+                    if not all(key in msg for key in ["sender", "message"]):
+                        return False, "Invalid message structure: missing required fields"
+                return True, data
+            except json.JSONDecodeError as e:
+                return False, f"Invalid JSON: {str(e)}"
+
+        def save_conversation(selected, conversation_mapping, content):
+            if not selected or selected not in conversation_mapping:
+                return "Please select a conversation before saving.", "<p>No changes made</p>"
+
+            conversation_id = conversation_mapping[selected]
+            is_valid, result = validate_conversation_json(content)
+
+            if not is_valid:
+                return f"Error: {result}", "<p>No changes made due to error</p>"
+
+            conversation_data = result
+            if conversation_data["conversation_id"] != conversation_id:
+                return "Error: Conversation ID mismatch.", "<p>No changes made due to ID mismatch</p>"
+
+            try:
+                with db.get_connection() as conn:
+                    conn.execute("BEGIN TRANSACTION")
+                    cursor = conn.cursor()
+
+                    # Backup original conversation
+                    cursor.execute("SELECT * FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
+                    original_messages = cursor.fetchall()
+                    backup_data = json.dumps({"conversation_id": conversation_id, "messages": original_messages})
+
+                    # You might want to save this backup_data somewhere
+
+                    # Delete existing messages
+                    cursor.execute("DELETE FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
+
+                    # Insert updated messages
+                    for message in conversation_data["messages"]:
+                        cursor.execute('''
+                            INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
+                            VALUES (?, ?, ?, COALESCE(?, CURRENT_TIMESTAMP))
+                        ''', (conversation_id, message["sender"], message["message"], message.get("timestamp")))
+
+                    conn.commit()
+
+                    # Create updated HTML preview
+                    html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
+                    for msg in conversation_data["messages"]:
+                        sender_style = "background-color: #e6f3ff;" if msg[
+                                                                           'sender'] == 'user' else "background-color: #f0f0f0;"
+                        html_preview += f"<div style='margin-bottom: 10px; padding: 10px; border-radius: 5px; {sender_style}'>"
+                        html_preview += f"<strong>{msg['sender']}:</strong> {html.escape(msg['message'])}<br>"
+                        html_preview += f"<small>Timestamp: {msg.get('timestamp', 'N/A')}</small>"
+                        html_preview += "</div>"
+                    html_preview += "</div>"
+
+                    return "Conversation updated successfully.", html_preview
+            except sqlite3.Error as e:
+                conn.rollback()
+                logging.error(f"Database error in save_conversation: {e}")
+                return f"Error updating conversation: {str(e)}", "<p>Error occurred while saving</p>"
+            except Exception as e:
+                conn.rollback()
+                logging.error(f"Unexpected error in save_conversation: {e}")
+                return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>"
+
+        def parse_formatted_content(formatted_content):
+            lines = formatted_content.split('\n')
+            conversation_id = int(lines[0].split(': ')[1])
+            timestamp = lines[1].split(': ')[1]
+            history = []
+            current_role = None
+            current_content = None
+            for line in lines[3:]:
+                if line.startswith("Role: "):
+                    if current_role is not None:
+                        history.append({"role": current_role, "content": ["", current_content]})
+                    current_role = line.split(': ')[1]
+                elif line.startswith("Content: "):
+                    current_content = line.split(': ', 1)[1]
+            if current_role is not None:
+                history.append({"role": current_role, "content": ["", current_content]})
+            return json.dumps({
+                "conversation_id": conversation_id,
+                "timestamp": timestamp,
+                "history": history
+            }, indent=2)
+
+        search_button.click(
+            search_conversations,
+            inputs=[search_query],
+            outputs=[conversation_list, conversation_mapping]
+        )
+
+        conversation_list.change(
+            load_conversations,
+            inputs=[conversation_list, conversation_mapping],
+            outputs=[chat_content, chat_preview]
+        )
+
+        save_button.click(
+            save_conversation,
+            inputs=[conversation_list, conversation_mapping, chat_content],
+            outputs=[result_message, chat_preview]
+        )
+
+    return search_query, search_button, conversation_list, conversation_mapping, chat_content, save_button, result_message, chat_preview
+
+
+# FIXME - busted and incomplete
+# Mock function to simulate LLM processing
+def process_with_llm(workflow, context, prompt):
+    return f"LLM output for {workflow} with context: {context[:30]}... and prompt: {prompt[:30]}..."
+
+
+# Load workflows from a JSON file
+json_path = Path('./Helper_Scripts/Workflows/Workflows.json')
+with json_path.open('r') as f:
+    workflows = json.load(f)
+
+
+# FIXME - broken Completely. Doesn't work.
+def chat_workflows_tab():
+    with gr.TabItem("Chat Workflows"):
+        with gr.Blocks() as chat_workflows_block:
+            gr.Markdown("# Workflows using LLMs")
+
+            workflow_selector = gr.Dropdown(label="Select Workflow", choices=[wf['name'] for wf in workflows])
+            context_input = gr.Textbox(label="Context", lines=5)
+
+            # Create lists to hold UI components
+            prompt_inputs = []
+            process_buttons = []
+            output_boxes = []
+            max_prompts = max(len(wf['prompts']) for wf in workflows)
+
+            # Pre-create the maximum number of prompt sections
+            for i in range(max_prompts):
+                prompt_input = gr.Textbox(label=f"Prompt {i + 1}", lines=2, visible=False)
+                output_box = gr.Textbox(label=f"Output {i + 1}", lines=5, visible=False)
+                process_button = gr.Button(f"Process Prompt {i + 1}", visible=False)
+
+                prompt_inputs.append(prompt_input)
+                output_boxes.append(output_box)
+                process_buttons.append(process_button)
+
+                process_button.click(
+                    fn=lambda context, prompt, workflow_name, step=i: process(context, prompt, workflow_name, step),
+                    inputs=[context_input, prompt_input, workflow_selector],
+                    outputs=[output_box]
+                )
+
+            def process(context, prompt, workflow_name, step):
+                selected_workflow = next(wf for wf in workflows if wf['name'] == workflow_name)
+                # Update context with previous outputs
+                for j in range(step):
+                    context += f"\n\n{output_boxes[j].value}"
+                result = process_with_llm(selected_workflow['name'], context, prompt)
+                return result
+
+            def update_prompt_sections(workflow_name):
+                selected_workflow = next(wf for wf in workflows if wf['name'] == workflow_name)
+                num_prompts = len(selected_workflow['prompts'])
+
+                for i in range(max_prompts):
+                    if i < num_prompts:
+                        prompt_inputs[i].visible = True
+                        prompt_inputs[i].value = selected_workflow['prompts'][i]
+                        process_buttons[i].visible = True
+                        output_boxes[i].visible = True
+                    else:
+                        prompt_inputs[i].visible = False
+                        process_buttons[i].visible = False
+                        output_boxes[i].visible = False
+
+            # Bind the workflow selector to update the UI
+            workflow_selector.change(update_prompt_sections, inputs=[workflow_selector], outputs=[])
+
+        return chat_workflows_block
+
+#
+# End of Chat_ui.py
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py

@@ -0,0 +1,193 @@
+# Explain_summarize_tab.py
+# Gradio UI for explaining and summarizing text
+#
+# Imports
+import logging
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Local_Summarization_Lib import summarize_with_llama, summarize_with_kobold, \
+    summarize_with_oobabooga, summarize_with_tabbyapi, summarize_with_vllm, summarize_with_local_llm, \
+    summarize_with_ollama
+from App_Function_Libraries.Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, \
+    summarize_with_cohere, summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, \
+    summarize_with_huggingface
+#
+#
+############################################################################################################
+#
+# Functions:
+
+def create_summarize_explain_tab():
+    with gr.TabItem("Explain/Summarize Text"):
+        gr.Markdown("# Explain or Summarize Text without ingesting it into the DB")
+        with gr.Row():
+            with gr.Column():
+                text_to_work_input = gr.Textbox(label="Text to be Explained or Summarized",
+                                                placeholder="Enter the text you want explained or summarized here",
+                                                lines=20)
+                with gr.Row():
+                    explanation_checkbox = gr.Checkbox(label="Explain Text", value=True)
+                    summarization_checkbox = gr.Checkbox(label="Summarize Text", value=True)
+                api_endpoint = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
+                             "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    value=None,
+                    label="API for Summarization (Optional)"
+                )
+                api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here",
+                                           type="password")
+                explain_summarize_button = gr.Button("Explain/Summarize")
+
+            with gr.Column():
+                summarization_output = gr.Textbox(label="Summary:", lines=20)
+                explanation_output = gr.Textbox(label="Explanation:", lines=50)
+
+        explain_summarize_button.click(
+            fn=summarize_explain_text,
+            inputs=[text_to_work_input, api_endpoint, api_key_input, summarization_checkbox, explanation_checkbox],
+            outputs=[summarization_output, explanation_output]
+        )
+
+
+def summarize_explain_text(message, api_endpoint, api_key, summarization, explanation):
+    summarization_response = None
+    explanation_response = None
+    temp = 0.7
+    try:
+        logging.info(f"Debug - summarize_explain_text Function - Message: {message}")
+        logging.info(f"Debug - summarize_explain_text Function - API Endpoint: {api_endpoint}")
+
+        # Prepare the input for the API
+        input_data = f"User: {message}\n"
+        # Print first 500 chars
+        logging.info(f"Debug - Chat Function - Input Data: {input_data[:500]}...")
+        logging.debug(f"Debug - Chat Function - API Key: {api_key[:10]}")
+        user_prompt = " "
+        if not api_endpoint:
+            return "Please select an API endpoint", "Please select an API endpoint"
+        try:
+            if summarization:
+                system_prompt = """<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                **Bulleted Note Creation Guidelines**
+
+                **Headings**:
+                - Based on referenced topics, not categories like quotes or terms
+                - Surrounded by **bold** formatting 
+                - Not listed as bullet points
+                - No space between headings and list items underneath
+
+                **Emphasis**:
+                - **Important terms** set in bold font
+                - **Text ending in a colon**: also bolded
+
+                **Review**:
+                - Ensure adherence to specified format
+                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]"""
+
+                # Use the existing API request code based on the selected endpoint
+                logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
+                if api_endpoint.lower() == 'openai':
+                    summarization_response = summarize_with_openai(api_key, input_data, user_prompt, temp,
+                                                                   system_prompt)
+                elif api_endpoint.lower() == "anthropic":
+                    summarization_response = summarize_with_anthropic(api_key, input_data, user_prompt, temp,
+                                                                      system_prompt)
+                elif api_endpoint.lower() == "cohere":
+                    summarization_response = summarize_with_cohere(api_key, input_data, user_prompt, temp,
+                                                                   system_prompt)
+                elif api_endpoint.lower() == "groq":
+                    summarization_response = summarize_with_groq(api_key, input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "openrouter":
+                    summarization_response = summarize_with_openrouter(api_key, input_data, user_prompt, temp,
+                                                                       system_prompt)
+                elif api_endpoint.lower() == "deepseek":
+                    summarization_response = summarize_with_deepseek(api_key, input_data, user_prompt, temp,
+                                                                     system_prompt)
+                elif api_endpoint.lower() == "llama.cpp":
+                    summarization_response = summarize_with_llama(input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "kobold":
+                    summarization_response = summarize_with_kobold(input_data, api_key, user_prompt, temp,
+                                                                   system_prompt)
+                elif api_endpoint.lower() == "ooba":
+                    summarization_response = summarize_with_oobabooga(input_data, api_key, user_prompt, temp,
+                                                                      system_prompt)
+                elif api_endpoint.lower() == "tabbyapi":
+                    summarization_response = summarize_with_tabbyapi(input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "vllm":
+                    summarization_response = summarize_with_vllm(input_data, user_prompt, system_prompt)
+                elif api_endpoint.lower() == "local-llm":
+                    summarization_response = summarize_with_local_llm(input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "huggingface":
+                    summarization_response = summarize_with_huggingface(api_key, input_data, user_prompt,
+                                                                        temp)  # , system_prompt)
+                elif api_endpoint.lower() == "ollama":
+                    summarization_response = summarize_with_ollama(input_data, user_prompt, temp, system_prompt)
+                else:
+                    raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
+        except Exception as e:
+            logging.error(f"Error in summarization: {str(e)}")
+            response1 = f"An error occurred during summarization: {str(e)}"
+
+        try:
+            if explanation:
+                system_prompt = """You are a professional teacher. Please explain the content presented in an easy to digest fashion so that a non-specialist may understand it."""
+                # Use the existing API request code based on the selected endpoint
+                logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
+                if api_endpoint.lower() == 'openai':
+                    explanation_response = summarize_with_openai(api_key, input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "anthropic":
+                    explanation_response = summarize_with_anthropic(api_key, input_data, user_prompt, temp,
+                                                                    system_prompt)
+                elif api_endpoint.lower() == "cohere":
+                    explanation_response = summarize_with_cohere(api_key, input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "groq":
+                    explanation_response = summarize_with_groq(api_key, input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "openrouter":
+                    explanation_response = summarize_with_openrouter(api_key, input_data, user_prompt, temp,
+                                                                     system_prompt)
+                elif api_endpoint.lower() == "deepseek":
+                    explanation_response = summarize_with_deepseek(api_key, input_data, user_prompt, temp,
+                                                                   system_prompt)
+                elif api_endpoint.lower() == "llama.cpp":
+                    explanation_response = summarize_with_llama(input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "kobold":
+                    explanation_response = summarize_with_kobold(input_data, api_key, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "ooba":
+                    explanation_response = summarize_with_oobabooga(input_data, api_key, user_prompt, temp,
+                                                                    system_prompt)
+                elif api_endpoint.lower() == "tabbyapi":
+                    explanation_response = summarize_with_tabbyapi(input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "vllm":
+                    explanation_response = summarize_with_vllm(input_data, user_prompt, system_prompt)
+                elif api_endpoint.lower() == "local-llm":
+                    explanation_response = summarize_with_local_llm(input_data, user_prompt, temp, system_prompt)
+                elif api_endpoint.lower() == "huggingface":
+                    explanation_response = summarize_with_huggingface(api_key, input_data, user_prompt,
+                                                                      temp)  # , system_prompt)
+                elif api_endpoint.lower() == "ollama":
+                    explanation_response = summarize_with_ollama(input_data, user_prompt, temp, system_prompt)
+                else:
+                    raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
+        except Exception as e:
+            logging.error(f"Error in summarization: {str(e)}")
+            response2 = f"An error occurred during summarization: {str(e)}"
+
+        if summarization_response:
+            response1 = f"Summary: {summarization_response}"
+        else:
+            response1 = "Summary: No summary requested"
+
+        if explanation_response:
+            response2 = f"Explanation: {explanation_response}"
+        else:
+            response2 = "Explanation: No explanation requested"
+
+        return response1, response2
+
+    except Exception as e:
+        logging.error(f"Error in chat function: {str(e)}")
+        return f"An error occurred: {str(e)}"

App_Function_Libraries/Gradio_UI/Export_Functionality.py

@@ -0,0 +1,314 @@
+# Export_Functionality.py
+# Functionality for exporting items as markdown files
+import os
+import json
+import math
+import logging
+import shutil
+import tempfile
+from typing import List, Dict, Optional, Tuple
+import gradio as gr
+from App_Function_Libraries.DB_Manager import DatabaseError, create_automated_backup, db_path, backup_dir
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import fetch_item_details, fetch_items_by_keyword, browse_items
+
+logger = logging.getLogger(__name__)
+
+def export_item_as_markdown(media_id: int) -> Tuple[Optional[str], str]:
+    try:
+        content, prompt, summary = fetch_item_details(media_id)
+        title = f"Item {media_id}"  # You might want to fetch the actual title
+        markdown_content = f"# {title}\n\n## Prompt\n{prompt}\n\n## Summary\n{summary}\n\n## Content\n{content}"
+
+        filename = f"export_item_{media_id}.md"
+        with open(filename, "w", encoding='utf-8') as f:
+            f.write(markdown_content)
+
+        logger.info(f"Successfully exported item {media_id} to {filename}")
+        return filename, f"Successfully exported item {media_id} to {filename}"
+    except Exception as e:
+        error_message = f"Error exporting item {media_id}: {str(e)}"
+        logger.error(error_message)
+        return None, error_message
+
+
+def export_items_by_keyword(keyword: str) -> str:
+    try:
+        items = fetch_items_by_keyword(keyword)
+        if not items:
+            logger.warning(f"No items found for keyword: {keyword}")
+            return None
+
+        # Create a temporary directory to store individual markdown files
+        with tempfile.TemporaryDirectory() as temp_dir:
+            folder_name = f"export_keyword_{keyword}"
+            export_folder = os.path.join(temp_dir, folder_name)
+            os.makedirs(export_folder)
+
+            for item in items:
+                content, prompt, summary = fetch_item_details(item['id'])
+                markdown_content = f"# {item['title']}\n\n## Prompt\n{prompt}\n\n## Summary\n{summary}\n\n## Content\n{content}"
+
+                # Create individual markdown file for each item
+                file_name = f"{item['id']}_{item['title'][:50]}.md"  # Limit filename length
+                file_path = os.path.join(export_folder, file_name)
+                with open(file_path, "w", encoding='utf-8') as f:
+                    f.write(markdown_content)
+
+            # Create a zip file containing all markdown files
+            zip_filename = f"{folder_name}.zip"
+            shutil.make_archive(os.path.join(temp_dir, folder_name), 'zip', export_folder)
+
+            # Move the zip file to a location accessible by Gradio
+            final_zip_path = os.path.join(os.getcwd(), zip_filename)
+            shutil.move(os.path.join(temp_dir, zip_filename), final_zip_path)
+
+        logger.info(f"Successfully exported {len(items)} items for keyword '{keyword}' to {zip_filename}")
+        return final_zip_path
+    except Exception as e:
+        logger.error(f"Error exporting items for keyword '{keyword}': {str(e)}")
+        return None
+
+
+def export_selected_items(selected_items: List[Dict]) -> Tuple[Optional[str], str]:
+    try:
+        logger.debug(f"Received selected_items: {selected_items}")
+        if not selected_items:
+            logger.warning("No items selected for export")
+            return None, "No items selected for export"
+
+        markdown_content = "# Selected Items\n\n"
+        for item in selected_items:
+            logger.debug(f"Processing item: {item}")
+            try:
+                # Check if 'value' is a string (JSON) or already a dictionary
+                if isinstance(item, str):
+                    item_data = json.loads(item)
+                elif isinstance(item, dict) and 'value' in item:
+                    item_data = item['value'] if isinstance(item['value'], dict) else json.loads(item['value'])
+                else:
+                    item_data = item
+
+                logger.debug(f"Item data after processing: {item_data}")
+
+                if 'id' not in item_data:
+                    logger.error(f"'id' not found in item data: {item_data}")
+                    continue
+
+                content, prompt, summary = fetch_item_details(item_data['id'])
+                markdown_content += f"## {item_data.get('title', 'Item {}'.format(item_data['id']))}\n\n### Prompt\n{prompt}\n\n### Summary\n{summary}\n\n### Content\n{content}\n\n---\n\n"
+            except Exception as e:
+                logger.error(f"Error processing item {item}: {str(e)}")
+                markdown_content += f"## Error\n\nUnable to process this item.\n\n---\n\n"
+
+        filename = "export_selected_items.md"
+        with open(filename, "w", encoding='utf-8') as f:
+            f.write(markdown_content)
+
+        logger.info(f"Successfully exported {len(selected_items)} selected items to {filename}")
+        return filename, f"Successfully exported {len(selected_items)} items to {filename}"
+    except Exception as e:
+        error_message = f"Error exporting selected items: {str(e)}"
+        logger.error(error_message)
+        return None, error_message
+
+
+def display_search_results_export_tab(search_query: str, search_type: str, page: int = 1, items_per_page: int = 10):
+    logger.info(f"Searching with query: '{search_query}', type: '{search_type}', page: {page}")
+    try:
+        results = browse_items(search_query, search_type)
+        logger.info(f"browse_items returned {len(results)} results")
+
+        if not results:
+            return [], f"No results found for query: '{search_query}'", 1, 1
+
+        total_pages = math.ceil(len(results) / items_per_page)
+        start_index = (page - 1) * items_per_page
+        end_index = start_index + items_per_page
+        paginated_results = results[start_index:end_index]
+
+        checkbox_data = [
+            {
+                "name": f"Name: {item[1]}\nURL: {item[2]}",
+                "value": {"id": item[0], "title": item[1], "url": item[2]}
+            }
+            for item in paginated_results
+        ]
+
+        logger.info(f"Returning {len(checkbox_data)} items for checkbox (page {page} of {total_pages})")
+        return checkbox_data, f"Found {len(results)} results (showing page {page} of {total_pages})", page, total_pages
+
+    except DatabaseError as e:
+        error_message = f"Error in display_search_results_export_tab: {str(e)}"
+        logger.error(error_message)
+        return [], error_message, 1, 1
+    except Exception as e:
+        error_message = f"Unexpected error in display_search_results_export_tab: {str(e)}"
+        logger.error(error_message)
+        return [], error_message, 1, 1
+
+
+def create_export_tab():
+    with gr.Tab("Search and Export"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Search and Export Items")
+                gr.Markdown("Search for items and export them as markdown files")
+                gr.Markdown("You can also export items by keyword")
+                search_query = gr.Textbox(label="Search Query")
+                search_type = gr.Radio(["Title", "URL", "Keyword", "Content"], label="Search By")
+                search_button = gr.Button("Search")
+
+            with gr.Column():
+                prev_button = gr.Button("Previous Page")
+                next_button = gr.Button("Next Page")
+
+        current_page = gr.State(1)
+        total_pages = gr.State(1)
+
+        search_results = gr.CheckboxGroup(label="Search Results", choices=[])
+        export_selected_button = gr.Button("Export Selected Items")
+
+        keyword_input = gr.Textbox(label="Enter keyword for export")
+        export_by_keyword_button = gr.Button("Export items by keyword")
+
+        export_output = gr.File(label="Download Exported File")
+        error_output = gr.Textbox(label="Status/Error Messages", interactive=False)
+
+    def search_and_update(query, search_type, page):
+        results, message, current, total = display_search_results_export_tab(query, search_type, page)
+        logger.debug(f"search_and_update results: {results}")
+        return results, message, current, total, gr.update(choices=results)
+
+    search_button.click(
+        fn=search_and_update,
+        inputs=[search_query, search_type, current_page],
+        outputs=[search_results, error_output, current_page, total_pages, search_results],
+        show_progress="full"
+    )
+
+
+    def update_page(current, total, direction):
+        new_page = max(1, min(total, current + direction))
+        return new_page
+
+    prev_button.click(
+        fn=update_page,
+        inputs=[current_page, total_pages, gr.State(-1)],
+        outputs=[current_page]
+    ).then(
+        fn=search_and_update,
+        inputs=[search_query, search_type, current_page],
+        outputs=[search_results, error_output, current_page, total_pages],
+        show_progress=True
+    )
+
+    next_button.click(
+        fn=update_page,
+        inputs=[current_page, total_pages, gr.State(1)],
+        outputs=[current_page]
+    ).then(
+        fn=search_and_update,
+        inputs=[search_query, search_type, current_page],
+        outputs=[search_results, error_output, current_page, total_pages],
+        show_progress=True
+    )
+
+    def handle_export_selected(selected_items):
+        logger.debug(f"Exporting selected items: {selected_items}")
+        return export_selected_items(selected_items)
+
+    export_selected_button.click(
+        fn=handle_export_selected,
+        inputs=[search_results],
+        outputs=[export_output, error_output],
+        show_progress="full"
+    )
+
+    export_by_keyword_button.click(
+        fn=export_items_by_keyword,
+        inputs=[keyword_input],
+        outputs=[export_output, error_output],
+        show_progress="full"
+    )
+
+    def handle_item_selection(selected_items):
+        logger.debug(f"Selected items: {selected_items}")
+        if not selected_items:
+            return None, "No item selected"
+
+        try:
+            # Assuming selected_items is a list of dictionaries
+            selected_item = selected_items[0]
+            logger.debug(f"First selected item: {selected_item}")
+
+            # Check if 'value' is a string (JSON) or already a dictionary
+            if isinstance(selected_item['value'], str):
+                item_data = json.loads(selected_item['value'])
+            else:
+                item_data = selected_item['value']
+
+            logger.debug(f"Item data: {item_data}")
+
+            item_id = item_data['id']
+            return export_item_as_markdown(item_id)
+        except Exception as e:
+            error_message = f"Error processing selected item: {str(e)}"
+            logger.error(error_message)
+            return None, error_message
+
+    search_results.select(
+        fn=handle_item_selection,
+        inputs=[search_results],
+        outputs=[export_output, error_output],
+        show_progress="full"
+    )
+
+
+
+def create_backup():
+    backup_file = create_automated_backup(db_path, backup_dir)
+    return f"Backup created: {backup_file}"
+
+def list_backups():
+    backups = [f for f in os.listdir(backup_dir) if f.endswith('.db')]
+    return "\n".join(backups)
+
+def restore_backup(backup_name):
+    backup_path = os.path.join(backup_dir, backup_name)
+    if os.path.exists(backup_path):
+        shutil.copy2(backup_path, db_path)
+        return f"Database restored from {backup_name}"
+    else:
+        return "Backup file not found"
+
+
+def create_backup_tab():
+    with gr.Tab("Create Backup"):
+        gr.Markdown("# Create a backup of the database")
+        with gr.Row():
+            with gr.Column():
+                create_button = gr.Button("Create Backup")
+                create_output = gr.Textbox(label="Result")
+            with gr.Column():
+                create_button.click(create_backup, inputs=[], outputs=create_output)
+
+def create_view_backups_tab():
+    with gr.TabItem("View Backups"):
+        gr.Markdown("# Browse available backups")
+        with gr.Row():
+            with gr.Column():
+                view_button = gr.Button("View Backups")
+            with gr.Column():
+                backup_list = gr.Textbox(label="Available Backups")
+                view_button.click(list_backups, inputs=[], outputs=backup_list)
+
+
+def create_restore_backup_tab():
+    with gr.TabItem("Restore Backup"):
+        gr.Markdown("# Restore a backup of the database")
+        with gr.Column():
+            backup_input = gr.Textbox(label="Backup Filename")
+            restore_button = gr.Button("Restore")
+        with gr.Column():
+            restore_output = gr.Textbox(label="Result")
+            restore_button.click(restore_backup, inputs=[backup_input], outputs=restore_output)

App_Function_Libraries/Gradio_UI/Gradio_Shared.py

@@ -0,0 +1,284 @@
+# Gradio_Shared.py
+# Gradio UI functions that are shared across multiple tabs
+#
+# Imports
+import logging
+import sqlite3
+import traceback
+from functools import wraps
+from typing import List, Tuple
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import list_prompts, db, search_and_display, fetch_prompt_details
+from App_Function_Libraries.SQLite_DB import DatabaseError
+from App_Function_Libraries.Utils import format_transcription
+#
+##############################################################################################################
+#
+# Functions:
+
+whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3",
+                  "distil-large-v2", "distil-medium.en", "distil-small.en"]
+
+# Sample data
+prompts_category_1 = [
+    "What are the key points discussed in the video?",
+    "Summarize the main arguments made by the speaker.",
+    "Describe the conclusions of the study presented."
+]
+
+prompts_category_2 = [
+    "How does the proposed solution address the problem?",
+    "What are the implications of the findings?",
+    "Can you explain the theory behind the observed phenomenon?"
+]
+
+all_prompts = prompts_category_1 + prompts_category_2
+
+
+
+#FIXME - SQL Functions that need to be addressed/added to DB manager
+def search_media(query, fields, keyword, page):
+    try:
+        results = search_and_display(query, fields, keyword, page)
+        return results
+    except Exception as e:
+        logger = logging.getLogger()
+        logger.error(f"Error searching media: {e}")
+        return str(e)
+
+def fetch_items_by_title_or_url(search_query: str, search_type: str):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            if search_type == 'Title':
+                cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',))
+            elif search_type == 'URL':
+                cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',))
+            results = cursor.fetchall()
+            return results
+    except sqlite3.Error as e:
+        raise DatabaseError(f"Error fetching items by {search_type}: {e}")
+
+def fetch_items_by_keyword(search_query: str):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT m.id, m.title, m.url
+                FROM Media m
+                JOIN MediaKeywords mk ON m.id = mk.media_id
+                JOIN Keywords k ON mk.keyword_id = k.id
+                WHERE k.keyword LIKE ?
+            """, (f'%{search_query}%',))
+            results = cursor.fetchall()
+            return results
+    except sqlite3.Error as e:
+        raise DatabaseError(f"Error fetching items by keyword: {e}")
+
+# FIXME - Raw SQL not using DB_Manager...
+def fetch_items_by_content(search_query: str):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT id, title, url FROM Media WHERE content LIKE ?", (f'%{search_query}%',))
+            results = cursor.fetchall()
+            return results
+    except sqlite3.Error as e:
+        raise DatabaseError(f"Error fetching items by content: {e}")
+
+
+
+# FIXME - RAW SQL not using DB_Manager...
+def fetch_item_details_single(media_id: int):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT prompt, summary 
+                FROM MediaModifications 
+                WHERE media_id = ? 
+                ORDER BY modification_date DESC 
+                LIMIT 1
+            """, (media_id,))
+            prompt_summary_result = cursor.fetchone()
+            cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,))
+            content_result = cursor.fetchone()
+
+            prompt = prompt_summary_result[0] if prompt_summary_result else ""
+            summary = prompt_summary_result[1] if prompt_summary_result else ""
+            content = content_result[0] if content_result else ""
+
+            return prompt, summary, content
+    except sqlite3.Error as e:
+        raise Exception(f"Error fetching item details: {e}")
+
+
+# FIXME - RAW SQL not using DB_Manager...
+def fetch_item_details(media_id: int):
+    try:
+        with db.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT prompt, summary 
+                FROM MediaModifications 
+                WHERE media_id = ? 
+                ORDER BY modification_date DESC 
+                LIMIT 1
+            """, (media_id,))
+            prompt_summary_result = cursor.fetchone()
+            cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,))
+            content_result = cursor.fetchone()
+
+            prompt = prompt_summary_result[0] if prompt_summary_result else ""
+            summary = prompt_summary_result[1] if prompt_summary_result else ""
+            content = content_result[0] if content_result else ""
+
+            return content, prompt, summary
+    except sqlite3.Error as e:
+        logging.error(f"Error fetching item details: {e}")
+        return "", "", ""  # Return empty strings if there's an error
+
+# Handle prompt selection
+def handle_prompt_selection(prompt):
+    return f"You selected: {prompt}"
+
+
+def update_user_prompt(preset_name):
+    details = fetch_prompt_details(preset_name)
+    if details:
+        # Return a dictionary with all details
+        return {
+            "title": details[0],
+            "details": details[1],
+            "system_prompt": details[2],
+            "user_prompt": details[3] if len(details) > 3 else ""
+        }
+    return {"title": "", "details": "", "system_prompt": "", "user_prompt": ""}
+
+def browse_items(search_query, search_type):
+    if search_type == 'Keyword':
+        results = fetch_items_by_keyword(search_query)
+    elif search_type == 'Content':
+        results = fetch_items_by_content(search_query)
+    else:
+        results = fetch_items_by_title_or_url(search_query, search_type)
+    return results
+
+
+def update_dropdown(search_query, search_type):
+    results = browse_items(search_query, search_type)
+    item_options = [f"{item[1]} ({item[2]})" for item in results]
+    new_item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results}
+    print(f"Debug - Update Dropdown - New Item Mapping: {new_item_mapping}")
+    return gr.update(choices=item_options), new_item_mapping
+
+
+
+def get_media_id(selected_item, item_mapping):
+    return item_mapping.get(selected_item)
+
+
+def update_detailed_view(item, item_mapping):
+    # Function to update the detailed view based on selected item
+    if item:
+        item_id = item_mapping.get(item)
+        if item_id:
+            content, prompt, summary = fetch_item_details(item_id)
+            if content or prompt or summary:
+                details_html = "<h4>Details:</h4>"
+                if prompt:
+                    formatted_prompt = format_transcription(prompt)
+                    details_html += f"<h4>Prompt:</h4>{formatted_prompt}</p>"
+                if summary:
+                    formatted_summary = format_transcription(summary)
+                    details_html += f"<h4>Summary:</h4>{formatted_summary}</p>"
+                # Format the transcription content for better readability
+                formatted_content = format_transcription(content)
+                #content_html = f"<h4>Transcription:</h4><div style='white-space: pre-wrap;'>{content}</div>"
+                content_html = f"<h4>Transcription:</h4><div style='white-space: pre-wrap;'>{formatted_content}</div>"
+                return details_html, content_html
+            else:
+                return "No details available.", "No details available."
+        else:
+            return "No item selected", "No item selected"
+    else:
+        return "No item selected", "No item selected"
+
+
+def format_content(content):
+    # Format content using markdown
+    formatted_content = f"```\n{content}\n```"
+    return formatted_content
+
+
+def update_prompt_dropdown():
+    prompt_names = list_prompts()
+    return gr.update(choices=prompt_names)
+
+
+def display_prompt_details(selected_prompt):
+    if selected_prompt:
+        prompts = update_user_prompt(selected_prompt)
+        if prompts["title"]:  # Check if we have any details
+            details_str = f"<h4>Details:</h4><p>{prompts['details']}</p>"
+            system_str = f"<h4>System:</h4><p>{prompts['system_prompt']}</p>"
+            user_str = f"<h4>User:</h4><p>{prompts['user_prompt']}</p>" if prompts['user_prompt'] else ""
+            return details_str + system_str + user_str
+    return "No details available."
+
+def search_media_database(query: str) -> List[Tuple[int, str, str]]:
+    return browse_items(query, 'Title')
+
+
+def load_media_content(media_id: int) -> dict:
+    try:
+        print(f"Debug - Load Media Content - Media ID: {media_id}")
+        item_details = fetch_item_details(media_id)
+        print(f"Debug - Load Media Content - Item Details: \n\n{item_details}\n\n\n\n")
+
+        if isinstance(item_details, tuple) and len(item_details) == 3:
+            content, prompt, summary = item_details
+        else:
+            print(f"Debug - Load Media Content - Unexpected item_details format: \n\n{item_details}\n\n\n\n")
+            content, prompt, summary = "", "", ""
+
+        return {
+            "content": content or "No content available",
+            "prompt": prompt or "No prompt available",
+            "summary": summary or "No summary available"
+        }
+    except Exception as e:
+        print(f"Debug - Load Media Content - Error: {str(e)}")
+        return {"content": "", "prompt": "", "summary": ""}
+
+
+def error_handler(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as e:
+            error_message = f"Error in {func.__name__}: {str(e)}"
+            logging.error(f"{error_message}\n{traceback.format_exc()}")
+            return {"error": error_message, "details": traceback.format_exc()}
+    return wrapper
+
+
+def create_chunking_inputs():
+    chunk_text_by_words_checkbox = gr.Checkbox(label="Chunk Text by Words", value=False, visible=True)
+    max_words_input = gr.Number(label="Max Words", value=300, precision=0, visible=True)
+    chunk_text_by_sentences_checkbox = gr.Checkbox(label="Chunk Text by Sentences", value=False, visible=True)
+    max_sentences_input = gr.Number(label="Max Sentences", value=10, precision=0, visible=True)
+    chunk_text_by_paragraphs_checkbox = gr.Checkbox(label="Chunk Text by Paragraphs", value=False, visible=True)
+    max_paragraphs_input = gr.Number(label="Max Paragraphs", value=5, precision=0, visible=True)
+    chunk_text_by_tokens_checkbox = gr.Checkbox(label="Chunk Text by Tokens", value=False, visible=True)
+    max_tokens_input = gr.Number(label="Max Tokens", value=1000, precision=0, visible=True)
+    gr_semantic_chunk_long_file = gr.Checkbox(label="Semantic Chunking by Sentence similarity", value=False, visible=True)
+    gr_semantic_chunk_long_file_size = gr.Number(label="Max Chunk Size", value=2000, visible=True)
+    gr_semantic_chunk_long_file_overlap = gr.Number(label="Max Chunk Overlap Size", value=100, visible=True)
+    return [chunk_text_by_words_checkbox, max_words_input, chunk_text_by_sentences_checkbox, max_sentences_input,
+            chunk_text_by_paragraphs_checkbox, max_paragraphs_input, chunk_text_by_tokens_checkbox, max_tokens_input]

App_Function_Libraries/Gradio_UI/Import_Functionality.py

@@ -0,0 +1,473 @@
+# Import_Functionality.py
+# Functionality to import content into the DB
+#
+# Imports
+from time import sleep
+import logging
+import re
+import shutil
+import tempfile
+import os
+import traceback
+import zipfile
+#
+# External Imports
+import gradio as gr
+import pypandoc
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import insert_prompt_to_db, load_preset_prompts, import_obsidian_note_to_db, \
+    add_media_to_database
+from App_Function_Libraries.Prompt_Handling import import_prompt_from_file, import_prompts_from_zip#
+from App_Function_Libraries.Summarization_General_Lib import perform_summarization
+
+###################################################################################################################
+#
+# Functions:
+
+logger = logging.getLogger()
+
+
+def import_data(file, title, author, keywords, custom_prompt, summary, auto_summarize, api_name, api_key):
+    if file is None:
+        return "No file uploaded. Please upload a file."
+
+    try:
+        logging.debug(f"File object type: {type(file)}")
+        logging.debug(f"File object attributes: {dir(file)}")
+
+        if hasattr(file, 'name'):
+            file_name = file.name
+        else:
+            file_name = 'unknown_file'
+
+        # Create a temporary file
+        with tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt', encoding='utf-8') as temp_file:
+            if isinstance(file, str):
+                # If file is a string, it's likely file content
+                temp_file.write(file)
+            elif hasattr(file, 'read'):
+                # If file has a 'read' method, it's likely a file-like object
+                content = file.read()
+                if isinstance(content, bytes):
+                    content = content.decode('utf-8')
+                temp_file.write(content)
+            else:
+                # If it's neither a string nor a file-like object, try converting it to a string
+                temp_file.write(str(file))
+
+            temp_file.seek(0)
+            file_content = temp_file.read()
+
+        logging.debug(f"File name: {file_name}")
+        logging.debug(f"File content (first 100 chars): {file_content[:100]}")
+
+        # Create info_dict
+        info_dict = {
+            'title': title or 'Untitled',
+            'uploader': author or 'Unknown',
+        }
+
+        # FIXME - Add chunking support... I added chapter chunking specifically for this...
+        # Create segments (assuming one segment for the entire content)
+        segments = [{'Text': file_content}]
+
+        # Process keywords
+        keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()]
+
+        # Handle summarization
+        if auto_summarize and api_name and api_key:
+            summary = perform_summarization(api_name, file_content, custom_prompt, api_key)
+        elif not summary:
+            summary = "No summary provided"
+
+        # Add to database
+        add_media_to_database(
+            url=file_name,  # Using filename as URL
+            info_dict=info_dict,
+            segments=segments,
+            summary=summary,
+            keywords=keyword_list,
+            custom_prompt_input=custom_prompt,
+            whisper_model="Imported",  # Indicating this was an imported file
+            media_type="document"
+        )
+
+        # Clean up the temporary file
+        os.unlink(temp_file.name)
+
+        return f"File '{file_name}' successfully imported with title '{title}' and author '{author}'."
+    except Exception as e:
+        logging.error(f"Error importing file: {str(e)}")
+        return f"Error importing file: {str(e)}"
+
+
+
+
+
+def process_obsidian_zip(zip_file):
+    with tempfile.TemporaryDirectory() as temp_dir:
+        try:
+            with zipfile.ZipFile(zip_file, 'r') as zip_ref:
+                zip_ref.extractall(temp_dir)
+
+            imported_files, total_files, errors = import_obsidian_vault(temp_dir)
+
+            return imported_files, total_files, errors
+        except zipfile.BadZipFile:
+            error_msg = "The uploaded file is not a valid zip file."
+            logger.error(error_msg)
+            return 0, 0, [error_msg]
+        except Exception as e:
+            error_msg = f"Error processing zip file: {str(e)}\n{traceback.format_exc()}"
+            logger.error(error_msg)
+            return 0, 0, [error_msg]
+        finally:
+            shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+
+def scan_obsidian_vault(vault_path):
+    markdown_files = []
+    for root, dirs, files in os.walk(vault_path):
+        for file in files:
+            if file.endswith('.md'):
+                markdown_files.append(os.path.join(root, file))
+    return markdown_files
+
+
+def parse_obsidian_note(file_path):
+    with open(file_path, 'r', encoding='utf-8') as file:
+        content = file.read()
+
+    frontmatter = {}
+    frontmatter_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
+    if frontmatter_match:
+        frontmatter_text = frontmatter_match.group(1)
+        import yaml
+        frontmatter = yaml.safe_load(frontmatter_text)
+        content = content[frontmatter_match.end():]
+
+    tags = re.findall(r'#(\w+)', content)
+    links = re.findall(r'\[\[(.*?)\]\]', content)
+
+    return {
+        'title': os.path.basename(file_path).replace('.md', ''),
+        'content': content,
+        'frontmatter': frontmatter,
+        'tags': tags,
+        'links': links,
+        'file_path': file_path  # Add this line
+    }
+
+def create_import_single_prompt_tab():
+    with gr.TabItem("Import a Prompt"):
+        gr.Markdown("# Import a prompt into the database")
+
+        with gr.Row():
+            with gr.Column():
+                import_file = gr.File(label="Upload file for import", file_types=["txt", "md"])
+                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
+                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
+                system_input = gr.Textbox(label="System", placeholder="Enter the system message for the prompt", lines=3)
+                user_input = gr.Textbox(label="User", placeholder="Enter the user message for the prompt", lines=3)
+                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords separated by commas")
+                import_button = gr.Button("Import Prompt")
+
+            with gr.Column():
+                import_output = gr.Textbox(label="Import Status")
+                save_button = gr.Button("Save to Database")
+                save_output = gr.Textbox(label="Save Status")
+
+        def handle_import(file):
+            result = import_prompt_from_file(file)
+            if isinstance(result, tuple) and len(result) == 5:
+                title, author, system, user, keywords = result
+                return gr.update(value="File successfully imported. You can now edit the content before saving."), \
+                       gr.update(value=title), gr.update(value=author), gr.update(value=system), \
+                       gr.update(value=user), gr.update(value=", ".join(keywords))
+            else:
+                return gr.update(value=result), gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
+
+        import_button.click(
+            fn=handle_import,
+            inputs=[import_file],
+            outputs=[import_output, title_input, author_input, system_input, user_input, keywords_input]
+        )
+
+        def save_prompt_to_db(title, author, system, user, keywords):
+            keyword_list = [k.strip() for k in keywords.split(',') if k.strip()]
+            return insert_prompt_to_db(title, author, system, user, keyword_list)
+
+        save_button.click(
+            fn=save_prompt_to_db,
+            inputs=[title_input, author_input, system_input, user_input, keywords_input],
+            outputs=save_output
+        )
+
+        def update_prompt_dropdown():
+            return gr.update(choices=load_preset_prompts())
+
+        save_button.click(
+            fn=update_prompt_dropdown,
+            inputs=[],
+            outputs=[gr.Dropdown(label="Select Preset Prompt")]
+        )
+
+def create_import_item_tab():
+    with gr.TabItem("Import Markdown/Text Files"):
+        gr.Markdown("# Import a markdown file or text file into the database")
+        gr.Markdown("...and have it tagged + summarized")
+        with gr.Row():
+            with gr.Column():
+                import_file = gr.File(label="Upload file for import", file_types=["txt", "md"])
+                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
+                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
+                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords, comma-separated")
+                custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                             placeholder="Enter a custom prompt for summarization (optional)")
+                summary_input = gr.Textbox(label="Summary",
+                                       placeholder="Enter a summary or leave blank for auto-summarization", lines=3)
+                auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
+                api_name_input = gr.Dropdown(
+                choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                         "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                label="API for Auto-summarization"
+                )
+                api_key_input = gr.Textbox(label="API Key", type="password")
+            with gr.Column():
+                import_button = gr.Button("Import Data")
+                import_output = gr.Textbox(label="Import Status")
+
+        import_button.click(
+            fn=import_data,
+            inputs=[import_file, title_input, author_input, keywords_input, custom_prompt_input,
+                    summary_input, auto_summarize_checkbox, api_name_input, api_key_input],
+            outputs=import_output
+        )
+
+
+def create_import_multiple_prompts_tab():
+    with gr.TabItem("Import Multiple Prompts"):
+        gr.Markdown("# Import multiple prompts into the database")
+        gr.Markdown("Upload a zip file containing multiple prompt files (txt or md)")
+
+        with gr.Row():
+            with gr.Column():
+                zip_file = gr.File(label="Upload zip file for import", file_types=["zip"])
+                import_button = gr.Button("Import Prompts")
+                prompts_dropdown = gr.Dropdown(label="Select Prompt to Edit", choices=[])
+                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
+                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
+                system_input = gr.Textbox(label="System", placeholder="Enter the system message for the prompt",
+                                          lines=3)
+                user_input = gr.Textbox(label="User", placeholder="Enter the user message for the prompt", lines=3)
+                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords separated by commas")
+
+            with gr.Column():
+                import_output = gr.Textbox(label="Import Status")
+                save_button = gr.Button("Save to Database")
+                save_output = gr.Textbox(label="Save Status")
+                prompts_display = gr.Textbox(label="Identified Prompts")
+
+        def handle_zip_import(zip_file):
+            result = import_prompts_from_zip(zip_file)
+            if isinstance(result, list):
+                prompt_titles = [prompt['title'] for prompt in result]
+                return gr.update(
+                    value="Zip file successfully imported. Select a prompt to edit from the dropdown."), prompt_titles, gr.update(
+                    value="\n".join(prompt_titles)), result
+            else:
+                return gr.update(value=result), [], gr.update(value=""), []
+
+        def handle_prompt_selection(selected_title, prompts):
+            selected_prompt = next((prompt for prompt in prompts if prompt['title'] == selected_title), None)
+            if selected_prompt:
+                return (
+                    selected_prompt['title'],
+                    selected_prompt.get('author', ''),
+                    selected_prompt['system'],
+                    selected_prompt.get('user', ''),
+                    ", ".join(selected_prompt.get('keywords', []))
+                )
+            else:
+                return "", "", "", "", ""
+
+        zip_import_state = gr.State([])
+
+        import_button.click(
+            fn=handle_zip_import,
+            inputs=[zip_file],
+            outputs=[import_output, prompts_dropdown, prompts_display, zip_import_state]
+        )
+
+        prompts_dropdown.change(
+            fn=handle_prompt_selection,
+            inputs=[prompts_dropdown, zip_import_state],
+            outputs=[title_input, author_input, system_input, user_input, keywords_input]
+        )
+
+        def save_prompt_to_db(title, author, system, user, keywords):
+            keyword_list = [k.strip() for k in keywords.split(',') if k.strip()]
+            return insert_prompt_to_db(title, author, system, user, keyword_list)
+
+        save_button.click(
+            fn=save_prompt_to_db,
+            inputs=[title_input, author_input, system_input, user_input, keywords_input],
+            outputs=save_output
+        )
+
+        def update_prompt_dropdown():
+            return gr.update(choices=load_preset_prompts())
+
+        save_button.click(
+            fn=update_prompt_dropdown,
+            inputs=[],
+            outputs=[gr.Dropdown(label="Select Preset Prompt")]
+        )
+
+
+def create_import_obsidian_vault_tab():
+    with gr.TabItem("Import Obsidian Vault"):
+        gr.Markdown("## Import Obsidian Vault")
+        with gr.Row():
+            with gr.Column():
+                vault_path_input = gr.Textbox(label="Obsidian Vault Path (Local)")
+                vault_zip_input = gr.File(label="Upload Obsidian Vault (Zip)")
+            with gr.Column():
+                import_vault_button = gr.Button("Import Obsidian Vault")
+                import_status = gr.Textbox(label="Import Status", interactive=False)
+
+
+    def import_vault(vault_path, vault_zip):
+        if vault_zip:
+            imported, total, errors = process_obsidian_zip(vault_zip.name)
+        elif vault_path:
+            imported, total, errors = import_obsidian_vault(vault_path)
+        else:
+            return "Please provide either a local vault path or upload a zip file."
+
+        status = f"Imported {imported} out of {total} files.\n"
+        if errors:
+            status += f"Encountered {len(errors)} errors:\n" + "\n".join(errors)
+        return status
+
+
+    import_vault_button.click(
+        fn=import_vault,
+        inputs=[vault_path_input, vault_zip_input],
+        outputs=[import_status],
+        show_progress=True
+    )
+
+
+
+# Using pypandoc to convert EPUB to Markdown
+def create_import_book_tab():
+    with gr.TabItem("Import .epub/ebook Files"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Ingest an .epub file using pypandoc")
+                gr.Markdown("...and have it tagged + summarized")
+                gr.Markdown(
+                    "How to remove DRM from your ebooks: https://www.reddit.com/r/Calibre/comments/1ck4w8e/2024_guide_on_removing_drm_from_kobo_kindle_ebooks/")
+                import_file = gr.File(label="Upload file for import", file_types=[".epub"])
+                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
+                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
+                keywords_input = gr.Textbox(label="Keywords(like genre or publish year)",
+                                            placeholder="Enter keywords, comma-separated")
+                system_prompt_input = gr.Textbox(label="System Prompt",
+                                                 lines=3,
+                                                 value=""""
+                                                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                                                    **Bulleted Note Creation Guidelines**
+
+                                                    **Headings**:
+                                                    - Based on referenced topics, not categories like quotes or terms
+                                                    - Surrounded by **bold** formatting 
+                                                    - Not listed as bullet points
+                                                    - No space between headings and list items underneath
+
+                                                    **Emphasis**:
+                                                    - **Important terms** set in bold font
+                                                    - **Text ending in a colon**: also bolded
+
+                                                    **Review**:
+                                                    - Ensure adherence to specified format
+                                                    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                                                """, )
+                custom_prompt_input = gr.Textbox(label="Custom User Prompt",
+                                                 placeholder="Enter a custom user prompt for summarization (optional)")
+                auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
+                             "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    label="API for Auto-summarization"
+                )
+                api_key_input = gr.Textbox(label="API Key", type="password")
+                import_button = gr.Button("Import eBook")
+            with gr.Column():
+                with gr.Row():
+                    import_output = gr.Textbox(label="Import Status")
+
+        def import_epub(epub_file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name,
+                        api_key):
+            try:
+                # Create a temporary directory to store the converted file
+                with tempfile.TemporaryDirectory() as temp_dir:
+                    epub_path = epub_file.name
+                    md_path = os.path.join(temp_dir, "converted.md")
+
+                    # Use pypandoc to convert EPUB to Markdown
+                    output = pypandoc.convert_file(epub_path, 'md', outputfile=md_path)
+
+                    if output != "":
+                        return f"Error converting EPUB: {output}"
+
+                    # Read the converted markdown content
+                    with open(md_path, "r", encoding="utf-8") as md_file:
+                        content = md_file.read()
+
+                    # Now process the content as you would with a text file
+                    return import_data(content, title, author, keywords, system_prompt,
+                                       user_prompt, auto_summarize, api_name, api_key)
+            except Exception as e:
+                return f"Error processing EPUB: {str(e)}"
+
+        import_button.click(
+            fn=import_epub,
+            inputs=[import_file, title_input, author_input, keywords_input, system_prompt_input,
+                    custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input],
+            outputs=import_output
+        )
+
+def import_obsidian_vault(vault_path, progress=gr.Progress()):
+    try:
+        from App_Function_Libraries.Gradio_UI.Export_Functionality import scan_obsidian_vault
+        markdown_files = scan_obsidian_vault(vault_path)
+        total_files = len(markdown_files)
+        imported_files = 0
+        errors = []
+
+        for i, file_path in enumerate(markdown_files):
+            try:
+                note_data = parse_obsidian_note(file_path)
+                success, error_msg = import_obsidian_note_to_db(note_data)
+                if success:
+                    imported_files += 1
+                else:
+                    errors.append(error_msg)
+            except Exception as e:
+                error_msg = f"Error processing {file_path}: {str(e)}"
+                logger.error(error_msg)
+                errors.append(error_msg)
+
+            progress((i + 1) / total_files, f"Imported {imported_files} of {total_files} files")
+            sleep(0.1)  # Small delay to prevent UI freezing
+
+        return imported_files, total_files, errors
+    except Exception as e:
+        error_msg = f"Error scanning vault: {str(e)}\n{traceback.format_exc()}"
+        logger.error(error_msg)
+        return 0, 0, [error_msg]

App_Function_Libraries/Gradio_UI/Introduction_tab.py

@@ -0,0 +1,161 @@
+# Introduction_tab.py
+# Gradio UI functions for the Introduction tab
+
+# Imports
+#
+# External Imports
+import gradio as gr
+
+from App_Function_Libraries.DB_Manager import get_db_config
+
+
+#
+# Local Imports
+
+
+def create_introduction_tab():
+    with (gr.TabItem("Introduction")):
+        db_config = get_db_config()
+        db_type = db_config['type']
+        gr.Markdown(f"# tldw: Your LLM-powered Research Multi-tool (Using {db_type.capitalize()} Database)")
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("""### What can it do?
+                - Transcribe and summarize videos from URLs/Local files
+                - Transcribe and Summarize Audio files/Podcasts (URL/local file)
+                - Summarize articles from URLs/Local notes
+                - Ingest and summarize books(epub/PDF)
+                - Ingest and summarize research papers (PDFs - WIP)
+                - Search and display ingested content + summaries
+                - Create and manage custom prompts
+                - Chat with an LLM of your choice to generate content using the selected item + Prompts
+                - Keyword support for content search and display
+                - Export keywords/items to markdown/CSV(csv is wip)
+                - Import existing notes from Obsidian to the database (Markdown/txt files or a zip containing a collection of files)
+                - View and manage chat history
+                - Writing Tools: Grammar & Style check, Tone Analyzer & Editor, more planned...
+                - RAG (Retrieval-Augmented Generation) support for content generation(think about asking questions about your entire library of items)
+                - More features planned...
+                - All powered by your choice of LLM. 
+                    - Currently supports: Local-LLM(llamafile-server), OpenAI, Anthropic, Cohere, Groq, DeepSeek, OpenRouter, Llama.cpp, Kobold, Ooba, Tabbyapi, VLLM and more to come...
+                - All data is stored locally in a SQLite database for easy access and management.
+                - No trackers (Gradio has some analytics but it's disabled here...)
+                - No ads, no tracking, no BS. Just you and your content.
+                - Open-source and free to use. Contributions welcome!
+                - If you have any thoughts or feedback, please let me know on github or via email.
+                """)
+                gr.Markdown(
+                    """Follow this project at [tl/dw: Too Long, Didn't Watch - Your Personal Research Multi-Tool - GitHub](https://github.com/rmusser01/tldw)""")
+            with gr.Column():
+                gr.Markdown("""### How to use:
+                ##### Quick Start: Just click on the appropriate tab for what you're trying to do and fill in the required fields. Click "Process <video/audio/article/etc>" and wait for the results.
+                #### Simple Instructions
+                - Basic Usage:
+                    - If you don't have an API key/don't know what an LLM is/don't know what an API key is, please look further down the page for information on getting started.
+                    - If you want summaries/chat with an LLM, you'll need:
+                        1. An API key for the LLM API service you want to use, or,
+                        2. A local inference server running an LLM (like llamafile-server/llama.cpp - for instructions on how to do so see the projects README or below), or,
+                        3. A "local" inference server you have access to running an LLM.
+                    - If you just want transcriptions you can ignore the above.
+                    - Select the tab for the task you want to perform
+                    - Fill in the required fields
+                    - Click the "Process" button
+                    - Wait for the results to appear
+                    - Download the results if needed
+                    - Repeat as needed
+                    - As of writing this, the UI is still a work in progress.
+                    - That being said, I plan to replace it all eventually. In the meantime, please have patience.
+                    - The UI is divided into tabs for different tasks.
+                    - Each tab has a set of fields that you can fill in to perform the task.
+                    - Some fields are mandatory, some are optional.
+                    - The fields are mostly self-explanatory, but I will try to add more detailed instructions as I go.
+                #### Detailed Usage:
+                - There are 8 Top-level tabs in the UI. Each tab has a specific set of tasks that you can perform by selecting one of the 'sub-tabs' made available by clicking on the top tab.
+                - The tabs are as follows:
+                    1. Transcription / Summarization / Ingestion - This tab is for processing videos, audio files, articles, books, and PDFs/office docs.
+                    2. Search / Detailed View - This tab is for searching and displaying content from the database. You can also view detailed information about the selected item.
+                    3. Chat with an LLM - This tab is for chatting with an LLM to generate content based on the selected item and prompts.
+                    4. Edit Existing Items - This tab is for editing existing items in the database (Prompts + ingested items).
+                    5. Writing Tools - This tab is for using various writing tools like Grammar & Style check, Tone Analyzer & Editor, etc.
+                    6. Keywords - This tab is for managing keywords for content search and display.
+                    7. Import/Export - This tab is for importing notes from Obsidian and exporting keywords/items to markdown/CSV.
+                    8. Utilities - This tab contains some random utilities that I thought might be useful.
+                - Each sub-tab is responsible for that set of functionality. This is reflected in the codebase as well, where I have split the functionality into separate files for each tab/larger goal.
+                """)
+        with gr.Row():
+            gr.Markdown("""### HELP! I don't know what any of this this shit is!
+            ### DON'T PANIC
+            #### Its ok, you're not alone, most people have no clue what any of this stuff is. 
+            - So let's try and fix that.
+
+            #### Introduction to LLMs:
+            - Non-Technical introduction to Generative AI and LLMs: https://paruir.medium.com/understanding-generative-ai-and-llms-a-non-technical-overview-part-1-788c0eb0dd64
+            - Google's Intro to LLMs: https://developers.google.com/machine-learning/resources/intro-llms#llm_considerations
+            - LLMs 101(coming from a tech background): https://vinija.ai/models/LLM/
+            - LLM Fundamentals / LLM Scientist / LLM Engineer courses(Free): https://github.com/mlabonne/llm-course
+
+            #### Various Phrases & Terms to know
+            - **LLM** - Large Language Model - A type of neural network that can generate human-like text.
+            - **API** - Application Programming Interface - A set of rules and protocols that allows one software application to communicate with another. 
+                * Think of it like a post address for a piece of software. You can send messages to and from it.
+            - **API Key** - A unique identifier that is used to authenticate a user, developer, or calling program to an API.
+                * Like the key to a post office box. You need it to access the contents.
+            - **GUI** - Graphical User Interface - the thing facilitating your interact with this application.
+            - **DB** - Database
+            - **Prompt Engineering** - The process of designing prompts that are used to guide the output of a language model. Is a meme but also very much not.
+            - **Quantization** - The process of converting a continuous range of values into a finite range of discrete values.
+                * https://github.com/ggerganov/llama.cpp/blob/cddae4884c853b1a7ab420458236d666e2e34423/examples/quantize/README.md#L27
+            - **GGUF Files** - GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML. https://github.com/ggerganov/ggml/blob/master/docs/gguf.md
+            - **Inference Engine** - A software system that is designed to execute a model that has been trained by a machine learning algorithm. Llama.cpp and Kobold.cpp are examples of inference engines.
+            - **Abliteration** - https://huggingface.co/blog/mlabonne/abliteration
+            """)
+        with gr.Row():
+            gr.Markdown("""### Ok cool, but how do I get started? I don't have an API key or a local server running...
+                #### Great, glad you asked! Getting Started:
+                - **Getting an API key for a commercial services provider:
+                    - **OpenAI:**
+                        * https://platform.openai.com/docs/quickstart
+                    - **Anthropic:**
+                        * https://docs.anthropic.com/en/api/getting-started
+                    - **Cohere:**
+                        * https://docs.cohere.com/
+                        * They offer 1k free requests a month(up to 1million tokens total I think?), so you can try it out without paying.
+                    - **Groq:**
+                        * https://console.groq.com/keys
+                        * Offer an account with free credits to try out their service. No idea how much you get.
+                    - **DeepSeek:**
+                        * https://platform.deepseek.com/ (Chinese-hosted/is in english)
+                    - **OpenRouter:**
+                        * https://openrouter.ai/
+                    - **Mistral:**
+                        * https://console.mistral.ai/
+                - **Choosing a Model to download**
+                    - You'll first need to select a model you want to use with the server.
+                        - Keep in mind that the model you select will determine the quality of the output you get, and that models run fastest when offloaded fully to your GPU.
+                        * So this means that you can run a large model (Command-R) on CPU+System RAM, but you're gonna see a massive performance hit. Not saying its unusable, but it's not ideal.
+                        * With that in mind, I would recommend an abliterated version of Meta's Llama3.1 model for most tasks. (Abliterated since it won't refuse requests)
+                        * I say this because of the general quality of the model + it's context size.
+                        * You can find the model here: https://huggingface.co/mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated-GGUF
+                        * And the Q8 quant(total size 8.6GB): https://huggingface.co/mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated-GGUF/resolve/main/meta-llama-3.1-8b-instruct-abliterated.Q8_0.gguf?download=true
+                - **Local Inference Server:**
+                    - **Llamafile-Server (wrapper for llama.cpp):**
+                        * Run this script with the `--local_llm` argument next time, and you'll be walked through setting up a local instance of llamafile-server.
+                    - **Llama.cpp Inference Engine:**
+                        * Download the latest release for your platform here: https://github.com/ggerganov/llama.cpp/releases
+                        * Windows: `llama-<release_number>-bin-win-cuda-cu<11.7.1 or 12.2.0 - version depends on installed cuda>-x64.zip`
+                            * Run it: `llama-server.exe --model <path_to_model> -ctx 8192 -ngl 999` 
+                                - `-ctx 8192` sets the context size to 8192 tokens, `-ngl 999` sets the number of layers to offload to the GPU to 999. (essentially ensuring we only use our GPU and not CPU for processing)
+                        * Macos: `llama-<release_number>-bin-macos-arm64.zip - for Apple Silicon / `llama-<release_number>-bin-macos-x64.zip` - for Intel Macs
+                            * Run it: `llama-server --model <path_to_model> -ctx 8192 -ngl 999` 
+                                - `-ctx 8192` sets the context size to 8192 tokens, `-ngl 999` sets the number of layers to offload to the GPU to 999. (essentially ensuring we only use our GPU and not CPU for processing)
+                        * Linux: You can probably figure it out.
+                    - **Kobold.cpp Server:**
+                        1. Download from here: https://github.com/LostRuins/koboldcpp/releases/latest
+                        2. `Double click KoboldCPP.exe and select model OR run "KoboldCPP.exe --help" in CMD prompt to get command line arguments for more control.`
+                        3. `Generally you don't have to change much besides the Presets and GPU Layers. Run with CuBLAS or CLBlast for GPU acceleration.`
+                        4. `Select your GGUF or GGML model you downloaded earlier, and connect to the displayed URL once it finishes loading.`
+                    - **Linux**
+                        1. `On Linux, we provide a koboldcpp-linux-x64 PyInstaller prebuilt binary on the releases page for modern systems. Simply download and run the binary.`
+                            * Alternatively, you can also install koboldcpp to the current directory by running the following terminal command: `curl -fLo koboldcpp https://github.com/LostRuins/koboldcpp/releases/latest/download/koboldcpp-linux-x64 && chmod +x koboldcpp`
+                        2. When you can't use the precompiled binary directly, we provide an automated build script which uses conda to obtain all dependencies, and generates (from source) a ready-to-use a pyinstaller binary for linux users. Simply execute the build script with `./koboldcpp.sh dist` and run the generated binary.
+            """)

App_Function_Libraries/Gradio_UI/Keywords.py

@@ -0,0 +1,65 @@
+# Keywords.py
+# Purpose: This file contains the functions to create the Keywords tab in the Gradio UI.
+#
+# The Keywords tab allows the user to add, delete, view, and export keywords from the database.
+#
+# Imports:
+
+#
+# External Imports
+import gradio as gr
+#
+# Internal Imports
+from App_Function_Libraries.DB_Manager import add_keyword, delete_keyword, keywords_browser_interface, export_keywords_to_csv
+#
+#
+######################################################################################################################
+#
+# Functions:
+
+
+def create_export_keywords_tab():
+    with gr.Tab("Export Keywords"):
+        with gr.Row():
+            with gr.Column():
+                export_keywords_button = gr.Button("Export Keywords")
+            with gr.Column():
+                export_keywords_output = gr.File(label="Download Exported Keywords")
+                export_keywords_status = gr.Textbox(label="Export Status")
+
+            export_keywords_button.click(
+                fn=export_keywords_to_csv,
+                outputs=[export_keywords_status, export_keywords_output]
+            )
+
+def create_view_keywords_tab():
+    with gr.TabItem("View Keywords"):
+        gr.Markdown("# Browse Keywords")
+        with gr.Column():
+            browse_output = gr.Markdown()
+            browse_button = gr.Button("View Existing Keywords")
+            browse_button.click(fn=keywords_browser_interface, outputs=browse_output)
+
+
+def create_add_keyword_tab():
+    with gr.TabItem("Add Keywords"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Add Keywords to the Database")
+                add_input = gr.Textbox(label="Add Keywords (comma-separated)", placeholder="Enter keywords here...")
+                add_button = gr.Button("Add Keywords")
+            with gr.Row():
+                add_output = gr.Textbox(label="Result")
+                add_button.click(fn=add_keyword, inputs=add_input, outputs=add_output)
+
+
+def create_delete_keyword_tab():
+    with gr.Tab("Delete Keywords"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Delete Keywords from the Database")
+                delete_input = gr.Textbox(label="Delete Keyword", placeholder="Enter keyword to delete here...")
+                delete_button = gr.Button("Delete Keyword")
+            with gr.Row():
+                delete_output = gr.Textbox(label="Result")
+                delete_button.click(fn=delete_keyword, inputs=delete_input, outputs=delete_output)

App_Function_Libraries/Gradio_UI/Llamafile_tab.py

@@ -0,0 +1,122 @@
+# Llamafile_tab.py
+# Description: Functions relating to the Llamafile tab
+#
+# Imports
+import os
+import glob
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Llamafile import start_llamafile
+#
+#######################################################################################################################
+#
+# Functions:
+
+
+def create_chat_with_llamafile_tab():
+    def get_model_files(directory):
+        pattern = os.path.join(directory, "*.{gguf,llamafile}")
+        return [os.path.basename(f) for f in glob.glob(pattern)]
+
+    def update_dropdowns():
+        current_dir_models = get_model_files(".")
+        parent_dir_models = get_model_files("..")
+        return (
+            {"choices": current_dir_models, "value": None},
+            {"choices": parent_dir_models, "value": None}
+        )
+
+    with gr.TabItem("Local LLM with Llamafile"):
+        gr.Markdown("# Settings for Llamafile")
+        with gr.Row():
+            with gr.Column():
+                am_noob = gr.Checkbox(label="Check this to enable sane defaults", value=False, visible=True)
+                # FIXME - these get deleted at some point?
+                advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Enable to show all settings", value=False)
+
+
+            with gr.Column():
+                # FIXME - make this actually work
+                model_checked = gr.Checkbox(label="Enable Setting Local LLM Model Path", value=False, visible=True)
+                current_dir_dropdown = gr.Dropdown(
+                    label="Select Model from Current Directory (.)",
+                    choices=[],  # Start with an empty list
+                    visible=True
+                )
+                parent_dir_dropdown = gr.Dropdown(
+                    label="Select Model from Parent Directory (..)",
+                    choices=[],  # Start with an empty list
+                    visible=True
+                )
+                refresh_button = gr.Button("Refresh Model Lists")
+                model_value = gr.Textbox(label="Selected Model File", value="", visible=True)
+        with gr.Row():
+            with gr.Column():
+                ngl_checked = gr.Checkbox(label="Enable Setting GPU Layers", value=False, visible=True)
+                ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True)
+                advanced_inputs = create_llamafile_advanced_inputs()
+            with gr.Column():
+                start_button = gr.Button("Start Llamafile")
+                stop_button = gr.Button("Stop Llamafile (doesn't work)")
+                output_display = gr.Markdown()
+
+
+        def update_model_value(current_dir_model, parent_dir_model):
+            if current_dir_model:
+                return current_dir_model
+            elif parent_dir_model:
+                return os.path.join("..", parent_dir_model)
+            else:
+                return ""
+
+        current_dir_dropdown.change(
+            fn=update_model_value,
+            inputs=[current_dir_dropdown, parent_dir_dropdown],
+            outputs=model_value
+        )
+        parent_dir_dropdown.change(
+            fn=update_model_value,
+            inputs=[current_dir_dropdown, parent_dir_dropdown],
+            outputs=model_value
+        )
+
+        refresh_button.click(
+            fn=update_dropdowns,
+            inputs=[],
+            outputs=[current_dir_dropdown, parent_dir_dropdown]
+        )
+
+        start_button.click(
+            fn=start_llamafile,
+            inputs=[am_noob, model_checked, model_value, ngl_checked, ngl_value] + advanced_inputs,
+            outputs=output_display
+        )
+
+
+def create_llamafile_advanced_inputs():
+    verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False)
+    threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False)
+    threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False)
+    http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False)
+    http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False)
+    hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False)
+    hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False)
+    hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False)
+    hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False)
+    ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False)
+    ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False)
+    host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False)
+    host_value = gr.Textbox(label="Host IP Address", value="", visible=False)
+    port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False)
+    port_value = gr.Number(label="Port Number", value=None, precision=0, visible=False)
+
+    return [verbose_checked, threads_checked, threads_value, http_threads_checked, http_threads_value,
+            hf_repo_checked, hf_repo_value, hf_file_checked, hf_file_value, ctx_size_checked, ctx_size_value,
+            host_checked, host_value, port_checked, port_value]
+
+#
+# End of Llamafile_tab.py
+#########################################################################################################################

App_Function_Libraries/Gradio_UI/Media_edit.py

@@ -0,0 +1,273 @@
+# Media_edit.py
+# Functions for Gradio Media_Edit UI
+
+# Imports
+import logging
+import uuid
+
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import add_prompt, update_media_content, db, add_or_update_prompt, \
+    load_prompt_details
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_prompt_dropdown
+from App_Function_Libraries.SQLite_DB import fetch_item_details
+
+
+def create_media_edit_tab():
+    with gr.TabItem("Edit Existing Items"):
+        gr.Markdown("# Search and Edit Media Items")
+
+        with gr.Row():
+            search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+            search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
+            search_button = gr.Button("Search")
+
+        with gr.Row():
+            items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+            item_mapping = gr.State({})
+
+        content_input = gr.Textbox(label="Edit Content", lines=10)
+        prompt_input = gr.Textbox(label="Edit Prompt", lines=3)
+        summary_input = gr.Textbox(label="Edit Summary", lines=5)
+
+        update_button = gr.Button("Update Media Content")
+        status_message = gr.Textbox(label="Status", interactive=False)
+
+        search_button.click(
+            fn=update_dropdown,
+            inputs=[search_query_input, search_type_input],
+            outputs=[items_output, item_mapping]
+        )
+
+        def load_selected_media_content(selected_item, item_mapping):
+            if selected_item and item_mapping and selected_item in item_mapping:
+                media_id = item_mapping[selected_item]
+                # FIXME - fetch_item_details is not handled by DB_Manager!
+                content, prompt, summary = fetch_item_details(media_id)
+                return content, prompt, summary
+            return "No item selected or invalid selection", "", ""
+
+        items_output.change(
+            fn=load_selected_media_content,
+            inputs=[items_output, item_mapping],
+            outputs=[content_input, prompt_input, summary_input]
+        )
+
+        update_button.click(
+            fn=update_media_content,
+            inputs=[items_output, item_mapping, content_input, prompt_input, summary_input],
+            outputs=status_message
+        )
+
+
+def create_media_edit_and_clone_tab():
+    with gr.TabItem("Clone and Edit Existing Items"):
+        gr.Markdown("# Search, Edit, and Clone Existing Items")
+
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
+                                         label="Search By")
+            with gr.Column():
+                search_button = gr.Button("Search")
+                clone_button = gr.Button("Clone Item")
+            save_clone_button = gr.Button("Save Cloned Item", visible=False)
+        with gr.Row():
+            items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+            item_mapping = gr.State({})
+
+        content_input = gr.Textbox(label="Edit Content", lines=10)
+        prompt_input = gr.Textbox(label="Edit Prompt", lines=3)
+        summary_input = gr.Textbox(label="Edit Summary", lines=5)
+        new_title_input = gr.Textbox(label="New Title (for cloning)", visible=False)
+        status_message = gr.Textbox(label="Status", interactive=False)
+
+        search_button.click(
+            fn=update_dropdown,
+            inputs=[search_query_input, search_type_input],
+            outputs=[items_output, item_mapping]
+        )
+
+        def load_selected_media_content(selected_item, item_mapping):
+            if selected_item and item_mapping and selected_item in item_mapping:
+                media_id = item_mapping[selected_item]
+                content, prompt, summary = fetch_item_details(media_id)
+                return content, prompt, summary, gr.update(visible=True), gr.update(visible=False)
+            return "No item selected or invalid selection", "", "", gr.update(visible=False), gr.update(visible=False)
+
+        items_output.change(
+            fn=load_selected_media_content,
+            inputs=[items_output, item_mapping],
+            outputs=[content_input, prompt_input, summary_input, clone_button, save_clone_button]
+        )
+
+        def prepare_for_cloning(selected_item):
+            return gr.update(value=f"Copy of {selected_item}", visible=True), gr.update(visible=True)
+
+        clone_button.click(
+            fn=prepare_for_cloning,
+            inputs=[items_output],
+            outputs=[new_title_input, save_clone_button]
+        )
+
+        def save_cloned_item(selected_item, item_mapping, content, prompt, summary, new_title):
+            if selected_item and item_mapping and selected_item in item_mapping:
+                original_media_id = item_mapping[selected_item]
+                try:
+                    with db.get_connection() as conn:
+                        cursor = conn.cursor()
+
+                        # Fetch the original item's details
+                        cursor.execute("SELECT type, url FROM Media WHERE id = ?", (original_media_id,))
+                        original_type, original_url = cursor.fetchone()
+
+                        # Generate a new unique URL
+                        new_url = f"{original_url}_clone_{uuid.uuid4().hex[:8]}"
+
+                        # Insert new item into Media table
+                        cursor.execute("""
+                            INSERT INTO Media (title, content, url, type)
+                            VALUES (?, ?, ?, ?)
+                        """, (new_title, content, new_url, original_type))
+
+                        new_media_id = cursor.lastrowid
+
+                        # Insert new item into MediaModifications table
+                        cursor.execute("""
+                            INSERT INTO MediaModifications (media_id, prompt, summary, modification_date)
+                            VALUES (?, ?, ?, CURRENT_TIMESTAMP)
+                        """, (new_media_id, prompt, summary))
+
+                        # Copy keywords from the original item
+                        cursor.execute("""
+                            INSERT INTO MediaKeywords (media_id, keyword_id)
+                            SELECT ?, keyword_id
+                            FROM MediaKeywords
+                            WHERE media_id = ?
+                        """, (new_media_id, original_media_id))
+
+                        # Update full-text search index
+                        cursor.execute("""
+                            INSERT INTO media_fts (rowid, title, content)
+                            VALUES (?, ?, ?)
+                        """, (new_media_id, new_title, content))
+
+                        conn.commit()
+
+                    return f"Cloned item saved successfully with ID: {new_media_id}", gr.update(
+                        visible=False), gr.update(visible=False)
+                except Exception as e:
+                    logging.error(f"Error saving cloned item: {e}")
+                    return f"Error saving cloned item: {str(e)}", gr.update(visible=True), gr.update(visible=True)
+            else:
+                return "No item selected or invalid selection", gr.update(visible=True), gr.update(visible=True)
+
+        save_clone_button.click(
+            fn=save_cloned_item,
+            inputs=[items_output, item_mapping, content_input, prompt_input, summary_input, new_title_input],
+            outputs=[status_message, new_title_input, save_clone_button]
+        )
+
+
+def create_prompt_edit_tab():
+    with gr.TabItem("Edit Prompts"):
+        with gr.Row():
+            with gr.Column():
+                prompt_dropdown = gr.Dropdown(
+                    label="Select Prompt",
+                    choices=[],
+                    interactive=True
+                )
+                prompt_list_button = gr.Button("List Prompts")
+
+            with gr.Column():
+                title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title")
+                description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3)
+                system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3)
+                user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3)
+                add_prompt_button = gr.Button("Add/Update Prompt")
+                add_prompt_output = gr.HTML()
+
+        # Event handlers
+        prompt_list_button.click(
+            fn=update_prompt_dropdown,
+            outputs=prompt_dropdown
+        )
+
+        add_prompt_button.click(
+            fn=add_or_update_prompt,
+            inputs=[title_input, description_input, system_prompt_input, user_prompt_input],
+            outputs=add_prompt_output
+        )
+
+        # Load prompt details when selected
+        prompt_dropdown.change(
+            fn=load_prompt_details,
+            inputs=[prompt_dropdown],
+            outputs=[title_input, description_input, system_prompt_input, user_prompt_input]
+        )
+
+
+def create_prompt_clone_tab():
+    with gr.TabItem("Clone and Edit Prompts"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Clone and Edit Prompts")
+                prompt_dropdown = gr.Dropdown(
+                    label="Select Prompt",
+                    choices=[],
+                    interactive=True
+                )
+                prompt_list_button = gr.Button("List Prompts")
+
+            with gr.Column():
+                title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title")
+                description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3)
+                system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3)
+                user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3)
+                clone_prompt_button = gr.Button("Clone Selected Prompt")
+                save_cloned_prompt_button = gr.Button("Save Cloned Prompt", visible=False)
+                add_prompt_output = gr.HTML()
+
+        # Event handlers
+        prompt_list_button.click(
+            fn=update_prompt_dropdown,
+            outputs=prompt_dropdown
+        )
+
+        # Load prompt details when selected
+        prompt_dropdown.change(
+            fn=load_prompt_details,
+            inputs=[prompt_dropdown],
+            outputs=[title_input, description_input, system_prompt_input, user_prompt_input]
+        )
+
+        def prepare_for_cloning(selected_prompt):
+            if selected_prompt:
+                return gr.update(value=f"Copy of {selected_prompt}"), gr.update(visible=True)
+            return gr.update(), gr.update(visible=False)
+
+        clone_prompt_button.click(
+            fn=prepare_for_cloning,
+            inputs=[prompt_dropdown],
+            outputs=[title_input, save_cloned_prompt_button]
+        )
+
+        def save_cloned_prompt(title, description, system_prompt, user_prompt):
+            try:
+                result = add_prompt(title, description, system_prompt, user_prompt)
+                if result == "Prompt added successfully.":
+                    return result, gr.update(choices=update_prompt_dropdown())
+                else:
+                    return result, gr.update()
+            except Exception as e:
+                return f"Error saving cloned prompt: {str(e)}", gr.update()
+
+        save_cloned_prompt_button.click(
+            fn=save_cloned_prompt,
+            inputs=[title_input, description_input, system_prompt_input, user_prompt_input],
+            outputs=[add_prompt_output, prompt_dropdown]
+        )

App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py

@@ -0,0 +1,152 @@
+# PDF_ingestion_tab.py
+# Gradio UI for ingesting PDFs into the database
+import os
+import shutil
+import tempfile
+
+# Imports
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import load_preset_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+from App_Function_Libraries.PDF_Ingestion_Lib import extract_metadata_from_pdf, extract_text_and_format_from_pdf, \
+    process_and_cleanup_pdf
+#
+#
+########################################################################################################################
+#
+# Functions:
+
+def create_pdf_ingestion_tab():
+    with gr.TabItem("PDF Ingestion"):
+        # TODO - Add functionality to extract metadata from pdf as part of conversion process in marker
+        gr.Markdown("# Ingest PDF Files and Extract Metadata")
+        with gr.Row():
+            with gr.Column():
+                pdf_file_input = gr.File(label="Uploaded PDF File", file_types=[".pdf"], visible=False)
+                pdf_upload_button = gr.UploadButton("Click to Upload PDF", file_types=[".pdf"])
+                pdf_title_input = gr.Textbox(label="Title (Optional)")
+                pdf_author_input = gr.Textbox(label="Author (Optional)")
+                pdf_keywords_input = gr.Textbox(label="Keywords (Optional, comma-separated)")
+                with gr.Row():
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=load_preset_prompts(),
+                                                visible=False)
+                with gr.Row():
+                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="""
+<s>You are a bulleted notes specialist.
+[INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""",
+                                                     lines=3,
+                                                     visible=False)
+
+                custom_prompt_checkbox.change(
+                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    inputs=[custom_prompt_checkbox],
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+                preset_prompt_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[preset_prompt_checkbox],
+                    outputs=[preset_prompt]
+                )
+
+                def update_prompts(preset_name):
+                    prompts = update_user_prompt(preset_name)
+                    return (
+                        gr.update(value=prompts["user_prompt"], visible=True),
+                        gr.update(value=prompts["system_prompt"], visible=True)
+                    )
+
+                preset_prompt.change(
+                    update_prompts,
+                    inputs=preset_prompt,
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+
+                pdf_ingest_button = gr.Button("Ingest PDF")
+
+                pdf_upload_button.upload(fn=lambda file: file, inputs=pdf_upload_button, outputs=pdf_file_input)
+            with gr.Column():
+                pdf_result_output = gr.Textbox(label="Result")
+
+            pdf_ingest_button.click(
+                fn=process_and_cleanup_pdf,
+                inputs=[pdf_file_input, pdf_title_input, pdf_author_input, pdf_keywords_input],
+                outputs=pdf_result_output
+            )
+
+
+def test_pdf_ingestion(pdf_file):
+    if pdf_file is None:
+        return "No file uploaded", ""
+
+    try:
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Create a path for the temporary PDF file
+            temp_path = os.path.join(temp_dir, "temp.pdf")
+
+            # Copy the contents of the uploaded file to the temporary file
+            shutil.copy(pdf_file.name, temp_path)
+
+            # Extract text and convert to Markdown
+            markdown_text = extract_text_and_format_from_pdf(temp_path)
+
+            # Extract metadata from PDF
+            metadata = extract_metadata_from_pdf(temp_path)
+
+            # Use metadata for title and author if not provided
+            title = metadata.get('title', os.path.splitext(os.path.basename(pdf_file.name))[0])
+            author = metadata.get('author', 'Unknown')
+
+        result = f"PDF '{title}' by {author} processed successfully."
+        return result, markdown_text
+    except Exception as e:
+        return f"Error ingesting PDF: {str(e)}", ""
+
+def create_pdf_ingestion_test_tab():
+    with gr.TabItem("Test PDF Ingestion"):
+        with gr.Row():
+            with gr.Column():
+                pdf_file_input = gr.File(label="Upload PDF for testing")
+                test_button = gr.Button("Test PDF Ingestion")
+            with gr.Column():
+                test_output = gr.Textbox(label="Test Result")
+                pdf_content_output = gr.Textbox(label="PDF Content", lines=200)
+        test_button.click(
+            fn=test_pdf_ingestion,
+            inputs=[pdf_file_input],
+            outputs=[test_output, pdf_content_output]
+        )
+

App_Function_Libraries/Gradio_UI/Podcast_tab.py

@@ -0,0 +1,164 @@
+# Podcast_tab.py
+# Description: Gradio UI for ingesting podcasts into the database
+#
+# Imports
+#
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Audio_Files import process_podcast
+from App_Function_Libraries.DB_Manager import load_preset_prompts
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models, update_user_prompt
+
+
+#
+########################################################################################################################
+#
+# Functions:
+
+
+def create_podcast_tab():
+    with gr.TabItem("Podcast"):
+        gr.Markdown("# Podcast Transcription and Ingestion")
+        with gr.Row():
+            with gr.Column():
+                podcast_url_input = gr.Textbox(label="Podcast URL", placeholder="Enter the podcast URL here")
+                podcast_title_input = gr.Textbox(label="Podcast Title", placeholder="Will be auto-detected if possible")
+                podcast_author_input = gr.Textbox(label="Podcast Author", placeholder="Will be auto-detected if possible")
+
+                podcast_keywords_input = gr.Textbox(
+                    label="Keywords",
+                    placeholder="Enter keywords here (comma-separated, include series name if applicable)",
+                    value="podcast,audio",
+                    elem_id="podcast-keywords-input"
+                )
+
+                with gr.Row():
+                    podcast_custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=load_preset_prompts(),
+                                                visible=False)
+                with gr.Row():
+                    podcast_custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+""",
+                                                     lines=3,
+                                                     visible=False)
+
+                podcast_custom_prompt_checkbox.change(
+                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    inputs=[podcast_custom_prompt_checkbox],
+                    outputs=[podcast_custom_prompt_input, system_prompt_input]
+                )
+                preset_prompt_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[preset_prompt_checkbox],
+                    outputs=[preset_prompt]
+                )
+
+                def update_prompts(preset_name):
+                    prompts = update_user_prompt(preset_name)
+                    return (
+                        gr.update(value=prompts["user_prompt"], visible=True),
+                        gr.update(value=prompts["system_prompt"], visible=True)
+                    )
+
+                preset_prompt.change(
+                    update_prompts,
+                    inputs=preset_prompt,
+                    outputs=[podcast_custom_prompt_input, system_prompt_input]
+                )
+
+                podcast_api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp",
+                             "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                    value=None,
+                    label="API Name for Summarization (Optional)"
+                )
+                podcast_api_key_input = gr.Textbox(label="API Key (if required)", type="password")
+                podcast_whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
+
+                keep_original_input = gr.Checkbox(label="Keep original audio file", value=False)
+                enable_diarization_input = gr.Checkbox(label="Enable speaker diarization", value=False)
+
+                use_cookies_input = gr.Checkbox(label="Use cookies for yt-dlp", value=False)
+                cookies_input = gr.Textbox(
+                    label="yt-dlp Cookies",
+                    placeholder="Paste your cookies here (JSON format)",
+                    lines=3,
+                    visible=False
+                )
+
+                use_cookies_input.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[use_cookies_input],
+                    outputs=[cookies_input]
+                )
+
+                chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False)
+                with gr.Row(visible=False) as chunking_options_box:
+                    gr.Markdown("### Chunking Options")
+                    with gr.Column():
+                        chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], label="Chunking Method")
+                        max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size")
+                        chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
+                        use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking")
+                        use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking")
+                        chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], label="Chunking Language")
+
+                chunking_options_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[chunking_options_checkbox],
+                    outputs=[chunking_options_box]
+                )
+
+                podcast_process_button = gr.Button("Process Podcast")
+
+            with gr.Column():
+                podcast_progress_output = gr.Textbox(label="Progress")
+                podcast_error_output = gr.Textbox(label="Error Messages")
+                podcast_transcription_output = gr.Textbox(label="Transcription")
+                podcast_summary_output = gr.Textbox(label="Summary")
+                download_transcription = gr.File(label="Download Transcription as JSON")
+                download_summary = gr.File(label="Download Summary as Text")
+
+        podcast_process_button.click(
+            fn=process_podcast,
+            inputs=[podcast_url_input, podcast_title_input, podcast_author_input,
+                    podcast_keywords_input, podcast_custom_prompt_input, podcast_api_name_input,
+                    podcast_api_key_input, podcast_whisper_model_input, keep_original_input,
+                    enable_diarization_input, use_cookies_input, cookies_input,
+                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                    use_multi_level_chunking, chunk_language],
+            outputs=[podcast_progress_output, podcast_transcription_output, podcast_summary_output,
+                     podcast_title_input, podcast_author_input, podcast_keywords_input, podcast_error_output,
+                     download_transcription, download_summary]
+        )

App_Function_Libraries/Gradio_UI/Re_summarize_tab.py

@@ -0,0 +1,268 @@
+# Re_summarize_tab.py
+# Gradio UI for Re-summarizing items in the database
+#
+# Imports
+import json
+import logging
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process
+from App_Function_Libraries.DB_Manager import update_media_content, load_preset_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import fetch_item_details, fetch_items_by_keyword, \
+    fetch_items_by_content, fetch_items_by_title_or_url
+from App_Function_Libraries.Summarization_General_Lib import summarize_chunk
+from App_Function_Libraries.Utils import load_comprehensive_config
+#
+#
+######################################################################################################################
+#
+# Functions:
+
+def create_resummary_tab():
+    with gr.TabItem("Re-Summarize"):
+        gr.Markdown("# Re-Summarize Existing Content")
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
+                search_button = gr.Button("Search")
+
+                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+                item_mapping = gr.State({})
+
+                with gr.Row():
+                    api_name_input = gr.Dropdown(
+                        choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                        value="Local-LLM", label="API Name")
+                    api_key_input = gr.Textbox(label="API Key", placeholder="Enter your API key here", type="password")
+
+                chunking_options_checkbox = gr.Checkbox(label="Use Chunking", value=False)
+                with gr.Row(visible=False) as chunking_options_box:
+                    chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens', 'chapters'],
+                                               label="Chunking Method", value='words')
+                    max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size")
+                    chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
+
+                with gr.Row():
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=load_preset_prompts(),
+                                                visible=False)
+                with gr.Row():
+                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+""",
+                                                     lines=3,
+                                                     visible=False)
+
+                def update_prompts(preset_name):
+                    prompts = update_user_prompt(preset_name)
+                    return (
+                        gr.update(value=prompts["user_prompt"], visible=True),
+                        gr.update(value=prompts["system_prompt"], visible=True)
+                    )
+
+                preset_prompt.change(
+                    update_prompts,
+                    inputs=preset_prompt,
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+
+                resummarize_button = gr.Button("Re-Summarize")
+            with gr.Column():
+                result_output = gr.Textbox(label="Result")
+
+        custom_prompt_checkbox.change(
+            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+            inputs=[custom_prompt_checkbox],
+            outputs=[custom_prompt_input, system_prompt_input]
+        )
+        preset_prompt_checkbox.change(
+            fn=lambda x: gr.update(visible=x),
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt]
+        )
+
+    # Connect the UI elements
+    search_button.click(
+        fn=update_resummarize_dropdown,
+        inputs=[search_query_input, search_type_input],
+        outputs=[items_output, item_mapping]
+    )
+
+    chunking_options_checkbox.change(
+        fn=lambda x: gr.update(visible=x),
+        inputs=[chunking_options_checkbox],
+        outputs=[chunking_options_box]
+    )
+
+    custom_prompt_checkbox.change(
+        fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+        inputs=[custom_prompt_checkbox],
+        outputs=[custom_prompt_input, system_prompt_input]
+    )
+
+    resummarize_button.click(
+        fn=resummarize_content_wrapper,
+        inputs=[items_output, item_mapping, api_name_input, api_key_input, chunking_options_checkbox, chunk_method,
+                max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt_input],
+        outputs=result_output
+    )
+
+    return search_query_input, search_type_input, search_button, items_output, item_mapping, api_name_input, api_key_input, chunking_options_checkbox, chunking_options_box, chunk_method, max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt_input, resummarize_button, result_output
+
+
+def update_resummarize_dropdown(search_query, search_type):
+    if search_type in ['Title', 'URL']:
+        results = fetch_items_by_title_or_url(search_query, search_type)
+    elif search_type == 'Keyword':
+        results = fetch_items_by_keyword(search_query)
+    else:  # Content
+        results = fetch_items_by_content(search_query)
+
+    item_options = [f"{item[1]} ({item[2]})" for item in results]
+    item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results}
+    logging.debug(f"item_options: {item_options}")
+    logging.debug(f"item_mapping: {item_mapping}")
+    return gr.update(choices=item_options), item_mapping
+
+
+def resummarize_content_wrapper(selected_item, item_mapping, api_name, api_key=None, chunking_options_checkbox=None, chunk_method=None,
+                                max_chunk_size=None, chunk_overlap=None, custom_prompt_checkbox=None, custom_prompt=None):
+    logging.debug(f"resummarize_content_wrapper called with item_mapping type: {type(item_mapping)}")
+    logging.debug(f"selected_item: {selected_item}")
+
+    if not selected_item or not api_name:
+        return "Please select an item and provide API details."
+
+    # Handle potential string representation of item_mapping
+    if isinstance(item_mapping, str):
+        try:
+            item_mapping = json.loads(item_mapping)
+        except json.JSONDecodeError:
+            return f"Error: item_mapping is a string but not valid JSON. Value: {item_mapping[:100]}..."
+
+    if not isinstance(item_mapping, dict):
+        return f"Error: item_mapping is not a dictionary or valid JSON string. Type: {type(item_mapping)}"
+
+    media_id = item_mapping.get(selected_item)
+    if not media_id:
+        return f"Invalid selection. Selected item: {selected_item}, Available items: {list(item_mapping.keys())[:5]}..."
+
+    content, old_prompt, old_summary = fetch_item_details(media_id)
+
+    if not content:
+        return "No content available for re-summarization."
+
+    # Prepare chunking options
+    chunk_options = {
+        'method': chunk_method,
+        'max_size': int(max_chunk_size) if max_chunk_size is not None else None,
+        'overlap': int(chunk_overlap) if chunk_overlap is not None else None,
+        'language': 'english',
+        'adaptive': True,
+        'multi_level': False,
+    } if chunking_options_checkbox else None
+
+    # Prepare summarization prompt
+    summarization_prompt = custom_prompt if custom_prompt_checkbox and custom_prompt else None
+
+    logging.debug(f"Calling resummarize_content with media_id: {media_id}")
+    # Call the resummarize_content function
+    result = resummarize_content(selected_item, item_mapping, content, api_name, api_key, chunk_options, summarization_prompt)
+
+    return result
+
+
+# FIXME - should be moved...
+def resummarize_content(selected_item, item_mapping, content, api_name, api_key=None, chunk_options=None, summarization_prompt=None):
+    logging.debug(f"resummarize_content called with selected_item: {selected_item}")
+    # Load configuration
+    config = load_comprehensive_config()
+
+    # Chunking logic
+    if chunk_options:
+        chunks = improved_chunking_process(content, chunk_options)
+    else:
+        chunks = [{'text': content, 'metadata': {}}]
+
+    # Use default prompt if not provided
+    if not summarization_prompt:
+        summarization_prompt = config.get('Prompts', 'default_summary_prompt', fallback="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""")
+
+    # Summarization logic
+    summaries = []
+    for chunk in chunks:
+        chunk_text = chunk['text']
+        try:
+            chunk_summary = summarize_chunk(api_name, chunk_text, summarization_prompt, api_key)
+            if chunk_summary:
+                summaries.append(chunk_summary)
+            else:
+                logging.warning(f"Summarization failed for chunk: {chunk_text[:100]}...")
+        except Exception as e:
+            logging.error(f"Error during summarization: {str(e)}")
+            return f"Error during summarization: {str(e)}"
+
+    if not summaries:
+        return "Summarization failed for all chunks."
+
+    new_summary = " ".join(summaries)
+
+    # Update the database with the new summary
+
+    try:
+        update_result = update_media_content(selected_item, item_mapping, content, summarization_prompt, new_summary)
+        if "successfully" in update_result.lower():
+            return f"Re-summarization complete. New summary: {new_summary}..."
+        else:
+            return f"Error during database update: {update_result}"
+    except Exception as e:
+        logging.error(f"Error updating database: {str(e)}")
+        return f"Error updating database: {str(e)}"

App_Function_Libraries/Gradio_UI/Search_Tab.py

@@ -0,0 +1,487 @@
+# Search_Tab.py
+# Description: This file contains the code for the search tab in the Gradio UI
+#
+# Imports
+import html
+import logging
+import sqlite3
+
+#
+# External Imports
+import gradio as gr
+
+from App_Function_Libraries.DB_Manager import view_database, search_and_display_items
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_detailed_view
+from App_Function_Libraries.RAG_Libary_2 import rag_search
+
+#
+# Local Imports
+#
+#
+###################################################################################################
+#
+# Functions:
+
+logger = logging.getLogger()
+
+
+
+
+# FIXME - SQL functions to be moved to DB_Manager
+def search_prompts(query):
+    try:
+        conn = sqlite3.connect('prompts.db')
+        cursor = conn.cursor()
+        cursor.execute("SELECT name, details, system, user FROM Prompts WHERE name LIKE ? OR details LIKE ?",
+                       (f"%{query}%", f"%{query}%"))
+        results = cursor.fetchall()
+        conn.close()
+        return results
+    except sqlite3.Error as e:
+        print(f"Error searching prompts: {e}")
+        return []
+
+
+
+
+
+
+
+
+
+
+
+
+def create_rag_tab():
+    with gr.TabItem("RAG Search"):
+        gr.Markdown("# Retrieval-Augmented Generation (RAG) Search")
+
+        with gr.Row():
+            with gr.Column():
+                search_query = gr.Textbox(label="Enter your question", placeholder="What would you like to know?")
+                api_choice = gr.Dropdown(
+                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    label="Select API for RAG",
+                    value="OpenAI"
+                )
+                search_button = gr.Button("Search")
+
+            with gr.Column():
+                result_output = gr.Textbox(label="Answer", lines=10)
+                context_output = gr.Textbox(label="Context", lines=10, visible=False)
+
+        def perform_rag_search(query, api_choice):
+            result = rag_search(query, api_choice)
+            return result['answer'], result['context']
+
+        search_button.click(perform_rag_search, inputs=[search_query, api_choice], outputs=[result_output, context_output])
+
+# FIXME - under construction
+def create_embeddings_tab():
+    with gr.TabItem("Create Embeddings"):
+        gr.Markdown("# Create Embeddings for All Content")
+
+        with gr.Row():
+            with gr.Column():
+                embedding_api_choice = gr.Dropdown(
+                    choices=["OpenAI", "Local", "HuggingFace"],
+                    label="Select API for Embeddings",
+                    value="OpenAI"
+                )
+                create_button = gr.Button("Create Embeddings")
+
+            with gr.Column():
+                status_output = gr.Textbox(label="Status", lines=10)
+
+        def create_embeddings(api_choice):
+            try:
+                # Assuming you have a function that handles the creation of embeddings
+                from App_Function_Libraries.ChromaDB_Library import create_all_embeddings
+                status = create_all_embeddings(api_choice)
+                return status
+            except Exception as e:
+                return f"Error: {str(e)}"
+
+        create_button.click(create_embeddings, inputs=[embedding_api_choice], outputs=status_output)
+
+
+
+
+def create_search_tab():
+    with gr.TabItem("Search / Detailed View"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Search across all ingested items in the Database")
+                gr.Markdown(" by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
+                search_button = gr.Button("Search")
+                items_output = gr.Dropdown(label="Select Item", choices=[])
+                item_mapping = gr.State({})
+                prompt_summary_output = gr.HTML(label="Prompt & Summary", visible=True)
+
+                search_button.click(
+                    fn=update_dropdown,
+                    inputs=[search_query_input, search_type_input],
+                    outputs=[items_output, item_mapping]
+                )
+            with gr.Column():
+                content_output = gr.Markdown(label="Content", visible=True)
+                items_output.change(
+                    fn=update_detailed_view,
+                    inputs=[items_output, item_mapping],
+                    outputs=[prompt_summary_output, content_output]
+                )
+
+
+def display_search_results(query):
+    if not query.strip():
+        return "Please enter a search query."
+
+    results = search_prompts(query)
+
+    # Debugging: Print the results to the console to see what is being returned
+    print(f"Processed search results for query '{query}': {results}")
+
+    if results:
+        result_md = "## Search Results:\n"
+        for result in results:
+            # Debugging: Print each result to see its format
+            print(f"Result item: {result}")
+
+            if len(result) == 2:
+                name, details = result
+                result_md += f"**Title:** {name}\n\n**Description:** {details}\n\n---\n"
+
+            elif len(result) == 4:
+                name, details, system, user = result
+                result_md += f"**Title:** {name}\n\n"
+                result_md += f"**Description:** {details}\n\n"
+                result_md += f"**System Prompt:** {system}\n\n"
+                result_md += f"**User Prompt:** {user}\n\n"
+                result_md += "---\n"
+            else:
+                result_md += "Error: Unexpected result format.\n\n---\n"
+        return result_md
+    return "No results found."
+
+
+def create_viewing_tab():
+    with gr.TabItem("View Database"):
+        gr.Markdown("# View Database Entries")
+        with gr.Row():
+            with gr.Column():
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+            with gr.Column():
+                results_display = gr.HTML()
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+
+        def update_page(page, entries_per_page):
+            results, pagination, total_pages = view_database(page, entries_per_page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(interactive=not prev_disabled)
+
+        def go_to_next_page(current_page, entries_per_page):
+            next_page = current_page + 1
+            return update_page(next_page, entries_per_page)
+
+        def go_to_previous_page(current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return update_page(previous_page, entries_per_page)
+
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+def create_search_summaries_tab():
+    with gr.TabItem("Search/View Title+Summary "):
+        gr.Markdown("# Search across all ingested items in the Database and review their summaries")
+        gr.Markdown("Search by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
+                                             label="Search By")
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                char_count_input = gr.Number(value=5000, label="Amount of characters to display from the main content",
+                                             precision=0)
+            with gr.Column():
+                search_button = gr.Button("Search")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+        search_results_output = gr.HTML()
+
+
+        def update_search_page(query, search_type, page, entries_per_page, char_count):
+            # Ensure char_count is a positive integer
+            char_count = max(1, int(char_count)) if char_count else 5000
+            results, pagination, total_pages = search_and_display_items(query, search_type, page, entries_per_page, char_count)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
+                interactive=not prev_disabled)
+
+        def go_to_next_search_page(query, search_type, current_page, entries_per_page, char_count):
+            next_page = current_page + 1
+            return update_search_page(query, search_type, next_page, entries_per_page, char_count)
+
+        def go_to_previous_search_page(query, search_type, current_page, entries_per_page, char_count):
+            previous_page = max(1, current_page - 1)
+            return update_search_page(query, search_type, previous_page, entries_per_page, char_count)
+
+        search_button.click(
+            fn=update_search_page,
+            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_search_page,
+            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_search_page,
+            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+
+def create_prompt_view_tab():
+    with gr.TabItem("View Prompt Database"):
+        gr.Markdown("# View Prompt Database Entries")
+        with gr.Row():
+            with gr.Column():
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+            with gr.Column():
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+        results_display = gr.HTML()
+
+        # FIXME - SQL functions to be moved to DB_Manager
+        def view_database(page, entries_per_page):
+            offset = (page - 1) * entries_per_page
+            try:
+                with sqlite3.connect('prompts.db') as conn:
+                    cursor = conn.cursor()
+                    cursor.execute('''
+                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+                        FROM Prompts p
+                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                        GROUP BY p.id
+                        ORDER BY p.name
+                        LIMIT ? OFFSET ?
+                    ''', (entries_per_page, offset))
+                    prompts = cursor.fetchall()
+
+                    cursor.execute('SELECT COUNT(*) FROM Prompts')
+                    total_prompts = cursor.fetchone()[0]
+
+                results = ""
+                for prompt in prompts:
+                    # Escape HTML special characters and replace newlines with <br> tags
+                    title = html.escape(prompt[0]).replace('\n', '<br>')
+                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
+                    system_prompt = html.escape(prompt[2] or '')
+                    user_prompt = html.escape(prompt[3] or '')
+                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
+
+                    results += f"""
+                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
+                            <div><strong>Title:</strong> {title}</div>
+                            <div><strong>Details:</strong> {details}</div>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>User Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>System Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>Keywords:</strong> {keywords}
+                        </div>
+                    </div>
+                    """
+
+                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
+                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
+
+                return results, pagination, total_pages
+            except sqlite3.Error as e:
+                return f"<p>Error fetching prompts: {e}</p>", "Error", 0
+
+        def update_page(page, entries_per_page):
+            results, pagination, total_pages = view_database(page, entries_per_page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
+                interactive=not prev_disabled)
+
+        def go_to_next_page(current_page, entries_per_page):
+            next_page = current_page + 1
+            return update_page(next_page, entries_per_page)
+
+        def go_to_previous_page(current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return update_page(previous_page, entries_per_page)
+
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+
+def create_prompt_search_tab():
+    with gr.TabItem("Search Prompts"):
+        gr.Markdown("# Search and View Prompt Details")
+        gr.Markdown("Currently has all of the https://github.com/danielmiessler/fabric prompts already available")
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Prompts", placeholder="Enter your search query...")
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+            with gr.Column():
+                search_button = gr.Button("Search Prompts")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+        search_results_output = gr.HTML()
+
+        def search_and_display_prompts(query, page, entries_per_page):
+            offset = (page - 1) * entries_per_page
+            try:
+                # FIXME - SQL functions to be moved to DB_Manager
+                with sqlite3.connect('prompts.db') as conn:
+                    cursor = conn.cursor()
+                    cursor.execute('''
+                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+                        FROM Prompts p
+                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                        WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
+                        GROUP BY p.id
+                        ORDER BY p.name
+                        LIMIT ? OFFSET ?
+                    ''', (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', entries_per_page, offset))
+                    prompts = cursor.fetchall()
+
+                    cursor.execute('''
+                        SELECT COUNT(DISTINCT p.id)
+                        FROM Prompts p
+                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                        WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
+                    ''', (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%'))
+                    total_prompts = cursor.fetchone()[0]
+
+                results = ""
+                for prompt in prompts:
+                    title = html.escape(prompt[0]).replace('\n', '<br>')
+                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
+                    system_prompt = html.escape(prompt[2] or '')
+                    user_prompt = html.escape(prompt[3] or '')
+                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
+
+                    results += f"""
+                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
+                            <div><strong>Title:</strong> {title}</div>
+                            <div><strong>Details:</strong> {details}</div>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>User Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>System Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>Keywords:</strong> {keywords}
+                        </div>
+                    </div>
+                    """
+
+                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
+                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
+
+                return results, pagination, total_pages
+            except sqlite3.Error as e:
+                return f"<p>Error searching prompts: {e}</p>", "Error", 0
+
+        def update_search_page(query, page, entries_per_page):
+            results, pagination, total_pages = search_and_display_prompts(query, page, entries_per_page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(interactive=not prev_disabled)
+
+        def go_to_next_search_page(query, current_page, entries_per_page):
+            next_page = current_page + 1
+            return update_search_page(query, next_page, entries_per_page)
+
+        def go_to_previous_search_page(query, current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return update_search_page(query, previous_page, entries_per_page)
+
+        search_button.click(
+            fn=update_search_page,
+            inputs=[search_query_input, page_number, entries_per_page],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_search_page,
+            inputs=[search_query_input, page_number, entries_per_page],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_search_page,
+            inputs=[search_query_input, page_number, entries_per_page],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )

App_Function_Libraries/Gradio_UI/Transcript_comparison.py

@@ -0,0 +1,94 @@
+# Transcript_comparison.py
+# Description: Gradio UI tab for comparing transcripts
+#
+# Imports
+import logging
+
+#
+# External Imports
+import gradio as gr
+
+from App_Function_Libraries.DB_Manager import get_transcripts
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import browse_items
+from App_Function_Libraries.Utils import format_transcription
+
+
+#
+# Local Imports
+
+def get_transcript_options(media_id):
+    transcripts = get_transcripts(media_id)
+    return [f"{t[0]}: {t[1]} ({t[3]})" for t in transcripts]
+
+
+def update_transcript_options(media_id):
+    options = get_transcript_options(media_id)
+    return gr.update(choices=options), gr.update(choices=options)
+
+def compare_transcripts(media_id, transcript1_id, transcript2_id):
+    try:
+        transcripts = get_transcripts(media_id)
+        transcript1 = next((t for t in transcripts if t[0] == int(transcript1_id)), None)
+        transcript2 = next((t for t in transcripts if t[0] == int(transcript2_id)), None)
+
+        if not transcript1 or not transcript2:
+            return "One or both selected transcripts not found."
+
+        comparison = f"Transcript 1 (Model: {transcript1[1]}, Created: {transcript1[3]}):\n\n"
+        comparison += format_transcription(transcript1[2])
+        comparison += f"\n\nTranscript 2 (Model: {transcript2[1]}, Created: {transcript2[3]}):\n\n"
+        comparison += format_transcription(transcript2[2])
+
+        return comparison
+    except Exception as e:
+        logging.error(f"Error in compare_transcripts: {str(e)}")
+        return f"Error comparing transcripts: {str(e)}"
+
+
+def create_compare_transcripts_tab():
+    with gr.TabItem("Compare Transcripts"):
+        gr.Markdown("# Compare Transcripts")
+
+        with gr.Row():
+            search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+            search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
+            search_button = gr.Button("Search")
+
+        with gr.Row():
+            media_id_output = gr.Dropdown(label="Select Media Item", choices=[], interactive=True)
+            media_mapping = gr.State({})
+
+        media_id_input = gr.Number(label="Media ID", visible=False)
+        transcript1_dropdown = gr.Dropdown(label="Transcript 1")
+        transcript2_dropdown = gr.Dropdown(label="Transcript 2")
+        compare_button = gr.Button("Compare Transcripts")
+        comparison_output = gr.Textbox(label="Comparison Result", lines=20)
+
+        def update_media_dropdown(search_query, search_type):
+            results = browse_items(search_query, search_type)
+            item_options = [f"{item[1]} ({item[2]})" for item in results]
+            new_item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results}
+            return gr.update(choices=item_options), new_item_mapping
+
+        search_button.click(
+            fn=update_media_dropdown,
+            inputs=[search_query_input, search_type_input],
+            outputs=[media_id_output, media_mapping]
+        )
+
+        def load_selected_media_id(selected_media, media_mapping):
+            if selected_media and media_mapping and selected_media in media_mapping:
+                media_id = media_mapping[selected_media]
+                return media_id
+            return None
+
+        media_id_output.change(
+            fn=load_selected_media_id,
+            inputs=[media_id_output, media_mapping],
+            outputs=[media_id_input]
+        )
+
+        media_id_input.change(update_transcript_options, inputs=[media_id_input],
+                              outputs=[transcript1_dropdown, transcript2_dropdown])
+        compare_button.click(compare_transcripts, inputs=[media_id_input, transcript1_dropdown, transcript2_dropdown],
+                             outputs=[comparison_output])

App_Function_Libraries/Gradio_UI/Trash.py

@@ -0,0 +1,134 @@
+# Trash.py
+# Gradio UI for deleting items from the database
+import html
+import sqlite3
+
+# Imports
+
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import delete_prompt, empty_trash, get_trashed_items, user_delete_item
+
+
+def delete_item(media_id, force):
+    return user_delete_item(media_id, force)
+
+def list_trash():
+    items = get_trashed_items()
+    return "\n".join(
+        [f"ID: {item['id']}, Title: {item['title']}, Trashed on: {item['trash_date']}" for item in items])
+
+def empty_trash_ui(days):
+    deleted, remaining = empty_trash(days)
+    return f"Deleted {deleted} items. {remaining} items remain in trash."
+
+def create_view_trash_tab():
+    with gr.TabItem("View Trash"):
+        view_button = gr.Button("View Trash")
+        trash_list = gr.Textbox(label="Trashed Items")
+        view_button.click(list_trash, inputs=[], outputs=trash_list)
+
+
+
+
+def search_prompts_for_deletion(query):
+    try:
+        with sqlite3.connect('prompts.db') as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT id, name, details
+                FROM Prompts
+                WHERE name LIKE ? OR details LIKE ?
+                LIMIT 10
+            ''', (f'%{query}%', f'%{query}%'))
+            results = cursor.fetchall()
+
+            if not results:
+                return "No matching prompts found."
+
+            output = "<h3>Matching Prompts:</h3>"
+            for row in results:
+                output += f"<p><strong>ID:</strong> {row[0]} | <strong>Name:</strong> {html.escape(row[1])} | <strong>Details:</strong> {html.escape(row[2][:100])}...</p>"
+            return output
+    except sqlite3.Error as e:
+        return f"An error occurred while searching prompts: {e}"
+
+
+def search_media_for_deletion(query):
+    try:
+        with sqlite3.connect('media.db') as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT id, title, description
+                FROM media
+                WHERE title LIKE ? OR description LIKE ?
+                LIMIT 10
+            ''', (f'%{query}%', f'%{query}%'))
+            results = cursor.fetchall()
+
+            if not results:
+                return "No matching media found."
+
+            output = "<h3>Matching Media:</h3>"
+            for row in results:
+                output += f"<p><strong>ID:</strong> {row[0]} | <strong>Title:</strong> {html.escape(row[1])} | <strong>Description:</strong> {html.escape(row[2][:100])}...</p>"
+            return output
+    except sqlite3.Error as e:
+        return f"An error occurred while searching media: {e}"
+
+def create_delete_trash_tab():
+    with gr.TabItem("Delete DB Item"):
+        gr.Markdown("# Search and Delete Items from Databases")
+
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("## Search and Delete Prompts")
+                prompt_search_input = gr.Textbox(label="Search Prompts")
+                prompt_search_button = gr.Button("Search Prompts")
+                prompt_search_results = gr.HTML()
+                prompt_id_input = gr.Number(label="Prompt ID")
+                prompt_delete_button = gr.Button("Delete Prompt")
+                prompt_delete_output = gr.Textbox(label="Delete Result")
+
+            with gr.Column():
+                gr.Markdown("## Search and Delete Media")
+                media_search_input = gr.Textbox(label="Search Media")
+                media_search_button = gr.Button("Search Media")
+                media_search_results = gr.HTML()
+                media_id_input = gr.Number(label="Media ID")
+                media_force_checkbox = gr.Checkbox(label="Force Delete")
+                media_delete_button = gr.Button("Delete Media")
+                media_delete_output = gr.Textbox(label="Delete Result")
+
+        prompt_search_button.click(
+            search_prompts_for_deletion,
+            inputs=[prompt_search_input],
+            outputs=prompt_search_results
+        )
+
+        prompt_delete_button.click(
+            delete_prompt,
+            inputs=[prompt_id_input],
+            outputs=prompt_delete_output
+        )
+
+        media_search_button.click(
+            search_media_for_deletion,
+            inputs=[media_search_input],
+            outputs=media_search_results
+        )
+
+        media_delete_button.click(
+            delete_item,
+            inputs=[media_id_input, media_force_checkbox],
+            outputs=media_delete_output
+        )
+
+def create_empty_trash_tab():
+    with gr.TabItem("Empty Trash"):
+        days_input = gr.Slider(minimum=15, maximum=90, step=5, label="Delete items older than (days)")
+        empty_button = gr.Button("Empty Trash")
+        empty_output = gr.Textbox(label="Result")
+        empty_button.click(empty_trash_ui, inputs=[days_input], outputs=empty_output)

App_Function_Libraries/Gradio_UI/Utilities.py

@@ -0,0 +1,118 @@
+import os
+import shutil
+import tempfile
+from pathlib import Path
+
+import gradio as gr
+import yt_dlp
+
+from App_Function_Libraries.Utils import sanitize_filename, downloaded_files
+
+
+def create_utilities_yt_video_tab():
+    with gr.Tab("YouTube Video Downloader"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(
+                    "<h3>Youtube Video Downloader</h3><p>This Input takes a Youtube URL as input and creates a webm file for you to download. </br><em>If you want a full-featured one:</em> <strong><em>https://github.com/StefanLobbenmeier/youtube-dl-gui</strong></em> or <strong><em>https://github.com/yt-dlg/yt-dlg</em></strong></p>")
+                youtube_url_input = gr.Textbox(label="YouTube URL", placeholder="Enter YouTube video URL here")
+                download_button = gr.Button("Download Video")
+            with gr.Column():
+                output_file = gr.File(label="Download Video")
+                output_message = gr.Textbox(label="Status")
+
+        download_button.click(
+            fn=gradio_download_youtube_video,
+            inputs=youtube_url_input,
+            outputs=[output_file, output_message]
+        )
+
+def create_utilities_yt_audio_tab():
+    with gr.Tab("YouTube Audio Downloader"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown(
+                    "<h3>Youtube Audio Downloader</h3><p>This Input takes a Youtube URL as input and creates an audio file for you to download.</p>"
+                    +"\n<em>If you want a full-featured one:</em> <strong><em>https://github.com/StefanLobbenmeier/youtube-dl-gui</strong></em>\n or \n<strong><em>https://github.com/yt-dlg/yt-dlg</em></strong></p>")
+                youtube_url_input_audio = gr.Textbox(label="YouTube URL", placeholder="Enter YouTube video URL here")
+                download_button_audio = gr.Button("Download Audio")
+            with gr.Column():
+                output_file_audio = gr.File(label="Download Audio")
+                output_message_audio = gr.Textbox(label="Status")
+
+        from App_Function_Libraries.Audio_Files import download_youtube_audio
+        download_button_audio.click(
+            fn=download_youtube_audio,
+            inputs=youtube_url_input_audio,
+            outputs=[output_file_audio, output_message_audio]
+        )
+
+def create_utilities_yt_timestamp_tab():
+    with gr.Tab("YouTube Timestamp URL Generator"):
+        gr.Markdown("## Generate YouTube URL with Timestamp")
+        with gr.Row():
+            with gr.Column():
+                url_input = gr.Textbox(label="YouTube URL")
+                hours_input = gr.Number(label="Hours", value=0, minimum=0, precision=0)
+                minutes_input = gr.Number(label="Minutes", value=0, minimum=0, maximum=59, precision=0)
+                seconds_input = gr.Number(label="Seconds", value=0, minimum=0, maximum=59, precision=0)
+                generate_button = gr.Button("Generate URL")
+            with gr.Column():
+                output_url = gr.Textbox(label="Timestamped URL")
+
+        from App_Function_Libraries.Video_DL_Ingestion_Lib import generate_timestamped_url
+        generate_button.click(
+            fn=generate_timestamped_url,
+            inputs=[url_input, hours_input, minutes_input, seconds_input],
+            outputs=output_url
+        )
+
+
+def gradio_download_youtube_video(url):
+    try:
+        # Determine ffmpeg path based on the operating system.
+        ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg'
+
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Extract information about the video
+            with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
+                info_dict = ydl.extract_info(url, download=False)
+                sanitized_title = sanitize_filename(info_dict['title'])
+                original_ext = info_dict['ext']
+
+            # Setup the temporary filename
+            temp_file_path = Path(temp_dir) / f"{sanitized_title}.{original_ext}"
+
+            # Initialize yt-dlp with generic options and the output template
+            ydl_opts = {
+                'format': 'bestvideo+bestaudio/best',
+                'ffmpeg_location': ffmpeg_path,
+                'outtmpl': str(temp_file_path),
+                'noplaylist': True,
+                'quiet': True
+            }
+
+            # Execute yt-dlp to download the video
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                ydl.download([url])
+
+            # Final check to ensure file exists
+            if not temp_file_path.exists():
+                raise FileNotFoundError(f"Expected file was not found: {temp_file_path}")
+
+            # Create a persistent directory for the download if it doesn't exist
+            persistent_dir = Path("downloads")
+            persistent_dir.mkdir(exist_ok=True)
+
+            # Move the file from the temporary directory to the persistent directory
+            persistent_file_path = persistent_dir / f"{sanitized_title}.{original_ext}"
+            shutil.move(str(temp_file_path), str(persistent_file_path))
+
+            # Add the file to the list of downloaded files
+            downloaded_files.append(str(persistent_file_path))
+
+            return str(persistent_file_path), f"Video downloaded successfully: {sanitized_title}.{original_ext}"
+    except Exception as e:
+        return None, f"Error downloading video: {str(e)}"
+

App_Function_Libraries/Gradio_UI/Video_transcription_tab.py

@@ -0,0 +1,691 @@
+# Video_transcription_tab.py
+# Description: This file contains the code for the video transcription tab in the Gradio UI.
+#
+# Imports
+import json
+import logging
+import os
+#
+# External Imports
+import gradio as gr
+import yt_dlp
+#
+# Local Imports
+from App_Function_Libraries.DB_Manager import load_preset_prompts, add_media_to_database
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models, update_user_prompt
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import error_handler
+from App_Function_Libraries.Summarization_General_Lib import perform_transcription, perform_summarization, \
+    save_transcription_and_summary
+from App_Function_Libraries.Utils import convert_to_seconds, safe_read_file, format_transcription, \
+    create_download_directory, generate_unique_identifier, extract_text_from_segments
+from App_Function_Libraries.Video_DL_Ingestion_Lib import parse_and_expand_urls, extract_metadata, download_video
+#
+################################################################################################################################################################
+#
+# Functions:
+
+def create_video_transcription_tab():
+    with (gr.TabItem("Video Transcription + Summarization")):
+        gr.Markdown("# Transcribe & Summarize Videos from URLs")
+        with gr.Row():
+            gr.Markdown("""Follow this project at [tldw - GitHub](https://github.com/rmusser01/tldw)""")
+        with gr.Row():
+            gr.Markdown(
+                """If you're wondering what all this is, please see the 'Introduction/Help' tab up above for more detailed information and how to obtain an API Key.""")
+        with gr.Row():
+            with gr.Column():
+                url_input = gr.Textbox(label="URL(s) (Mandatory)",
+                                       placeholder="Enter video URLs here, one per line. Supports YouTube, Vimeo, other video sites and Youtube playlists.",
+                                       lines=5)
+                video_file_input = gr.File(label="Upload Video File (Optional)", file_types=["video/*"])
+                diarize_input = gr.Checkbox(label="Enable Speaker Diarization", value=False)
+                whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
+
+                with gr.Row():
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                         value=False,
+                                                         visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                         value=False,
+                                                         visible=True)
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=load_preset_prompts(),
+                                                visible=False)
+                with gr.Row():
+                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+""",
+                                                     lines=3,
+                                                     visible=False,
+                                                     interactive=True)
+                custom_prompt_checkbox.change(
+                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    inputs=[custom_prompt_checkbox],
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+                preset_prompt_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[preset_prompt_checkbox],
+                    outputs=[preset_prompt]
+                )
+
+                def update_prompts(preset_name):
+                    prompts = update_user_prompt(preset_name)
+                    return (
+                        gr.update(value=prompts["user_prompt"], visible=True),
+                        gr.update(value=prompts["system_prompt"], visible=True)
+                    )
+
+                preset_prompt.change(
+                    update_prompts,
+                    inputs=preset_prompt,
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
+                             "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    value=None, label="API Name (Mandatory)")
+                api_key_input = gr.Textbox(label="API Key (Mandatory)", placeholder="Enter your API key here",
+                                           type="password")
+                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)",
+                                            value="default,no_keyword_set")
+                batch_size_input = gr.Slider(minimum=1, maximum=10, value=1, step=1,
+                                             label="Batch Size (Number of videos to process simultaneously)")
+                timestamp_option = gr.Radio(choices=["Include Timestamps", "Exclude Timestamps"],
+                                            value="Include Timestamps", label="Timestamp Option")
+                keep_original_video = gr.Checkbox(label="Keep Original Video", value=False)
+                # First, create a checkbox to toggle the chunking options
+                chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False)
+                summarize_recursively = gr.Checkbox(label="Enable Recursive Summarization", value=False)
+                use_cookies_input = gr.Checkbox(label="Use cookies for authenticated download", value=False)
+                use_time_input = gr.Checkbox(label="Use Start and End Time", value=False)
+
+                with gr.Row(visible=False) as time_input_box:
+                    gr.Markdown("### Start and End time")
+                    with gr.Column():
+                        start_time_input = gr.Textbox(label="Start Time (Optional)",
+                                                      placeholder="e.g., 1:30 or 90 (in seconds)")
+                        end_time_input = gr.Textbox(label="End Time (Optional)",
+                                                    placeholder="e.g., 5:45 or 345 (in seconds)")
+
+                use_time_input.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[use_time_input],
+                    outputs=[time_input_box]
+                )
+
+                cookies_input = gr.Textbox(
+                    label="User Session Cookies",
+                    placeholder="Paste your cookies here (JSON format)",
+                    lines=3,
+                    visible=False
+                )
+
+                use_cookies_input.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[use_cookies_input],
+                    outputs=[cookies_input]
+                )
+                # Then, create a Box to group the chunking options
+                with gr.Row(visible=False) as chunking_options_box:
+                    gr.Markdown("### Chunking Options")
+                    with gr.Column():
+                        chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'],
+                                                   label="Chunking Method")
+                        max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50,
+                                                   label="Max Chunk Size")
+                        chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
+                        use_adaptive_chunking = gr.Checkbox(
+                            label="Use Adaptive Chunking (Adjust chunking based on text complexity)")
+                        use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking")
+                        chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'],
+                                                     label="Chunking Language")
+
+                # Add JavaScript to toggle the visibility of the chunking options box
+                chunking_options_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[chunking_options_checkbox],
+                    outputs=[chunking_options_box]
+                )
+                process_button = gr.Button("Process Videos")
+
+            with gr.Column():
+                progress_output = gr.Textbox(label="Progress")
+                error_output = gr.Textbox(label="Errors", visible=False)
+                results_output = gr.HTML(label="Results")
+                download_transcription = gr.File(label="Download All Transcriptions as JSON")
+                download_summary = gr.File(label="Download All Summaries as Text")
+
+            @error_handler
+            def process_videos_with_error_handling(inputs, start_time, end_time, diarize, whisper_model,
+                                                   custom_prompt_checkbox, custom_prompt, chunking_options_checkbox,
+                                                   chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                                                   use_multi_level_chunking, chunk_language, api_name,
+                                                   api_key, keywords, use_cookies, cookies, batch_size,
+                                                   timestamp_option, keep_original_video, summarize_recursively,
+                                                   progress: gr.Progress = gr.Progress()) -> tuple:
+                try:
+                    logging.info("Entering process_videos_with_error_handling")
+                    logging.info(f"Received inputs: {inputs}")
+
+                    if not inputs:
+                        raise ValueError("No inputs provided")
+
+                    logging.debug("Input(s) is(are) valid")
+
+                    # Ensure batch_size is an integer
+                    try:
+                        batch_size = int(batch_size)
+                    except (ValueError, TypeError):
+                        batch_size = 1  # Default to processing one video at a time if invalid
+
+                    # Separate URLs and local files
+                    urls = [input for input in inputs if
+                            isinstance(input, str) and input.startswith(('http://', 'https://'))]
+                    local_files = [input for input in inputs if
+                                   isinstance(input, str) and not input.startswith(('http://', 'https://'))]
+
+                    # Parse and expand URLs if there are any
+                    expanded_urls = parse_and_expand_urls(urls) if urls else []
+
+                    valid_local_files = []
+                    invalid_local_files = []
+
+                    for file_path in local_files:
+                        if os.path.exists(file_path):
+                            valid_local_files.append(file_path)
+                        else:
+                            invalid_local_files.append(file_path)
+                            error_message = f"Local file not found: {file_path}"
+                            logging.error(error_message)
+
+                    if invalid_local_files:
+                        logging.warning(f"Found {len(invalid_local_files)} invalid local file paths")
+                        # FIXME - Add more complete error handling for invalid local files
+
+                    all_inputs = expanded_urls + valid_local_files
+                    logging.info(f"Total valid inputs to process: {len(all_inputs)} "
+                                 f"({len(expanded_urls)} URLs, {len(valid_local_files)} local files)")
+
+                    all_inputs = expanded_urls + local_files
+                    logging.info(f"Total inputs to process: {len(all_inputs)}")
+                    results = []
+                    errors = []
+                    results_html = ""
+                    all_transcriptions = {}
+                    all_summaries = ""
+
+                    for i in range(0, len(all_inputs), batch_size):
+                        batch = all_inputs[i:i + batch_size]
+                        batch_results = []
+
+                        for input_item in batch:
+                            try:
+                                start_seconds = convert_to_seconds(start_time)
+                                end_seconds = convert_to_seconds(end_time) if end_time else None
+
+                                logging.info(f"Attempting to extract metadata for {input_item}")
+
+                                if input_item.startswith(('http://', 'https://')):
+                                    logging.info(f"Attempting to extract metadata for URL: {input_item}")
+                                    video_metadata = extract_metadata(input_item, use_cookies, cookies)
+                                    if not video_metadata:
+                                        raise ValueError(f"Failed to extract metadata for {input_item}")
+                                else:
+                                    logging.info(f"Processing local file: {input_item}")
+                                    video_metadata = {"title": os.path.basename(input_item), "url": input_item}
+
+                                chunk_options = {
+                                    'method': chunk_method,
+                                    'max_size': max_chunk_size,
+                                    'overlap': chunk_overlap,
+                                    'adaptive': use_adaptive_chunking,
+                                    'multi_level': use_multi_level_chunking,
+                                    'language': chunk_language
+                                } if chunking_options_checkbox else None
+
+                                if custom_prompt_checkbox:
+                                    custom_prompt = custom_prompt
+                                else:
+                                    custom_prompt = ("""
+                                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                                        **Bulleted Note Creation Guidelines**
+
+                                        **Headings**:
+                                        - Based on referenced topics, not categories like quotes or terms
+                                        - Surrounded by **bold** formatting 
+                                        - Not listed as bullet points
+                                        - No space between headings and list items underneath
+
+                                        **Emphasis**:
+                                        - **Important terms** set in bold font
+                                        - **Text ending in a colon**: also bolded
+
+                                        **Review**:
+                                        - Ensure adherence to specified format
+                                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                                    """)
+
+                                logging.debug("Gradio_Related.py: process_url_with_metadata being called")
+                                result = process_url_with_metadata(
+                                    input_item, 2, whisper_model,
+                                    custom_prompt,
+                                    start_seconds, api_name, api_key,
+                                    False, False, False, False, 0.01, None, keywords, None, diarize,
+                                    end_time=end_seconds,
+                                    include_timestamps=(timestamp_option == "Include Timestamps"),
+                                    metadata=video_metadata,
+                                    use_chunking=chunking_options_checkbox,
+                                    chunk_options=chunk_options,
+                                    keep_original_video=keep_original_video,
+                                    current_whisper_model=whisper_model,
+                                )
+
+                                if result[0] is None:
+                                    error_message = "Processing failed without specific error"
+                                    batch_results.append(
+                                        (input_item, error_message, "Error", video_metadata, None, None))
+                                    errors.append(f"Error processing {input_item}: {error_message}")
+                                else:
+                                    url, transcription, summary, json_file, summary_file, result_metadata = result
+                                    if transcription is None:
+                                        error_message = f"Processing failed for {input_item}: Transcription is None"
+                                        batch_results.append(
+                                            (input_item, error_message, "Error", result_metadata, None, None))
+                                        errors.append(error_message)
+                                    else:
+                                        batch_results.append(
+                                            (input_item, transcription, "Success", result_metadata, json_file,
+                                             summary_file))
+
+
+                            except Exception as e:
+                                error_message = f"Error processing {input_item}: {str(e)}"
+                                logging.error(error_message, exc_info=True)
+                                batch_results.append((input_item, error_message, "Error", {}, None, None))
+                                errors.append(error_message)
+
+                        results.extend(batch_results)
+                        logging.debug(f"Processed {len(batch_results)} videos in batch")
+                        if isinstance(progress, gr.Progress):
+                            progress((i + len(batch)) / len(all_inputs),
+                                     f"Processed {i + len(batch)}/{len(all_inputs)} videos")
+
+                    # Generate HTML for results
+                    logging.debug(f"Generating HTML for {len(results)} results")
+                    for url, transcription, status, metadata, json_file, summary_file in results:
+                        if status == "Success":
+                            title = metadata.get('title', 'Unknown Title')
+
+                            # Check if transcription is a string (which it should be now)
+                            if isinstance(transcription, str):
+                                # Split the transcription into metadata and actual transcription
+                                parts = transcription.split('\n\n', 1)
+                                if len(parts) == 2:
+                                    metadata_text, transcription_text = parts
+                                else:
+                                    metadata_text = "Metadata not found"
+                                    transcription_text = transcription
+                            else:
+                                metadata_text = "Metadata format error"
+                                transcription_text = "Transcription format error"
+
+                            summary = safe_read_file(summary_file) if summary_file else "No summary available"
+
+                            # FIXME - Add to other functions that generate HTML
+                            # Format the transcription
+                            formatted_transcription = format_transcription(transcription_text)
+                            # Format the summary
+                            formatted_summary = format_transcription(summary)
+
+                            results_html += f"""
+                            <div class="result-box">
+                                <gradio-accordion>
+                                    <gradio-accordion-item label="{title}">
+                                        <p><strong>URL:</strong> <a href="{url}" target="_blank">{url}</a></p>
+                                        <h4>Metadata:</h4>
+                                        <pre>{metadata_text}</pre>
+                                        <h4>Transcription:</h4>
+                                        <div class="transcription" style="white-space: pre-wrap; word-wrap: break-word;">
+                                            {formatted_transcription}
+                                        </div>
+                                        <h4>Summary:</h4>
+                                        <div class="summary">{formatted_summary}</div>
+                                    </gradio-accordion-item>
+                                </gradio-accordion>
+                            </div>
+                            """
+                            logging.debug(f"Transcription for {url}: {transcription[:200]}...")
+                            all_transcriptions[url] = transcription
+                            all_summaries += f"Title: {title}\nURL: {url}\n\n{metadata_text}\n\nTranscription:\n{transcription_text}\n\nSummary:\n{summary}\n\n---\n\n"
+                        else:
+                            results_html += f"""
+                            <div class="result-box error">
+                                <h3>Error processing {url}</h3>
+                                <p>{transcription}</p>
+                            </div>
+                            """
+
+                    # Save all transcriptions and summaries to files
+                    logging.debug("Saving all transcriptions and summaries to files")
+                    with open('all_transcriptions.json', 'w', encoding='utf-8') as f:
+                        json.dump(all_transcriptions, f, indent=2, ensure_ascii=False)
+
+                    with open('all_summaries.txt', 'w', encoding='utf-8') as f:
+                        f.write(all_summaries)
+
+                    error_summary = "\n".join(errors) if errors else "No errors occurred."
+
+                    total_inputs = len(all_inputs)
+                    return (
+                        f"Processed {total_inputs} videos. {len(errors)} errors occurred.",
+                        error_summary,
+                        results_html,
+                        'all_transcriptions.json',
+                        'all_summaries.txt'
+                    )
+                except Exception as e:
+                    logging.error(f"Unexpected error in process_videos_with_error_handling: {str(e)}", exc_info=True)
+                    return (
+                        f"An unexpected error occurred: {str(e)}",
+                        str(e),
+                        "<div class='result-box error'><h3>Unexpected Error</h3><p>" + str(e) + "</p></div>",
+                        None,
+                        None
+                    )
+
+            def process_videos_wrapper(url_input, video_file, start_time, end_time, diarize, whisper_model,
+                                       custom_prompt_checkbox, custom_prompt, chunking_options_checkbox,
+                                       chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                                       use_multi_level_chunking, chunk_language, summarize_recursively, api_name,
+                                       api_key, keywords, use_cookies, cookies, batch_size,
+                                       timestamp_option, keep_original_video):
+                global result
+                try:
+                    logging.info("process_videos_wrapper(): process_videos_wrapper called")
+
+                    # Define file paths
+                    transcriptions_file = os.path.join('all_transcriptions.json')
+                    summaries_file = os.path.join('all_summaries.txt')
+
+                    # Delete existing files if they exist
+                    for file_path in [transcriptions_file, summaries_file]:
+                        try:
+                            if os.path.exists(file_path):
+                                os.remove(file_path)
+                                logging.info(f"Deleted existing file: {file_path}")
+                        except Exception as e:
+                            logging.warning(f"Failed to delete file {file_path}: {str(e)}")
+
+                    # Handle both URL input and file upload
+                    inputs = []
+                    if url_input:
+                        inputs.extend([url.strip() for url in url_input.split('\n') if url.strip()])
+                    if video_file is not None:
+                        # Assuming video_file is a file object with a 'name' attribute
+                        inputs.append(video_file.name)
+
+                    if not inputs:
+                        raise ValueError("No input provided. Please enter URLs or upload a video file.")
+                    try:
+                        result = process_videos_with_error_handling(
+                            inputs, start_time, end_time, diarize, whisper_model,
+                            custom_prompt_checkbox, custom_prompt, chunking_options_checkbox,
+                            chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                            use_multi_level_chunking, chunk_language, api_name,
+                            api_key, keywords, use_cookies, cookies, batch_size,
+                            timestamp_option, keep_original_video, summarize_recursively
+                        )
+                    except Exception as e:
+                        logging.error(
+                            f"process_videos_wrapper(): Error in process_videos_with_error_handling: {str(e)}",
+                            exc_info=True)
+
+                    logging.info("process_videos_wrapper(): process_videos_with_error_handling completed")
+
+                    # Ensure that result is a tuple with 5 elements
+                    if not isinstance(result, tuple) or len(result) != 5:
+                        raise ValueError(
+                            f"process_videos_wrapper(): Expected 5 outputs, but got {len(result) if isinstance(result, tuple) else 1}")
+
+                    return result
+                except Exception as e:
+                    logging.error(f"process_videos_wrapper(): Error in process_videos_wrapper: {str(e)}", exc_info=True)
+                    # Return a tuple with 5 elements in case of any error
+                    return (
+                        # progress_output
+                        f"process_videos_wrapper(): An error occurred: {str(e)}",
+                        # error_output
+                        str(e),
+                        # results_output
+                        f"<div class='error'>Error: {str(e)}</div>",
+                        # download_transcription
+                        None,
+                        # download_summary
+                        None
+                    )
+
+            # FIXME - remove dead args for process_url_with_metadata
+            @error_handler
+            def process_url_with_metadata(input_item, num_speakers, whisper_model, custom_prompt, offset, api_name,
+                                          api_key,
+                                          vad_filter, download_video_flag, download_audio, rolling_summarization,
+                                          detail_level, question_box, keywords, local_file_path, diarize, end_time=None,
+                                          include_timestamps=True, metadata=None, use_chunking=False,
+                                          chunk_options=None, keep_original_video=False, current_whisper_model="Blank"):
+
+                try:
+                    logging.info(f"Starting process_url_metadata for URL: {input_item}")
+                    # Create download path
+                    download_path = create_download_directory("Video_Downloads")
+                    logging.info(f"Download path created at: {download_path}")
+
+                    # Initialize info_dict
+                    info_dict = {}
+
+                    # Handle URL or local file
+                    if os.path.isfile(input_item):
+                        video_file_path = input_item
+                        unique_id = generate_unique_identifier(input_item)
+                        # Extract basic info from local file
+                        info_dict = {
+                            'webpage_url': unique_id,
+                            'title': os.path.basename(input_item),
+                            'description': "Local file",
+                            'channel_url': None,
+                            'duration': None,
+                            'channel': None,
+                            'uploader': None,
+                            'upload_date': None
+                        }
+                    else:
+                        # Extract video information
+                        with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
+                            try:
+                                full_info = ydl.extract_info(input_item, download=False)
+
+                                # Create a safe subset of info to log
+                                safe_info = {
+                                    'title': full_info.get('title', 'No title'),
+                                    'duration': full_info.get('duration', 'Unknown duration'),
+                                    'upload_date': full_info.get('upload_date', 'Unknown upload date'),
+                                    'uploader': full_info.get('uploader', 'Unknown uploader'),
+                                    'view_count': full_info.get('view_count', 'Unknown view count')
+                                }
+
+                                logging.debug(f"Full info extracted for {input_item}: {safe_info}")
+                            except Exception as e:
+                                logging.error(f"Error extracting video info: {str(e)}")
+                                return None, None, None, None, None, None
+
+                        # Filter the required metadata
+                        if full_info:
+                            info_dict = {
+                                'webpage_url': full_info.get('webpage_url', input_item),
+                                'title': full_info.get('title'),
+                                'description': full_info.get('description'),
+                                'channel_url': full_info.get('channel_url'),
+                                'duration': full_info.get('duration'),
+                                'channel': full_info.get('channel'),
+                                'uploader': full_info.get('uploader'),
+                                'upload_date': full_info.get('upload_date')
+                            }
+                            logging.debug(f"Filtered info_dict: {info_dict}")
+                        else:
+                            logging.error("Failed to extract video information")
+                            return None, None, None, None, None, None
+
+                        # Download video/audio
+                        logging.info("Downloading video/audio...")
+                        video_file_path = download_video(input_item, download_path, full_info, download_video_flag,
+                                                         current_whisper_model="Blank")
+                        if not video_file_path:
+                            logging.error(f"Failed to download video/audio from {input_item}")
+                            return None, None, None, None, None, None
+
+                    logging.info(f"Processing file: {video_file_path}")
+
+                    # Perform transcription
+                    logging.info("Starting transcription...")
+                    audio_file_path, segments = perform_transcription(video_file_path, offset, whisper_model,
+                                                                      vad_filter, diarize)
+
+                    if audio_file_path is None or segments is None:
+                        logging.error("Transcription failed or segments not available.")
+                        return None, None, None, None, None, None
+
+                    logging.info(f"Transcription completed. Number of segments: {len(segments)}")
+
+                    # Add metadata to segments
+                    segments_with_metadata = {
+                        "metadata": info_dict,
+                        "segments": segments
+                    }
+
+                    # Save segments with metadata to JSON file
+                    segments_json_path = os.path.splitext(audio_file_path)[0] + ".segments.json"
+                    with open(segments_json_path, 'w') as f:
+                        json.dump(segments_with_metadata, f, indent=2)
+
+                    # FIXME - why isnt this working?
+                    # Delete the .wav file after successful transcription
+                    files_to_delete = [audio_file_path]
+                    for file_path in files_to_delete:
+                        if file_path and os.path.exists(file_path):
+                            try:
+                                os.remove(file_path)
+                                logging.info(f"Successfully deleted file: {file_path}")
+                            except Exception as e:
+                                logging.warning(f"Failed to delete file {file_path}: {str(e)}")
+
+                    # Delete the mp4 file after successful transcription if not keeping original audio
+                    # Modify the file deletion logic to respect keep_original_video
+                    if not keep_original_video:
+                        files_to_delete = [audio_file_path, video_file_path]
+                        for file_path in files_to_delete:
+                            if file_path and os.path.exists(file_path):
+                                try:
+                                    os.remove(file_path)
+                                    logging.info(f"Successfully deleted file: {file_path}")
+                                except Exception as e:
+                                    logging.warning(f"Failed to delete file {file_path}: {str(e)}")
+                    else:
+                        logging.info(f"Keeping original video file: {video_file_path}")
+                        logging.info(f"Keeping original audio file: {audio_file_path}")
+
+                    # Process segments based on the timestamp option
+                    if not include_timestamps:
+                        segments = [{'Text': segment['Text']} for segment in segments]
+
+                    logging.info(f"Segments processed for timestamp inclusion: {segments}")
+
+                    # Extract text from segments
+                    transcription_text = extract_text_from_segments(segments)
+
+                    if transcription_text.startswith("Error:"):
+                        logging.error(f"Failed to extract transcription: {transcription_text}")
+                        return None, None, None, None, None, None
+
+                    # Use transcription_text instead of segments for further processing
+                    full_text_with_metadata = f"{json.dumps(info_dict, indent=2)}\n\n{transcription_text}"
+
+                    logging.debug(f"Full text with metadata extracted: {full_text_with_metadata[:100]}...")
+
+                    # Perform summarization if API is provided
+                    summary_text = None
+                    if api_name:
+                        # API key resolution handled at base of function if none provided
+                        api_key = api_key if api_key else None
+                        logging.info(f"Starting summarization with {api_name}...")
+                        summary_text = perform_summarization(api_name, full_text_with_metadata, custom_prompt, api_key)
+                        if summary_text is None:
+                            logging.error("Summarization failed.")
+                            return None, None, None, None, None, None
+                        logging.debug(f"Summarization completed: {summary_text[:100]}...")
+
+                    # Save transcription and summary
+                    logging.info("Saving transcription and summary...")
+                    download_path = create_download_directory("Audio_Processing")
+                    json_file_path, summary_file_path = save_transcription_and_summary(full_text_with_metadata,
+                                                                                       summary_text,
+                                                                                       download_path, info_dict)
+                    logging.info(f"Transcription saved to: {json_file_path}")
+                    logging.info(f"Summary saved to: {summary_file_path}")
+
+                    # Prepare keywords for database
+                    if isinstance(keywords, str):
+                        keywords_list = [kw.strip() for kw in keywords.split(',') if kw.strip()]
+                    elif isinstance(keywords, (list, tuple)):
+                        keywords_list = keywords
+                    else:
+                        keywords_list = []
+                    logging.info(f"Keywords prepared: {keywords_list}")
+
+                    # Add to database
+                    logging.info("Adding to database...")
+                    add_media_to_database(info_dict['webpage_url'], info_dict, full_text_with_metadata, summary_text,
+                                          keywords_list, custom_prompt, whisper_model)
+                    logging.info(f"Media added to database: {info_dict['webpage_url']}")
+
+                    return info_dict[
+                        'webpage_url'], full_text_with_metadata, summary_text, json_file_path, summary_file_path, info_dict
+
+                except Exception as e:
+                    logging.error(f"Error in process_url_with_metadata: {str(e)}", exc_info=True)
+                    return None, None, None, None, None, None
+
+            process_button.click(
+                fn=process_videos_wrapper,
+                inputs=[
+                    url_input, video_file_input, start_time_input, end_time_input, diarize_input, whisper_model_input,
+                    custom_prompt_checkbox, custom_prompt_input, chunking_options_checkbox,
+                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                    use_multi_level_chunking, chunk_language, summarize_recursively, api_name_input, api_key_input,
+                    keywords_input, use_cookies_input, cookies_input, batch_size_input,
+                    timestamp_option, keep_original_video
+                ],
+                outputs=[progress_output, error_output, results_output, download_transcription, download_summary]
+            )

App_Function_Libraries/Gradio_UI/Website_scraping_tab.py

@@ -0,0 +1,113 @@
+# Website_scraping_tab.py
+# Gradio UI for scraping websites
+
+# Imports
+#
+# External Imports
+import gradio as gr
+
+from App_Function_Libraries.Article_Summarization_Lib import scrape_and_summarize_multiple
+from App_Function_Libraries.DB_Manager import load_preset_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+
+
+#
+# Local Imports
+#
+#
+########################################################################################################################
+#
+# Functions:
+
+
+def create_website_scraping_tab():
+    with gr.TabItem("Website Scraping"):
+        gr.Markdown("# Scrape Websites & Summarize Articles using a Headless Chrome Browser!")
+        with gr.Row():
+            with gr.Column():
+                url_input = gr.Textbox(label="Article URLs", placeholder="Enter article URLs here, one per line", lines=5)
+                custom_article_title_input = gr.Textbox(label="Custom Article Titles (Optional, one per line)",
+                                                        placeholder="Enter custom titles for the articles, one per line",
+                                                        lines=5)
+                with gr.Row():
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=load_preset_prompts(),
+                                                visible=False)
+                with gr.Row():
+                    website_custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+""",
+                                                     lines=3,
+                                                     visible=False)
+
+                custom_prompt_checkbox.change(
+                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    inputs=[custom_prompt_checkbox],
+                    outputs=[website_custom_prompt_input, system_prompt_input]
+                )
+                preset_prompt_checkbox.change(
+                    fn=lambda x: gr.update(visible=x),
+                    inputs=[preset_prompt_checkbox],
+                    outputs=[preset_prompt]
+                )
+
+                def update_prompts(preset_name):
+                    prompts = update_user_prompt(preset_name)
+                    return (
+                        gr.update(value=prompts["user_prompt"], visible=True),
+                        gr.update(value=prompts["system_prompt"], visible=True)
+                    )
+
+                preset_prompt.change(
+                    update_prompts,
+                    inputs=preset_prompt,
+                    outputs=[website_custom_prompt_input, system_prompt_input]
+                )
+
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"], value=None, label="API Name (Mandatory for Summarization)")
+                api_key_input = gr.Textbox(label="API Key (Mandatory if API Name is specified)",
+                                           placeholder="Enter your API key here; Ignore if using Local API or Built-in API", type="password")
+                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)",
+                                            value="default,no_keyword_set", visible=True)
+
+                scrape_button = gr.Button("Scrape and Summarize")
+            with gr.Column():
+                result_output = gr.Textbox(label="Result", lines=20)
+
+                scrape_button.click(
+                    fn=scrape_and_summarize_multiple,
+                    inputs=[url_input, website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
+                            custom_article_title_input, system_prompt_input],
+                    outputs=result_output
+                )
+
+

App_Function_Libraries/Gradio_UI/Writing.py

@@ -0,0 +1,700 @@
+# Writing.py
+# Description: This file contains the functions that are used for writing in the Gradio UI.
+#
+# Imports
+import base64
+from datetime import datetime as datetime
+import logging
+import json
+import os
+#
+# External Imports
+import gradio as gr
+from PIL import Image
+import textstat
+#
+# Local Imports
+from App_Function_Libraries.Summarization_General_Lib import perform_summarization
+from App_Function_Libraries.Chat import chat
+#
+########################################################################################################################
+#
+# Functions:
+
+def adjust_tone(text, concise, casual, api_name, api_key):
+    tones = [
+        {"tone": "concise", "weight": concise},
+        {"tone": "casual", "weight": casual},
+        {"tone": "professional", "weight": 1 - casual},
+        {"tone": "expanded", "weight": 1 - concise}
+    ]
+    tones = sorted(tones, key=lambda x: x['weight'], reverse=True)[:2]
+
+    tone_prompt = " and ".join([f"{t['tone']} (weight: {t['weight']:.2f})" for t in tones])
+
+    prompt = f"Rewrite the following text to match these tones: {tone_prompt}. Text: {text}"
+    # Performing tone adjustment request...
+    adjusted_text = perform_summarization(api_name, text, prompt, api_key)
+
+    return adjusted_text
+
+
+def grammar_style_check(input_text, custom_prompt, api_name, api_key, system_prompt):
+    default_prompt = "Please analyze the following text for grammar and style. Offer suggestions for improvement and point out any misused words or incorrect spellings:\n\n"
+    full_prompt = custom_prompt if custom_prompt else default_prompt
+    full_text = full_prompt + input_text
+
+    return perform_summarization(api_name, full_text, custom_prompt, api_key, system_prompt)
+
+
+def create_grammar_style_check_tab():
+    with gr.TabItem("Grammar and Style Check"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Grammar and Style Check")
+                gr.Markdown("This utility checks the grammar and style of the provided text by feeding it to an LLM and returning suggestions for improvement.")
+                input_text = gr.Textbox(label="Input Text", lines=10)
+                custom_prompt_checkbox = gr.Checkbox(label="Use Custom Prompt", value=False, visible=True)
+                system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Please analyze the provided text for grammar and style. Offer any suggestions or points to improve you can identify. Additionally please point out any misuses of any words or incorrect spellings.", lines=5, visible=False)
+                custom_prompt_input = gr.Textbox(label="user Prompt",
+                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+**Bulleted Note Creation Guidelines**
+
+**Headings**:
+- Based on referenced topics, not categories like quotes or terms
+- Surrounded by **bold** formatting 
+- Not listed as bullet points
+- No space between headings and list items underneath
+
+**Emphasis**:
+- **Important terms** set in bold font
+- **Text ending in a colon**: also bolded
+
+**Review**:
+- Ensure adherence to specified format
+- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+""",
+                                                     lines=3,
+                                                     visible=False)
+                custom_prompt_checkbox.change(
+                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    inputs=[custom_prompt_checkbox],
+                    outputs=[custom_prompt_input, system_prompt_input]
+                )
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                    value=None,
+                    label="API for Grammar Check"
+                )
+                api_key_input = gr.Textbox(label="API Key (if not set in config.txt)", placeholder="Enter your API key here",
+                                               type="password")
+                check_grammar_button = gr.Button("Check Grammar and Style")
+
+            with gr.Column():
+                gr.Markdown("# Resulting Suggestions")
+                gr.Markdown("(Keep in mind the API used can affect the quality of the suggestions)")
+
+                output_text = gr.Textbox(label="Grammar and Style Suggestions", lines=15)
+
+            check_grammar_button.click(
+                fn=grammar_style_check,
+                inputs=[input_text, custom_prompt_input, api_name_input, api_key_input, system_prompt_input],
+                outputs=output_text
+            )
+
+
+def create_tone_adjustment_tab():
+    with gr.TabItem("Tone Analyzer & Editor"):
+        with gr.Row():
+            with gr.Column():
+                input_text = gr.Textbox(label="Input Text", lines=10)
+                concise_slider = gr.Slider(minimum=0, maximum=1, value=0.5, label="Concise vs Expanded")
+                casual_slider = gr.Slider(minimum=0, maximum=1, value=0.5, label="Casual vs Professional")
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                    value=None,
+                    label="API for Grammar Check"
+                )
+                api_key_input = gr.Textbox(label="API Key (if not set in config.txt)", placeholder="Enter your API key here",
+                                               type="password")
+                adjust_btn = gr.Button("Adjust Tone")
+
+            with gr.Column():
+                output_text = gr.Textbox(label="Adjusted Text", lines=15)
+
+                adjust_btn.click(
+                    adjust_tone,
+                    inputs=[input_text, concise_slider, casual_slider],
+                    outputs=output_text
+                )
+
+
+persona_prompts = {
+    "Hemingway": "As Ernest Hemingway, known for concise and straightforward prose, provide feedback on the following text:",
+    "Shakespeare": "Channel William Shakespeare's poetic style and provide feedback on the following text:",
+    "Jane Austen": "Embodying Jane Austen's wit and social commentary, critique the following text:",
+    "Stephen King": "With Stephen King's flair for suspense and horror, analyze the following text:",
+    "J.K. Rowling": "As J.K. Rowling, creator of the magical world of Harry Potter, review the following text:"
+}
+
+def generate_writing_feedback(text, persona, aspect, api_name, api_key):
+    if isinstance(persona, dict):  # If it's a character card
+        base_prompt = f"You are {persona['name']}. {persona['personality']}\n\nScenario: {persona['scenario']}\n\nRespond to the following message in character:"
+    else:  # If it's a regular persona
+        base_prompt = persona_prompts.get(persona, f"As {persona}, provide feedback on the following text:")
+
+    if aspect != "Overall":
+        prompt = f"{base_prompt}\n\nFocus specifically on the {aspect.lower()} in the following text:\n\n{text}"
+    else:
+        prompt = f"{base_prompt}\n\n{text}"
+
+    return perform_summarization(api_name, text, prompt, api_key, system_message="You are a helpful AI assistant. You will respond to the user as if you were the persona declared in the user prompt.")
+
+def generate_writing_prompt(persona, api_name, api_key):
+    prompt = f"Generate a writing prompt in the style of {persona}. The prompt should inspire a short story or scene that reflects {persona}'s typical themes and writing style."
+    #FIXME
+    return perform_summarization(api_name, prompt, "", api_key, system_message="You are a helpful AI assistant. You will respond to the user as if you were the persona declared in the user prompt." )
+
+def calculate_readability(text):
+    ease = textstat.flesch_reading_ease(text)
+    grade = textstat.flesch_kincaid_grade(text)
+    return f"Readability: Flesch Reading Ease: {ease:.2f}, Flesch-Kincaid Grade Level: {grade:.2f}"
+
+
+def generate_feedback_history_html(history):
+    html = "<h3>Recent Feedback History</h3>"
+    for entry in reversed(history):
+        html += f"<details><summary>{entry['persona']} Feedback</summary>"
+        html += f"<p><strong>Original Text:</strong> {entry['text'][:100]}...</p>"
+
+        feedback = entry.get('feedback')
+        if feedback:
+            html += f"<p><strong>Feedback:</strong> {feedback[:200]}...</p>"
+        else:
+            html += "<p><strong>Feedback:</strong> No feedback provided.</p>"
+
+        html += "</details>"
+    return html
+
+
+# FIXME
+def create_document_feedback_tab():
+    with gr.TabItem("Writing Feedback"):
+        with gr.Row():
+            with gr.Column(scale=2):
+                input_text = gr.Textbox(label="Your Writing", lines=10)
+                persona_dropdown = gr.Dropdown(
+                    label="Select Persona",
+                    choices=[
+                        "Agatha Christie",
+                        "Arthur Conan Doyle",
+                        "Charles Bukowski",
+                        "Charles Dickens",
+                        "Chinua Achebe",
+                        "Cormac McCarthy",
+                        "David Foster Wallace",
+                        "Edgar Allan Poe",
+                        "F. Scott Fitzgerald",
+                        "Flannery O'Connor",
+                        "Franz Kafka",
+                        "Fyodor Dostoevsky",
+                        "Gabriel Garcia Marquez",
+                        "George R.R. Martin",
+                        "George Orwell",
+                        "Haruki Murakami",
+                        "Hemingway",
+                        "Herman Melville",
+                        "Isabel Allende",
+                        "James Joyce",
+                        "Jane Austen",
+                        "J.K. Rowling",
+                        "J.R.R. Tolkien",
+                        "Jorge Luis Borges",
+                        "Kurt Vonnegut",
+                        "Leo Tolstoy",
+                        "Margaret Atwood",
+                        "Mark Twain",
+                        "Mary Shelley",
+                        "Milan Kundera",
+                        "Naguib Mahfouz",
+                        "Neil Gaiman",
+                        "Octavia Butler",
+                        "Philip K Dick",
+                        "Ray Bradbury",
+                        "Salman Rushdie",
+                        "Shakespeare",
+                        "Stephen King",
+                        "Toni Morrison",
+                        "T.S. Eliot",
+                        "Ursula K. Le Guin",
+                        "Virginia Woolf",
+                        "Virginia Woolf",
+                        "Zadie Smith"],
+                    value="Hemingway"
+                )
+                custom_persona_name = gr.Textbox(label="Custom Persona Name")
+                custom_persona_description = gr.Textbox(label="Custom Persona Description", lines=3)
+                add_custom_persona_button = gr.Button("Add Custom Persona")
+                aspect_dropdown = gr.Dropdown(
+                    label="Focus Feedback On",
+                    choices=["Overall", "Grammar", "Word choice", "Structure of delivery", "Character Development", "Character Dialogue", "Descriptive Language", "Plot Structure"],
+                    value="Overall"
+                )
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    value=None,
+                    label="API for Feedback"
+                )
+                api_key_input = gr.Textbox(label="API Key (if not set in config.txt)", type="password")
+                get_feedback_button = gr.Button("Get Feedback")
+                generate_prompt_button = gr.Button("Generate Writing Prompt")
+
+            with gr.Column(scale=2):
+                feedback_output = gr.Textbox(label="Feedback", lines=15)
+                readability_output = gr.Textbox(label="Readability Metrics")
+                feedback_history_display = gr.HTML(label="Feedback History")
+
+        with gr.Row():
+            compare_personas = gr.CheckboxGroup(
+                choices=[
+                    "Agatha Christie",
+                    "Arthur Conan Doyle",
+                    "Charles Bukowski",
+                    "Charles Dickens",
+                    "Chinua Achebe",
+                    "Cormac McCarthy",
+                    "David Foster Wallace",
+                    "Edgar Allan Poe",
+                    "F. Scott Fitzgerald",
+                    "Flannery O'Connor",
+                    "Franz Kafka",
+                    "Fyodor Dostoevsky",
+                    "Gabriel Garcia Marquez",
+                    "George R.R. Martin",
+                    "George Orwell",
+                    "Haruki Murakami",
+                    "Hemingway",
+                    "Herman Melville",
+                    "Isabel Allende",
+                    "James Joyce",
+                    "Jane Austen",
+                    "J.K. Rowling",
+                    "J.R.R. Tolkien",
+                    "Jorge Luis Borges",
+                    "Kurt Vonnegut",
+                    "Leo Tolstoy",
+                    "Margaret Atwood",
+                    "Mark Twain",
+                    "Mary Shelley",
+                    "Milan Kundera",
+                    "Naguib Mahfouz",
+                    "Neil Gaiman",
+                    "Octavia Butler",
+                    "Philip K Dick",
+                    "Ray Bradbury",
+                    "Salman Rushdie",
+                    "Shakespeare",
+                    "Stephen King",
+                    "Toni Morrison",
+                    "T.S. Eliot",
+                    "Ursula K. Le Guin",
+                    "Virginia Woolf",
+                    "Virginia Woolf",
+                    "Zadie Smith"],
+                label="Compare Multiple Persona's Feedback at Once"
+            )
+        with gr.Row():
+            compare_button = gr.Button("Compare Feedback")
+
+    feedback_history = gr.State([])
+
+    def add_custom_persona(name, description):
+        updated_choices = persona_dropdown.choices + [name]
+        persona_prompts[name] = f"As {name}, {description}, provide feedback on the following text:"
+        return gr.update(choices=updated_choices)
+
+    def update_feedback_history(current_text, persona, feedback):
+        # Ensure feedback_history.value is initialized and is a list
+        if feedback_history.value is None:
+            feedback_history.value = []
+
+        history = feedback_history.value
+
+        # Append the new entry to the history
+        history.append({"text": current_text, "persona": persona, "feedback": feedback})
+
+        # Keep only the last 5 entries in the history
+        feedback_history.value = history[-10:]
+
+        # Generate and return the updated HTML
+        return generate_feedback_history_html(feedback_history.value)
+
+    def compare_feedback(text, selected_personas, api_name, api_key):
+        results = []
+        for persona in selected_personas:
+            feedback = generate_writing_feedback(text, persona, "Overall", api_name, api_key)
+            results.append(f"### {persona}'s Feedback:\n{feedback}\n\n")
+        return "\n".join(results)
+
+    add_custom_persona_button.click(
+        fn=add_custom_persona,
+        inputs=[custom_persona_name, custom_persona_description],
+        outputs=persona_dropdown
+    )
+
+    get_feedback_button.click(
+        fn=lambda text, persona, aspect, api_name, api_key: (
+            generate_writing_feedback(text, persona, aspect, api_name, api_key),
+            calculate_readability(text),
+            update_feedback_history(text, persona, generate_writing_feedback(text, persona, aspect, api_name, api_key))
+        ),
+        inputs=[input_text, persona_dropdown, aspect_dropdown, api_name_input, api_key_input],
+        outputs=[feedback_output, readability_output, feedback_history_display]
+    )
+
+    compare_button.click(
+        fn=compare_feedback,
+        inputs=[input_text, compare_personas, api_name_input, api_key_input],
+        outputs=feedback_output
+    )
+
+    generate_prompt_button.click(
+        fn=generate_writing_prompt,
+        inputs=[persona_dropdown, api_name_input, api_key_input],
+        outputs=input_text
+    )
+
+    return input_text, feedback_output, readability_output, feedback_history_display
+
+
+def create_creative_writing_tab():
+    with gr.TabItem("Creative Writing Assistant"):
+        gr.Markdown("# Utility to be added...")
+
+
+#FIXME - change to use chat function
+def chat_with_character(user_message, history, char_data, api_name_input, api_key):
+    if char_data is None:
+        return history, "Please import a character card first."
+
+    bot_message = generate_writing_feedback(user_message, char_data['name'], "Overall", api_name_input,
+                                            api_key)
+    history.append((user_message, bot_message))
+    return history, ""
+
+def import_character_card(file):
+    if file is None:
+        logging.warning("No file provided for character card import")
+        return None
+    try:
+        if file.name.lower().endswith(('.png', '.webp')):
+            logging.info(f"Attempting to import character card from image: {file.name}")
+            json_data = extract_json_from_image(file)
+            if json_data:
+                logging.info("JSON data extracted from image, attempting to parse")
+                return import_character_card_json(json_data)
+            else:
+                logging.warning("No JSON data found in the image")
+        else:
+            logging.info(f"Attempting to import character card from JSON file: {file.name}")
+            content = file.read().decode('utf-8')
+            return import_character_card_json(content)
+    except Exception as e:
+        logging.error(f"Error importing character card: {e}")
+    return None
+
+
+def import_character_card_json(json_content):
+    try:
+        # Remove any leading/trailing whitespace
+        json_content = json_content.strip()
+
+        # Log the first 100 characters of the content
+        logging.debug(f"JSON content (first 100 chars): {json_content[:100]}...")
+
+        card_data = json.loads(json_content)
+        logging.debug(f"Parsed JSON data keys: {list(card_data.keys())}")
+        if 'spec' in card_data and card_data['spec'] == 'chara_card_v2':
+            logging.info("Detected V2 character card")
+            return card_data['data']
+        else:
+            logging.info("Assuming V1 character card")
+            return card_data
+    except json.JSONDecodeError as e:
+        logging.error(f"JSON decode error: {e}")
+        logging.error(f"Problematic JSON content: {json_content[:500]}...")
+    except Exception as e:
+        logging.error(f"Unexpected error parsing JSON: {e}")
+    return None
+
+
+def extract_json_from_image(image_file):
+    logging.debug(f"Attempting to extract JSON from image: {image_file.name}")
+    try:
+        with Image.open(image_file) as img:
+            logging.debug("Image opened successfully")
+            metadata = img.info
+            if 'chara' in metadata:
+                logging.debug("Found 'chara' in image metadata")
+                chara_content = metadata['chara']
+                logging.debug(f"Content of 'chara' metadata (first 100 chars): {chara_content[:100]}...")
+                try:
+                    decoded_content = base64.b64decode(chara_content).decode('utf-8')
+                    logging.debug(f"Decoded content (first 100 chars): {decoded_content[:100]}...")
+                    return decoded_content
+                except Exception as e:
+                    logging.error(f"Error decoding base64 content: {e}")
+
+            logging.debug("'chara' not found in metadata, checking for base64 encoded data")
+            raw_data = img.tobytes()
+            possible_json = raw_data.split(b'{', 1)[-1].rsplit(b'}', 1)[0]
+            if possible_json:
+                try:
+                    decoded = base64.b64decode(possible_json).decode('utf-8')
+                    if decoded.startswith('{') and decoded.endswith('}'):
+                        logging.debug("Found and decoded base64 JSON data")
+                        return '{' + decoded + '}'
+                except Exception as e:
+                    logging.error(f"Error decoding base64 data: {e}")
+
+            logging.warning("No JSON data found in the image")
+    except Exception as e:
+        logging.error(f"Error extracting JSON from image: {e}")
+    return None
+
+def load_chat_history(file):
+    try:
+        content = file.read().decode('utf-8')
+        chat_data = json.loads(content)
+        return chat_data['history'], chat_data['character']
+    except Exception as e:
+        logging.error(f"Error loading chat history: {e}")
+        return None, None
+
+def create_character_card_interaction_tab():
+    with gr.TabItem("Chat with a Character Card"):
+        gr.Markdown("# Chat with a Character Card")
+        with gr.Row():
+            with gr.Column(scale=1):
+                character_card_upload = gr.File(label="Upload Character Card")
+                import_card_button = gr.Button("Import Character Card")
+                load_characters_button = gr.Button("Load Existing Characters")
+                from App_Function_Libraries.Chat_related_functions import get_character_names
+                character_dropdown = gr.Dropdown(label="Select Character", choices=get_character_names())
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
+                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                    value=None,
+                    label="API for Interaction"
+                )
+                api_key_input = gr.Textbox(label="API Key (if not set in config.txt)",
+                                           placeholder="Enter your API key here", type="password")
+                temperature_slider = gr.Slider(minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature")
+                import_chat_button = gr.Button("Import Chat History")
+                chat_file_upload = gr.File(label="Upload Chat History JSON", visible=False)
+
+
+            with gr.Column(scale=2):
+                chat_history = gr.Chatbot(label="Conversation")
+                user_input = gr.Textbox(label="Your message")
+                send_message_button = gr.Button("Send Message")
+                regenerate_button = gr.Button("Regenerate Last Message")
+                save_chat_button = gr.Button("Save This Chat")
+                save_status = gr.Textbox(label="Save Status", interactive=False)
+
+    character_data = gr.State(None)
+
+    def import_chat_history(file, current_history, char_data):
+        loaded_history, char_name = load_chat_history(file)
+        if loaded_history is None:
+            return current_history, char_data, "Failed to load chat history."
+
+        # Check if the loaded chat is for the current character
+        if char_data and char_data.get('name') != char_name:
+            return current_history, char_data, f"Warning: Loaded chat is for character '{char_name}', but current character is '{char_data.get('name')}'. Chat not imported."
+
+        # If no character is selected, try to load the character from the chat
+        if not char_data:
+            new_char_data = load_character(char_name)[0]
+            if new_char_data:
+                char_data = new_char_data
+            else:
+                return current_history, char_data, f"Warning: Character '{char_name}' not found. Please select the character manually."
+
+        return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully."
+
+    def import_character(file):
+        card_data = import_character_card(file)
+        if card_data:
+            from App_Function_Libraries.Chat_related_functions import save_character
+            save_character(card_data)
+            return card_data, gr.update(choices=get_character_names())
+        else:
+            return None, gr.update()
+
+    def load_character(name):
+        from App_Function_Libraries.Chat_related_functions import load_characters
+        characters = load_characters()
+        char_data = characters.get(name)
+        if char_data:
+            first_message = char_data.get('first_mes', "Hello! I'm ready to chat.")
+            return char_data, [(None, first_message)] if first_message else []
+        return None, []
+
+    def character_chat_wrapper(message, history, char_data, api_endpoint, api_key, temperature):
+        logging.debug("Entered character_chat_wrapper")
+        if char_data is None:
+            return "Please select a character first.", history
+
+        # Prepare the character's background information
+        char_background = f"""
+        Name: {char_data.get('name', 'Unknown')}
+        Description: {char_data.get('description', 'N/A')}
+        Personality: {char_data.get('personality', 'N/A')}
+        Scenario: {char_data.get('scenario', 'N/A')}
+        """
+
+        # Prepare the system prompt for character impersonation
+        system_message = f"""You are roleplaying as the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI.
+
+        {char_background}
+
+        Additional instructions: {char_data.get('post_history_instructions', '')}
+        """
+
+        # Prepare media_content and selected_parts
+        media_content = {
+            'id': char_data.get('name'),
+            'title': char_data.get('name', 'Unknown Character'),
+            'content': char_background,
+            'description': char_data.get('description', ''),
+            'personality': char_data.get('personality', ''),
+            'scenario': char_data.get('scenario', '')
+        }
+        selected_parts = ['description', 'personality', 'scenario']
+
+        prompt = char_data.get('post_history_instructions', '')
+
+        # Prepare the input for the chat function
+        if not history:
+            full_message = f"{prompt}\n\n{message}" if prompt else message
+        else:
+            full_message = message
+
+        # Call the chat function
+        bot_message = chat(
+            message,
+            history,
+            media_content,
+            selected_parts,
+            api_endpoint,
+            api_key,
+            prompt,
+            temperature,
+            system_message
+        )
+
+        # Update history
+        history.append((message, bot_message))
+        return history
+
+    def save_chat_history(history, character_name):
+        # Create the Saved_Chats folder if it doesn't exist
+        save_directory = "Saved_Chats"
+        os.makedirs(save_directory, exist_ok=True)
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        filename = f"chat_history_{character_name}_{timestamp}.json"
+        filepath = os.path.join(save_directory, filename)
+
+        chat_data = {
+            "character": character_name,
+            "timestamp": timestamp,
+            "history": history
+        }
+
+        try:
+            with open(filepath, 'w', encoding='utf-8') as f:
+                json.dump(chat_data, f, ensure_ascii=False, indent=2)
+            return filepath
+        except Exception as e:
+            return f"Error saving chat: {str(e)}"
+
+    def save_current_chat(history, char_data):
+        if not char_data or not history:
+            return "No chat to save or character not selected."
+
+        character_name = char_data.get('name', 'Unknown')
+        result = save_chat_history(history, character_name)
+        if result.startswith("Error"):
+            return result
+        return f"Chat saved successfully as {result}"
+
+    def regenerate_last_message(history, char_data, api_name, api_key, temperature):
+        if not history:
+            return history
+
+        last_user_message = history[-1][0]
+        new_history = history[:-1]
+
+        return character_chat_wrapper(last_user_message, new_history, char_data, api_name, api_key, temperature)
+
+    import_chat_button.click(
+        fn=lambda: gr.update(visible=True),
+        outputs=chat_file_upload
+    )
+
+    chat_file_upload.change(
+        fn=import_chat_history,
+        inputs=[chat_file_upload, chat_history, character_data],
+        outputs=[chat_history, character_data, save_status]
+    )
+
+    import_card_button.click(
+        fn=import_character,
+        inputs=[character_card_upload],
+        outputs=[character_data, character_dropdown]
+    )
+
+    load_characters_button.click(
+        fn=lambda: gr.update(choices=get_character_names()),
+        outputs=character_dropdown
+    )
+
+    character_dropdown.change(
+        fn=load_character,
+        inputs=[character_dropdown],
+        outputs=[character_data, chat_history]
+    )
+
+    send_message_button.click(
+        fn=character_chat_wrapper,
+        inputs=[user_input, chat_history, character_data, api_name_input, api_key_input, temperature_slider],
+        outputs=[chat_history]
+    ).then(lambda: "", outputs=user_input)
+
+    regenerate_button.click(
+        fn=regenerate_last_message,
+        inputs=[chat_history, character_data, api_name_input, api_key_input, temperature_slider],
+        outputs=[chat_history]
+    )
+
+    save_chat_button.click(
+        fn=save_current_chat,
+        inputs=[chat_history, character_data],
+        outputs=[save_status]
+    )
+
+    return character_data, chat_history, user_input
+
+
+def create_mikupad_tab():
+    with gr.TabItem("Mikupad"):
+        gr.Markdown("I Wish. Gradio won't embed it successfully...")
+
+#
+# End of Writing.py
+########################################################################################################################