Upload 29 files

App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py

@@ -104,7 +104,7 @@ def create_audio_processing_tab():
 
                 api_name_input = gr.Dropdown(
                     choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
                     value=None,
                     label="API for Summarization (Optional)"
                 )

App_Function_Libraries/Gradio_UI/Backup_Functionality.py

@@ -0,0 +1,71 @@
+# Backup_Functionality.py
+# Functionality for exporting items as markdown files
+#
+# Imports:
+import os
+import shutil
+import gradio as gr
+#
+# Local Imports:
+from App_Function_Libraries.DB.DB_Manager import create_automated_backup, db_path, backup_dir
+#
+# End of Imports
+#######################################################################################################################
+#
+# Functions:
+
+def create_backup():
+    backup_file = create_automated_backup(db_path, backup_dir)
+    return f"Backup created: {backup_file}"
+
+
+def list_backups():
+    backups = [f for f in os.listdir(backup_dir) if f.endswith('.db')]
+    return "\n".join(backups)
+
+
+def restore_backup(backup_name: str) -> str:
+    backup_path_location: str = os.path.join(str(backup_dir), backup_name)
+    if os.path.exists(backup_path_location):
+        shutil.copy2(str(backup_path_location), str(db_path))
+        return f"Database restored from {backup_name}"
+    else:
+        return "Backup file not found"
+
+
+def create_backup_tab():
+    with gr.Tab("Create Backup"):
+        gr.Markdown("# Create a backup of the database")
+        gr.Markdown("This will create a backup of the database in the backup directory(the default backup directory is `/tldw_DB_Backups/')")
+        with gr.Row():
+            with gr.Column():
+                create_button = gr.Button("Create Backup")
+                create_output = gr.Textbox(label="Result")
+            with gr.Column():
+                create_button.click(create_backup, inputs=[], outputs=create_output)
+
+
+def create_view_backups_tab():
+    with gr.TabItem("View Backups"):
+        gr.Markdown("# Browse available backups")
+        with gr.Row():
+            with gr.Column():
+                view_button = gr.Button("View Backups")
+            with gr.Column():
+                backup_list = gr.Textbox(label="Available Backups")
+                view_button.click(list_backups, inputs=[], outputs=backup_list)
+
+
+def create_restore_backup_tab():
+    with gr.TabItem("Restore Backup"):
+        gr.Markdown("# Restore a backup of the database")
+        with gr.Column():
+            backup_input = gr.Textbox(label="Backup Filename")
+            restore_button = gr.Button("Restore")
+        with gr.Column():
+            restore_output = gr.Textbox(label="Result")
+            restore_button.click(restore_backup, inputs=[backup_input], outputs=restore_output)
+
+#
+# End of Functions
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Chat_Workflows.py

@@ -0,0 +1,189 @@
+# Chat_Workflows.py
+# Description: UI for Chat Workflows
+#
+# Imports
+import json
+import logging
+from pathlib import Path
+#
+# External Imports
+import gradio as gr
+#
+from App_Function_Libraries.Gradio_UI.Chat_ui import process_with_llm
+#
+############################################################################################################
+#
+# Functions:
+
+# Load workflows from a JSON file
+json_path = Path('./Helper_Scripts/Workflows/Workflows.json')
+with json_path.open('r') as f:
+    workflows = json.load(f)
+
+
+# FIXME - broken Completely. Doesn't work.
+def chat_workflows_tab():
+    with gr.TabItem("Chat Workflows"):
+        gr.Markdown("# Workflows using LLMs")
+
+        with gr.Row():
+            workflow_selector = gr.Dropdown(label="Select Workflow", choices=[wf['name'] for wf in workflows])
+            api_selector = gr.Dropdown(
+                label="Select API Endpoint",
+                choices=["OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
+                         "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                value="OpenAI"
+            )
+            api_key_input = gr.Textbox(label="API Key (optional)", type="password")
+
+        context_input = gr.Textbox(label="Initial Context (optional)", lines=5)
+
+        # Create a container for dynamic components
+        with gr.Column() as dynamic_components:
+            prompt_displays = []
+            user_inputs = []
+            output_boxes = []
+            process_buttons = []
+            regenerate_buttons = []
+
+            # Create the maximum number of components needed
+            max_steps = max(len(wf['prompts']) for wf in workflows)
+            for i in range(max_steps):
+                prompt_displays.append(gr.Markdown(visible=False))
+                user_inputs.append(gr.Textbox(label=f"Your Input", lines=2, visible=False))
+                output_boxes.append(gr.Textbox(label=f"AI Output", lines=5, visible=False))
+                with gr.Row():
+                    process_buttons.append(gr.Button(f"Process Step {i + 1}", visible=False))
+                    regenerate_buttons.append(gr.Button(f"🔄 Regenerate", visible=False))
+
+        def update_workflow_ui(workflow_name):
+            selected_workflow = next(wf for wf in workflows if wf['name'] == workflow_name)
+            num_prompts = len(selected_workflow['prompts'])
+
+            prompt_updates = []
+            input_updates = []
+            output_updates = []
+            button_updates = []
+            regenerate_updates = []
+
+            for i in range(max_steps):
+                if i < num_prompts:
+                    prompt_updates.append(
+                        gr.update(value=f"**Step {i + 1}:** {selected_workflow['prompts'][i]}", visible=True))
+                    input_updates.append(gr.update(value="", visible=True, interactive=(i == 0)))
+                    output_updates.append(gr.update(value="", visible=True))
+                    button_updates.append(gr.update(visible=(i == 0)))
+                    regenerate_updates.append(gr.update(visible=False))
+                else:
+                    prompt_updates.append(gr.update(visible=False))
+                    input_updates.append(gr.update(visible=False))
+                    output_updates.append(gr.update(visible=False))
+                    button_updates.append(gr.update(visible=False))
+                    regenerate_updates.append(gr.update(visible=False))
+
+            return prompt_updates + input_updates + output_updates + button_updates + regenerate_updates
+
+        def process(context, workflow_name, api_endpoint, api_key, step, *user_inputs):
+            try:
+                selected_workflow = next(wf for wf in workflows if wf['name'] == workflow_name)
+            except StopIteration:
+                # Handle the case where no matching workflow is found
+                error_message = f"No workflow found with name: {workflow_name}"
+                logging.error(error_message)
+                return [gr.update(value=error_message)] * (
+                            len(prompt_displays) + len(user_inputs) + len(output_boxes) + len(process_buttons) + len(
+                        regenerate_buttons))
+
+            # Ensure we don't go out of bounds
+            if step >= len(selected_workflow['prompts']):
+                error_message = f"Step {step} is out of range for workflow: {workflow_name}"
+                logging.error(error_message)
+                return [gr.update(value=error_message)] * (
+                            len(prompt_displays) + len(user_inputs) + len(output_boxes) + len(process_buttons) + len(
+                        regenerate_buttons))
+
+            # Build up the context from previous steps
+            full_context = context + "\n\n"
+            for i in range(step + 1):
+                full_context += f"Question: {selected_workflow['prompts'][i]}\n"
+                full_context += f"Answer: {user_inputs[i]}\n"
+                if i < step:
+                    full_context += f"AI Output: {output_boxes[i].value}\n\n"
+
+            try:
+                result = process_with_llm(workflow_name, full_context, selected_workflow['prompts'][step], api_endpoint,
+                                          api_key)
+            except Exception as e:
+                error_message = f"Error processing with LLM: {str(e)}"
+                logging.error(error_message)
+                result = error_message
+
+            updates = []
+            for i in range(max_steps):
+                if i == step:
+                    updates.extend([
+                        gr.update(),  # Markdown (prompt_displays)
+                        gr.update(interactive=False),  # Textbox (user_inputs)
+                        gr.update(value=result),  # Textbox (output_boxes)
+                        gr.update(visible=False),  # Button (process_buttons)
+                        gr.update(visible=True)  # Button (regenerate_buttons)
+                    ])
+                elif i == step + 1:
+                    updates.extend([
+                        gr.update(),  # Markdown (prompt_displays)
+                        gr.update(interactive=True),  # Textbox (user_inputs)
+                        gr.update(),  # Textbox (output_boxes)
+                        gr.update(visible=True),  # Button (process_buttons)
+                        gr.update(visible=False)  # Button (regenerate_buttons)
+                    ])
+                elif i > step + 1:
+                    updates.extend([
+                        gr.update(),  # Markdown (prompt_displays)
+                        gr.update(interactive=False),  # Textbox (user_inputs)
+                        gr.update(),  # Textbox (output_boxes)
+                        gr.update(visible=False),  # Button (process_buttons)
+                        gr.update(visible=False)  # Button (regenerate_buttons)
+                    ])
+                else:
+                    updates.extend([
+                        gr.update(),  # Markdown (prompt_displays)
+                        gr.update(interactive=False),  # Textbox (user_inputs)
+                        gr.update(),  # Textbox (output_boxes)
+                        gr.update(visible=False),  # Button (process_buttons)
+                        gr.update(visible=True)  # Button (regenerate_buttons)
+                    ])
+
+            return updates
+
+        # Set up event handlers
+        workflow_selector.change(
+            update_workflow_ui,
+            inputs=[workflow_selector],
+            outputs=prompt_displays + user_inputs + output_boxes + process_buttons + regenerate_buttons
+        )
+
+        # Set up process button click events
+        for i, button in enumerate(process_buttons):
+            button.click(
+                fn=lambda context, wf_name, api_endpoint, api_key, *inputs, step=i: process(context, wf_name,
+                                                                                            api_endpoint, api_key, step,
+                                                                                            *inputs),
+                inputs=[context_input, workflow_selector, api_selector, api_key_input] + user_inputs,
+                outputs=prompt_displays + user_inputs + output_boxes + process_buttons + regenerate_buttons
+            ).then(lambda: gr.update(value=""), outputs=[user_inputs[i]])
+
+        # Set up regenerate button click events
+        for i, button in enumerate(regenerate_buttons):
+            button.click(
+                fn=lambda context, wf_name, api_endpoint, api_key, *inputs, step=i: process(context, wf_name,
+                                                                                            api_endpoint, api_key, step,
+                                                                                            *inputs),
+                inputs=[context_input, workflow_selector, api_selector, api_key_input] + user_inputs,
+                outputs=prompt_displays + user_inputs + output_boxes + process_buttons + regenerate_buttons
+            )
+
+    return workflow_selector, api_selector, api_key_input, context_input, dynamic_components
+
+#
+# End of script
+############################################################################################################

App_Function_Libraries/Gradio_UI/Chat_ui.py

@@ -8,7 +8,6 @@ import logging
 import os
 import sqlite3
 from datetime import datetime
-from pathlib import Path
 #
 # External Imports
 import gradio as gr
@@ -96,7 +95,7 @@ def chat_wrapper(message, history, media_content, selected_parts, api_endpoint,
         # Generate bot response
         bot_message = chat(full_message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt,
                            temperature, system_prompt)
-
+        logging.debug(f"Bot message being returned: {bot_message}")
         if save_conversation:
             # Add assistant message to the database
             add_chat_message(conversation_id, "assistant", bot_message)
@@ -281,11 +280,10 @@ def create_chat_interface():
             inputs=[preset_prompt_checkbox],
             outputs=[preset_prompt]
         )
-
         submit.click(
             chat_wrapper,
-            inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt,
-                    conversation_id, save_conversation, temperature, system_prompt_input],
+            inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, conversation_id,
+                    save_conversation, temperature, system_prompt_input],
             outputs=[msg, chatbot, conversation_id]
         ).then(  # Clear the message box after submission
             lambda x: gr.update(value=""),
@@ -777,6 +775,7 @@ def create_chat_management_tab():
             with gr.TabItem("Edit"):
                 chat_content = gr.TextArea(label="Chat Content (JSON)", lines=20, max_lines=50)
                 save_button = gr.Button("Save Changes")
+                delete_button = gr.Button("Delete Conversation", variant="stop")
 
             with gr.TabItem("Preview"):
                 chat_preview = gr.HTML(label="Chat Preview")
@@ -898,6 +897,38 @@ def create_chat_management_tab():
                 logging.error(f"Unexpected error in save_conversation: {e}")
                 return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>"
 
+        def delete_conversation(selected, conversation_mapping):
+            if not selected or selected not in conversation_mapping:
+                return "Please select a conversation before deleting.", "<p>No changes made</p>", gr.update(choices=[])
+
+            conversation_id = conversation_mapping[selected]
+
+            try:
+                with db.get_connection() as conn:
+                    cursor = conn.cursor()
+
+                    # Delete messages associated with the conversation
+                    cursor.execute("DELETE FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
+
+                    # Delete the conversation itself
+                    cursor.execute("DELETE FROM ChatConversations WHERE id = ?", (conversation_id,))
+
+                    conn.commit()
+
+                # Update the conversation list
+                remaining_conversations = [choice for choice in conversation_mapping.keys() if choice != selected]
+                updated_mapping = {choice: conversation_mapping[choice] for choice in remaining_conversations}
+
+                return "Conversation deleted successfully.", "<p>Conversation deleted</p>", gr.update(choices=remaining_conversations)
+            except sqlite3.Error as e:
+                conn.rollback()
+                logging.error(f"Database error in delete_conversation: {e}")
+                return f"Error deleting conversation: {str(e)}", "<p>Error occurred while deleting</p>", gr.update()
+            except Exception as e:
+                conn.rollback()
+                logging.error(f"Unexpected error in delete_conversation: {e}")
+                return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>", gr.update()
+
         def parse_formatted_content(formatted_content):
             lines = formatted_content.split('\n')
             conversation_id = int(lines[0].split(': ')[1])
@@ -938,145 +969,14 @@ def create_chat_management_tab():
             outputs=[result_message, chat_preview]
         )
 
-    return search_query, search_button, conversation_list, conversation_mapping, chat_content, save_button, result_message, chat_preview
-
-
-# Load workflows from a JSON file
-json_path = Path('./Helper_Scripts/Workflows/Workflows.json')
-with json_path.open('r') as f:
-    workflows = json.load(f)
-
-
-# FIXME - broken Completely. Doesn't work.
-def chat_workflows_tab():
-    with gr.TabItem("Chat Workflows"):
-        gr.Markdown("# Workflows using LLMs")
-
-        with gr.Row():
-            workflow_selector = gr.Dropdown(label="Select Workflow", choices=[wf['name'] for wf in workflows])
-            api_selector = gr.Dropdown(
-                label="Select API Endpoint",
-                choices=["OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                         "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                value="OpenAI"
-            )
-            api_key_input = gr.Textbox(label="API Key (optional)", type="password")
-
-        context_input = gr.Textbox(label="Initial Context (optional)", lines=5)
-
-        # Create a container for dynamic components
-        with gr.Column() as dynamic_components:
-            prompt_displays = []
-            user_inputs = []
-            output_boxes = []
-            process_buttons = []
-            regenerate_buttons = []
-
-            # Create the maximum number of components needed
-            max_steps = max(len(wf['prompts']) for wf in workflows)
-            for i in range(max_steps):
-                prompt_displays.append(gr.Markdown(visible=False))
-                user_inputs.append(gr.Textbox(label=f"Your Response", lines=2, visible=False))
-                output_boxes.append(gr.Textbox(label=f"AI Output", lines=5, visible=False))
-                with gr.Row():
-                    process_buttons.append(gr.Button(f"Process Step {i + 1}", visible=False))
-                    regenerate_buttons.append(gr.Button(f"🔄 Regenerate", visible=False))
-
-        def update_workflow_ui(workflow_name):
-            selected_workflow = next(wf for wf in workflows if wf['name'] == workflow_name)
-            num_prompts = len(selected_workflow['prompts'])
-
-            prompt_updates = []
-            input_updates = []
-            output_updates = []
-            button_updates = []
-            regenerate_updates = []
-
-            for i in range(max_steps):
-                if i < num_prompts:
-                    prompt_updates.append(
-                        gr.update(value=f"**Step {i + 1}:** {selected_workflow['prompts'][i]}", visible=True))
-                    input_updates.append(gr.update(value="", visible=True, interactive=(i == 0)))
-                    output_updates.append(gr.update(value="", visible=True))
-                    button_updates.append(gr.update(visible=(i == 0)))
-                    regenerate_updates.append(gr.update(visible=False))
-                else:
-                    prompt_updates.append(gr.update(visible=False))
-                    input_updates.append(gr.update(visible=False))
-                    output_updates.append(gr.update(visible=False))
-                    button_updates.append(gr.update(visible=False))
-                    regenerate_updates.append(gr.update(visible=False))
-
-            return prompt_updates + input_updates + output_updates + button_updates + regenerate_updates
-
-        def process(context, user_inputs, workflow_name, api_endpoint, api_key, step):
-            selected_workflow = next(wf for wf in workflows if wf['name'] == workflow_name)
-
-            # Build up the context from previous steps
-            full_context = context + "\n\n"
-            for i in range(step + 1):
-                full_context += f"Question: {selected_workflow['prompts'][i]}\n"
-                full_context += f"Answer: {user_inputs[i]}\n"
-                if i < step:
-                    full_context += f"AI Output: {output_boxes[i].value}\n\n"
-
-            result = process_with_llm(workflow_name, full_context, selected_workflow['prompts'][step], api_endpoint,
-                                      api_key)
-
-            prompt_updates = [gr.update() for _ in range(max_steps)]
-            input_updates = []
-            output_updates = [gr.update() for _ in range(max_steps)]
-            button_updates = []
-            regenerate_updates = []
-
-            for i in range(len(selected_workflow['prompts'])):
-                if i == step:
-                    regenerate_updates.append(gr.update(visible=True))
-                elif i == step + 1:
-                    input_updates.append(gr.update(interactive=True))
-                    button_updates.append(gr.update(visible=True))
-                    regenerate_updates.append(gr.update(visible=False))
-                elif i > step + 1:
-                    input_updates.append(gr.update(interactive=False))
-                    button_updates.append(gr.update(visible=False))
-                    regenerate_updates.append(gr.update(visible=False))
-                else:
-                    input_updates.append(gr.update(interactive=False))
-                    button_updates.append(gr.update(visible=False))
-                    regenerate_updates.append(gr.update(visible=True))
-
-            return [result] + prompt_updates + input_updates + output_updates + button_updates + regenerate_updates
-
-        # Set up event handlers
-        workflow_selector.change(
-            update_workflow_ui,
-            inputs=[workflow_selector],
-            outputs=prompt_displays + user_inputs + output_boxes + process_buttons + regenerate_buttons
+        delete_button.click(
+            delete_conversation,
+            inputs=[conversation_list, conversation_mapping],
+            outputs=[result_message, chat_preview, conversation_list]
         )
 
-        # Set up process button click events
-        for i, button in enumerate(process_buttons):
-            button.click(
-                fn=lambda context, *user_inputs, wf_name, api_endpoint, api_key, step=i: process(context, user_inputs,
-                                                                                                 wf_name, api_endpoint,
-                                                                                                 api_key, step),
-                inputs=[context_input] + user_inputs + [workflow_selector, api_selector, api_key_input],
-                outputs=[output_boxes[
-                             i]] + prompt_displays + user_inputs + output_boxes + process_buttons + regenerate_buttons
-            )
-
-        # Set up regenerate button click events
-        for i, button in enumerate(regenerate_buttons):
-            button.click(
-                fn=lambda context, *user_inputs, wf_name, api_endpoint, api_key, step=i: process(context, user_inputs,
-                                                                                                 wf_name, api_endpoint,
-                                                                                                 api_key, step),
-                inputs=[context_input] + user_inputs + [workflow_selector, api_selector, api_key_input],
-                outputs=[output_boxes[
-                             i]] + prompt_displays + user_inputs + output_boxes + process_buttons + regenerate_buttons
-            )
+    return search_query, search_button, conversation_list, conversation_mapping, chat_content, save_button, delete_button, result_message, chat_preview
 
-    return workflow_selector, api_selector, api_key_input, context_input, dynamic_components
 
 
 # Mock function to simulate LLM processing

App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py

@@ -0,0 +1,60 @@
+###################################################################################################
+# Evaluations_Benchmarks_tab.py - Gradio code for G-Eval testing
+# We will use the G-Eval API to evaluate the quality of the generated summaries.
+
+import gradio as gr
+from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import run_geval
+
+def create_geval_tab():
+    with gr.Tab("G-Eval"):
+        gr.Markdown("# G-Eval Summarization Evaluation")
+        with gr.Row():
+            with gr.Column():
+                document_input = gr.Textbox(label="Source Document", lines=10)
+                summary_input = gr.Textbox(label="Summary", lines=5)
+                api_name_input = gr.Dropdown(
+                    choices=["OpenAI", "Anthropic", "Cohere", "Groq", "OpenRouter", "DeepSeek", "HuggingFace", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "Local-LLM", "Ollama"],
+                    label="Select API"
+                )
+                api_key_input = gr.Textbox(label="API Key (if required)", type="password")
+                evaluate_button = gr.Button("Evaluate Summary")
+            with gr.Column():
+                output = gr.Textbox(label="Evaluation Results", lines=10)
+
+        evaluate_button.click(
+            fn=run_geval,
+            inputs=[document_input, summary_input, api_name_input, api_key_input],
+            outputs=output
+        )
+
+    return document_input, summary_input, api_name_input, api_key_input, evaluate_button, output
+
+
+def create_infinite_bench_tab():
+    with gr.Tab("Infinite Bench"):
+        gr.Markdown("# Infinite Bench Evaluation (Coming Soon)")
+        with gr.Row():
+            with gr.Column():
+                api_name_input = gr.Dropdown(
+                    choices=["OpenAI", "Anthropic", "Cohere", "Groq", "OpenRouter", "DeepSeek", "HuggingFace", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "Local-LLM", "Ollama"],
+                    label="Select API"
+                )
+                api_key_input = gr.Textbox(label="API Key (if required)", type="password")
+                evaluate_button = gr.Button("Evaluate Summary")
+            with gr.Column():
+                output = gr.Textbox(label="Evaluation Results", lines=10)
+
+        # evaluate_button.click(
+        #     fn=run_geval,
+        #     inputs=[api_name_input, api_key_input],
+        #     outputs=output
+        # )
+
+    return api_name_input, api_key_input, evaluate_button, output
+
+
+# If you want to run this as a standalone Gradio app
+if __name__ == "__main__":
+    with gr.Blocks() as demo:
+        create_geval_tab()
+    demo.launch()

App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py

@@ -75,10 +75,11 @@ def create_summarize_explain_tab():
                     api_endpoint = gr.Dropdown(
                         choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
                                  "OpenRouter",
-                                 "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                                 "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace", "Custom-OpenAI-API"],
                         value=None,
-                        label="API for Summarization (Optional)"
+                        label="API to be used for request (Mandatory)"
                     )
+                with gr.Row():
                     api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here",
                                                type="password")
                 with gr.Row():

App_Function_Libraries/Gradio_UI/Export_Functionality.py

@@ -8,7 +8,7 @@ import shutil
 import tempfile
 from typing import List, Dict, Optional, Tuple
 import gradio as gr
-from App_Function_Libraries.DB.DB_Manager import DatabaseError, create_automated_backup, db_path, backup_dir
+from App_Function_Libraries.DB.DB_Manager import DatabaseError
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import fetch_item_details, fetch_items_by_keyword, browse_items
 
 logger = logging.getLogger(__name__)
@@ -264,51 +264,3 @@ def create_export_tab():
     )
 
 
-
-def create_backup():
-    backup_file = create_automated_backup(db_path, backup_dir)
-    return f"Backup created: {backup_file}"
-
-def list_backups():
-    backups = [f for f in os.listdir(backup_dir) if f.endswith('.db')]
-    return "\n".join(backups)
-
-def restore_backup(backup_name):
-    backup_path = os.path.join(backup_dir, backup_name)
-    if os.path.exists(backup_path):
-        shutil.copy2(backup_path, db_path)
-        return f"Database restored from {backup_name}"
-    else:
-        return "Backup file not found"
-
-
-def create_backup_tab():
-    with gr.Tab("Create Backup"):
-        gr.Markdown("# Create a backup of the database")
-        with gr.Row():
-            with gr.Column():
-                create_button = gr.Button("Create Backup")
-                create_output = gr.Textbox(label="Result")
-            with gr.Column():
-                create_button.click(create_backup, inputs=[], outputs=create_output)
-
-def create_view_backups_tab():
-    with gr.TabItem("View Backups"):
-        gr.Markdown("# Browse available backups")
-        with gr.Row():
-            with gr.Column():
-                view_button = gr.Button("View Backups")
-            with gr.Column():
-                backup_list = gr.Textbox(label="Available Backups")
-                view_button.click(list_backups, inputs=[], outputs=backup_list)
-
-
-def create_restore_backup_tab():
-    with gr.TabItem("Restore Backup"):
-        gr.Markdown("# Restore a backup of the database")
-        with gr.Column():
-            backup_input = gr.Textbox(label="Backup Filename")
-            restore_button = gr.Button("Restore")
-        with gr.Column():
-            restore_output = gr.Textbox(label="Result")
-            restore_button.click(restore_backup, inputs=[backup_input], outputs=restore_output)

App_Function_Libraries/Gradio_UI/Import_Functionality.py

@@ -231,7 +231,7 @@ def create_import_item_tab():
                 auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
                 api_name_input = gr.Dropdown(
                 choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                         "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                         "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
                 label="API for Auto-summarization"
                 )
                 api_key_input = gr.Textbox(label="API Key", type="password")

App_Function_Libraries/Gradio_UI/Media_edit.py

@@ -10,7 +10,7 @@ import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import add_prompt, update_media_content, db, add_or_update_prompt, \
-    load_prompt_details
+    load_prompt_details, fetch_keywords_for_media, update_keywords_for_media
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_prompt_dropdown
 from App_Function_Libraries.DB.SQLite_DB import fetch_item_details
 
@@ -32,32 +32,58 @@ def create_media_edit_tab():
         prompt_input = gr.Textbox(label="Edit Prompt", lines=3)
         summary_input = gr.Textbox(label="Edit Summary", lines=5)
 
+        # Adding keyword input box for editing
+        keywords_input = gr.Textbox(label="Edit Keywords (comma-separated)", placeholder="Enter keywords here...")
+
         update_button = gr.Button("Update Media Content")
         status_message = gr.Textbox(label="Status", interactive=False)
 
+        # Function to update the dropdown with search results
         search_button.click(
             fn=update_dropdown,
             inputs=[search_query_input, search_type_input],
             outputs=[items_output, item_mapping]
         )
 
+        # Function to load selected media content including keywords
         def load_selected_media_content(selected_item, item_mapping):
             if selected_item and item_mapping and selected_item in item_mapping:
                 media_id = item_mapping[selected_item]
-                # FIXME - fetch_item_details is not handled by DB_Manager!
                 content, prompt, summary = fetch_item_details(media_id)
-                return content, prompt, summary
-            return "No item selected or invalid selection", "", ""
 
+                # Fetch keywords for the selected item
+                keywords = fetch_keywords_for_media(media_id)
+                keywords_str = ", ".join(keywords) if keywords else ""
+
+                return content, prompt, summary, keywords_str
+            return "No item selected or invalid selection", "", "", ""
+
+        # Load the selected media content and associated keywords
         items_output.change(
             fn=load_selected_media_content,
             inputs=[items_output, item_mapping],
-            outputs=[content_input, prompt_input, summary_input]
+            outputs=[content_input, prompt_input, summary_input, keywords_input]
         )
 
+        # Function to update media content, prompt, summary, and keywords
+        def update_media_with_keywords(selected_item, item_mapping, content, prompt, summary, keywords):
+            if selected_item and item_mapping and selected_item in item_mapping:
+                media_id = item_mapping[selected_item]
+
+                # Split keywords into a list
+                keyword_list = [kw.strip() for kw in keywords.split(",") if kw.strip()]
+
+                # Update content, prompt, summary, and keywords in the database
+                status = update_media_content(media_id, content, prompt, summary)
+                keyword_status = update_keywords_for_media(media_id, keyword_list)
+
+                return f"{status}\nKeywords: {keyword_status}"
+            return "No item selected or invalid selection"
+
+        # Update button click event
         update_button.click(
-            fn=update_media_content,
-            inputs=[items_output, item_mapping, content_input, prompt_input, summary_input],
+            fn=update_media_with_keywords,
+            inputs=[items_output, item_mapping, content_input, prompt_input, summary_input, keywords_input],
             outputs=status_message
         )
 

App_Function_Libraries/Gradio_UI/Media_wiki_tab.py

@@ -7,9 +7,11 @@ from threading import Thread
 #
 # 3rd-party Imports
 import gradio as gr
+import yaml
+from ruamel.yaml import YAML
 #
 # Local Imports
-from App_Function_Libraries.MediaWiki.Media_Wiki import import_mediawiki_dump
+from App_Function_Libraries.MediaWiki.Media_Wiki import import_mediawiki_dump, config
 #
 #######################################################################################################################
 #
@@ -32,6 +34,13 @@ def create_mediawiki_import_tab():
                 )
                 chunk_size = gr.Slider(minimum=100, maximum=2000, value=1000, step=100, label="Chunk Size")
                 chunk_overlap = gr.Slider(minimum=0, maximum=500, value=100, step=10, label="Chunk Overlap")
+                # FIXME - Add checkbox for 'Enable Summarization upon ingestion' for API summarization of chunks
+                # api_endpoint = gr.Dropdown(label="Select API Endpoint",
+                #                            choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
+                #                                     "Mistral", "OpenRouter",
+                #                                     "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama",
+                #                                     "HuggingFace"])
+                # api_key = gr.Textbox(label="API Key (if required)", type="password")
                 import_button = gr.Button("Import MediaWiki Dump")
                 cancel_button = gr.Button("Cancel Import", visible=False)
             with gr.Column():
@@ -61,7 +70,7 @@ def create_mediawiki_import_tab():
         cancel_flag = False
 
         def run_import(file_path, wiki_name, namespaces, skip_redirects, single_item, chunk_method, chunk_size,
-                       chunk_overlap, progress=gr.Progress()):
+                       chunk_overlap, progress=gr.Progress()):#, api_endpoint=None, api_key=None):
             validation_error = validate_inputs(file_path, wiki_name, namespaces)
             if validation_error:
                 return gr.update(), gr.update(), validation_error
@@ -90,7 +99,9 @@ def create_mediawiki_import_tab():
                         skip_redirects=skip_redirects,
                         chunk_options=chunk_options,
                         single_item=single_item,
-                        progress_callback=progress
+                        progress_callback=progress,
+#                        api_name=api_endpoint,
+#                        api_key=api_key
                 ):
                     if progress_info.startswith("Found"):
                         status_text += f"\n## Parsing\n- {progress_info}\n"
@@ -126,7 +137,7 @@ def create_mediawiki_import_tab():
         import_button.click(
             run_import,
             inputs=[file_path, wiki_name, namespaces, skip_redirects, single_item, chunk_method, chunk_size,
-                    chunk_overlap],
+                    chunk_overlap],#, api_endpoint, api_key],
             outputs=[cancel_button, import_button, output]
         )
 
@@ -137,6 +148,193 @@ def create_mediawiki_import_tab():
 
     return file_path, wiki_name, namespaces, skip_redirects, single_item, chunk_method, chunk_size, chunk_overlap, import_button, output
 
+
+class PreservedTokenSafeDumper(yaml.SafeDumper):
+    def represent_scalar(self, tag, value, style=None):
+        if style is None and isinstance(value, str) and '\n' in value:
+            style = '|'
+        return super().represent_scalar(tag, value, style)
+
+
+def update_yaml_file(file_path, updates):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    def format_value(value):
+        if isinstance(value, bool):
+            return str(value).lower()
+        elif isinstance(value, (int, float)):
+            return str(value)
+        elif isinstance(value, list):
+            return '[' + ', '.join(map(str, value)) + ']'
+        else:
+            return f"'{value}'"
+
+    def update_line(line, updates, prefix=''):
+        for key, value in updates.items():
+            full_key = f"{prefix}{key}:" if prefix else f"{key}:"
+            if line.strip().startswith(full_key):
+                indentation = line[:line.index(full_key)]
+                if isinstance(value, dict):
+                    return line  # Keep the line as is for nested structures
+                else:
+                    return f"{indentation}{full_key} {format_value(value)}\n"
+        return line
+
+    updated_lines = []
+    current_prefix = ''
+    for line in lines:
+        stripped = line.strip()
+        if stripped and not stripped.startswith('#'):
+            indent = len(line) - len(line.lstrip())
+            if indent == 0:
+                current_prefix = ''
+            elif ':' in stripped and not stripped.endswith(':'):
+                current_prefix = '.'.join(current_prefix.split('.')[:-1]) + '.' if current_prefix else ''
+
+            updated_line = update_line(line, updates, current_prefix)
+
+            if updated_line == line and ':' in stripped and stripped.endswith(':'):
+                key = stripped[:-1].strip()
+                if current_prefix:
+                    current_prefix += f"{key}."
+                else:
+                    current_prefix = f"{key}."
+
+            updated_lines.append(updated_line)
+        else:
+            updated_lines.append(line)
+
+    with open(file_path, 'w') as file:
+        file.writelines(updated_lines)
+
+#
+#
+#######################################################################################################################
+#
+# Config tab
+
+yaml = YAML()
+yaml.preserve_quotes = True
+yaml.indent(mapping=2, sequence=4, offset=2)
+
+def load_config():
+    config_path = os.path.join('Config_Files', 'mediawiki_import_config.yaml')
+    with open(config_path, 'r') as file:
+        return yaml.load(file)
+
+def save_config(updated_config):
+    config_path = os.path.join('Config_Files', 'mediawiki_import_config.yaml')
+    config = load_config()
+
+
+def create_mediawiki_config_tab():
+    with gr.TabItem("MediaWiki Import Configuration"):
+        gr.Markdown("# MediaWiki Import Configuration (Broken currently/doesn't work)")
+        with gr.Row():
+            with gr.Column():
+                namespaces = gr.Textbox(label="Default Namespaces (comma-separated integers)",
+                                        value=','.join(map(str, config['import']['default_namespaces'])))
+                skip_redirects = gr.Checkbox(label="Skip Redirects by Default",
+                                             value=config['import']['default_skip_redirects'])
+                single_item = gr.Checkbox(label="Import as Single Item by Default",
+                                          value=config['import']['single_item_default'])
+                batch_size = gr.Number(value=config['import']['batch_size'], label="Batch Size")
+
+                chunk_method = gr.Dropdown(
+                    choices=config['chunking']['methods'],
+                    value=config['chunking']['default_method'],
+                    label="Default Chunking Method"
+                )
+                chunk_size = gr.Slider(minimum=100, maximum=2000, value=config['chunking']['default_size'], step=100,
+                                       label="Default Chunk Size")
+                chunk_overlap = gr.Slider(minimum=0, maximum=500, value=config['chunking']['default_overlap'], step=10,
+                                          label="Default Chunk Overlap")
+
+            with gr.Column():
+                max_workers = gr.Slider(minimum=1, maximum=16, value=config['processing']['max_workers'], step=1,
+                                        label="Max Worker Threads")
+
+                embedding_provider = gr.Dropdown(
+                    choices=['openai', 'local', 'huggingface'],
+                    value=config['embeddings']['provider'],
+                    label="Embedding Provider"
+                )
+                embedding_model = gr.Textbox(label="Embedding Model", value=config['embeddings']['model'])
+                api_key = gr.Textbox(label="API Key (if required)", type="password",
+                                     value=config['embeddings'].get('api_key', ''))
+                local_embedding_url = gr.Textbox(label="Local Embedding URL",
+                                                 value=config['embeddings'].get('local_url', ''))
+
+                checkpoints_enabled = gr.Checkbox(label="Enable Checkpoints", value=config['checkpoints']['enabled'])
+                checkpoint_directory = gr.Textbox(label="Checkpoint Directory", value=config['checkpoints']['directory'])
+
+                max_retries = gr.Number(value=config['error_handling']['max_retries'], label="Max Retries")
+                retry_delay = gr.Number(value=config['error_handling']['retry_delay'], label="Retry Delay (seconds)")
+
+        save_config_button = gr.Button("Save Configuration")
+        config_output = gr.Markdown(label="Configuration Status")
+
+        def update_config_from_ui(namespaces, skip_redirects, single_item, batch_size, chunk_method, chunk_size,
+                                  chunk_overlap, max_workers, embedding_provider, embedding_model, api_key,
+                                  local_embedding_url, checkpoints_enabled, checkpoint_directory, max_retries,
+                                  retry_delay):
+            current_config = load_config()
+            updated_config = {}
+
+            if namespaces != ','.join(map(str, current_config['import']['default_namespaces'])):
+                updated_config.setdefault('import', {})['default_namespaces'] = [int(ns.strip()) for ns in
+                                                                                 namespaces.split(',') if ns.strip()]
+            if skip_redirects != current_config['import']['default_skip_redirects']:
+                updated_config.setdefault('import', {})['default_skip_redirects'] = skip_redirects
+            if single_item != current_config['import']['single_item_default']:
+                updated_config.setdefault('import', {})['single_item_default'] = single_item
+            if int(batch_size) != current_config['import']['batch_size']:
+                updated_config.setdefault('import', {})['batch_size'] = int(batch_size)
+            if chunk_method != current_config['chunking']['default_method']:
+                updated_config.setdefault('chunking', {})['default_method'] = chunk_method
+            if int(chunk_size) != current_config['chunking']['default_size']:
+                updated_config.setdefault('chunking', {})['default_size'] = int(chunk_size)
+            if int(chunk_overlap) != current_config['chunking']['default_overlap']:
+                updated_config.setdefault('chunking', {})['default_overlap'] = int(chunk_overlap)
+            if int(max_workers) != current_config['processing']['max_workers']:
+                updated_config.setdefault('processing', {})['max_workers'] = int(max_workers)
+            if embedding_provider != current_config['embeddings']['provider']:
+                updated_config.setdefault('embeddings', {})['provider'] = embedding_provider
+            if embedding_model != current_config['embeddings']['model']:
+                updated_config.setdefault('embeddings', {})['model'] = embedding_model
+            if api_key != current_config['embeddings'].get('api_key', ''):
+                updated_config.setdefault('embeddings', {})['api_key'] = api_key
+            if local_embedding_url != current_config['embeddings'].get('local_url', ''):
+                updated_config.setdefault('embeddings', {})['local_url'] = local_embedding_url
+            if checkpoints_enabled != current_config['checkpoints']['enabled']:
+                updated_config.setdefault('checkpoints', {})['enabled'] = checkpoints_enabled
+            if checkpoint_directory != current_config['checkpoints']['directory']:
+                updated_config.setdefault('checkpoints', {})['directory'] = checkpoint_directory
+            if int(max_retries) != current_config['error_handling']['max_retries']:
+                updated_config.setdefault('error_handling', {})['max_retries'] = int(max_retries)
+            if int(retry_delay) != current_config['error_handling']['retry_delay']:
+                updated_config.setdefault('error_handling', {})['retry_delay'] = int(retry_delay)
+
+            return updated_config
+
+        def save_config_callback(*args):
+            updated_config = update_config_from_ui(*args)
+            save_config(updated_config)
+            return "Configuration saved successfully."
+
+        save_config_button.click(
+            save_config_callback,
+            inputs=[namespaces, skip_redirects, single_item, batch_size, chunk_method, chunk_size,
+                    chunk_overlap, max_workers, embedding_provider, embedding_model, api_key,
+                    local_embedding_url, checkpoints_enabled, checkpoint_directory, max_retries, retry_delay],
+            outputs=config_output
+        )
+
+    return namespaces, skip_redirects, single_item, batch_size, chunk_method, chunk_size, chunk_overlap, max_workers, \
+           embedding_provider, embedding_model, api_key, local_embedding_url, checkpoints_enabled, checkpoint_directory, \
+           max_retries, retry_delay, save_config_button, config_output
+
 #
 # End of MediaWiki Import Tab
 #######################################################################################################################

App_Function_Libraries/Gradio_UI/Podcast_tab.py

@@ -99,7 +99,7 @@ def create_podcast_tab():
 
                 podcast_api_name_input = gr.Dropdown(
                     choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp",
-                             "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
+                             "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
                     value=None,
                     label="API Name for Summarization (Optional)"
                 )

App_Function_Libraries/Gradio_UI/Search_Tab.py

@@ -1,426 +1,404 @@
-# Search_Tab.py
-# Description: This file contains the code for the search tab in the Gradio UI
-#
-# Imports
-import html
-import logging
-import sqlite3
-
-#
-# External Imports
-import gradio as gr
-
-from App_Function_Libraries.DB.DB_Manager import view_database, search_and_display_items
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_detailed_view
-#from App_Function_Libraries.RAG.RAG_Libary_2 import rag_search
-
-#
-# Local Imports
-#
-#
-###################################################################################################
-#
-# Functions:
-
-logger = logging.getLogger()
-
-
-
-
-# FIXME - SQL functions to be moved to DB_Manager
-def search_prompts(query):
-    try:
-        conn = sqlite3.connect('prompts.db')
-        cursor = conn.cursor()
-        cursor.execute("SELECT name, details, system, user FROM Prompts WHERE name LIKE ? OR details LIKE ?",
-                       (f"%{query}%", f"%{query}%"))
-        results = cursor.fetchall()
-        conn.close()
-        return results
-    except sqlite3.Error as e:
-        print(f"Error searching prompts: {e}")
-        return []
-
-
-
-
-
-
-
-def display_search_results(query):
-    if not query.strip():
-        return "Please enter a search query."
-
-    results = search_prompts(query)
-
-    # Debugging: Print the results to the console to see what is being returned
-    print(f"Processed search results for query '{query}': {results}")
-
-    if results:
-        result_md = "## Search Results:\n"
-        for result in results:
-            # Debugging: Print each result to see its format
-            print(f"Result item: {result}")
-
-            if len(result) == 2:
-                name, details = result
-                result_md += f"**Title:** {name}\n\n**Description:** {details}\n\n---\n"
-
-            elif len(result) == 4:
-                name, details, system, user = result
-                result_md += f"**Title:** {name}\n\n"
-                result_md += f"**Description:** {details}\n\n"
-                result_md += f"**System Prompt:** {system}\n\n"
-                result_md += f"**User Prompt:** {user}\n\n"
-                result_md += "---\n"
-            else:
-                result_md += "Error: Unexpected result format.\n\n---\n"
-        return result_md
-    return "No results found."
-
-
-def create_viewing_tab():
-    with gr.TabItem("View Database"):
-        gr.Markdown("# View Database Entries")
-        with gr.Row():
-            with gr.Column():
-                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
-                page_number = gr.Number(value=1, label="Page Number", precision=0)
-                view_button = gr.Button("View Page")
-                next_page_button = gr.Button("Next Page")
-                previous_page_button = gr.Button("Previous Page")
-            with gr.Column():
-                results_display = gr.HTML()
-                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
-
-        def update_page(page, entries_per_page):
-            results, pagination, total_pages = view_database(page, entries_per_page)
-            next_disabled = page >= total_pages
-            prev_disabled = page <= 1
-            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(interactive=not prev_disabled)
-
-        def go_to_next_page(current_page, entries_per_page):
-            next_page = current_page + 1
-            return update_page(next_page, entries_per_page)
-
-        def go_to_previous_page(current_page, entries_per_page):
-            previous_page = max(1, current_page - 1)
-            return update_page(previous_page, entries_per_page)
-
-        view_button.click(
-            fn=update_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        next_page_button.click(
-            fn=go_to_next_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        previous_page_button.click(
-            fn=go_to_previous_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-
-def create_search_tab():
-    with gr.TabItem("Search / Detailed View"):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Search across all ingested items in the Database")
-                gr.Markdown(" by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
-                search_button = gr.Button("Search")
-                items_output = gr.Dropdown(label="Select Item", choices=[])
-                item_mapping = gr.State({})
-                prompt_summary_output = gr.HTML(label="Prompt & Summary", visible=True)
-
-                search_button.click(
-                    fn=update_dropdown,
-                    inputs=[search_query_input, search_type_input],
-                    outputs=[items_output, item_mapping]
-                )
-            with gr.Column():
-                content_output = gr.Markdown(label="Content", visible=True)
-                items_output.change(
-                    fn=update_detailed_view,
-                    inputs=[items_output, item_mapping],
-                    outputs=[prompt_summary_output, content_output]
-                )
-
-def create_search_summaries_tab():
-    with gr.TabItem("Search/View Title+Summary "):
-        gr.Markdown("# Search across all ingested items in the Database and review their summaries")
-        gr.Markdown("Search by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
-        with gr.Row():
-            with gr.Column():
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                             label="Search By")
-                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
-                page_number = gr.Number(value=1, label="Page Number", precision=0)
-                char_count_input = gr.Number(value=5000, label="Amount of characters to display from the main content",
-                                             precision=0)
-            with gr.Column():
-                search_button = gr.Button("Search")
-                next_page_button = gr.Button("Next Page")
-                previous_page_button = gr.Button("Previous Page")
-                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
-        search_results_output = gr.HTML()
-
-
-        def update_search_page(query, search_type, page, entries_per_page, char_count):
-            # Ensure char_count is a positive integer
-            char_count = max(1, int(char_count)) if char_count else 5000
-            results, pagination, total_pages = search_and_display_items(query, search_type, page, entries_per_page, char_count)
-            next_disabled = page >= total_pages
-            prev_disabled = page <= 1
-            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
-                interactive=not prev_disabled)
-
-        def go_to_next_search_page(query, search_type, current_page, entries_per_page, char_count):
-            next_page = current_page + 1
-            return update_search_page(query, search_type, next_page, entries_per_page, char_count)
-
-        def go_to_previous_search_page(query, search_type, current_page, entries_per_page, char_count):
-            previous_page = max(1, current_page - 1)
-            return update_search_page(query, search_type, previous_page, entries_per_page, char_count)
-
-        search_button.click(
-            fn=update_search_page,
-            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
-            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        next_page_button.click(
-            fn=go_to_next_search_page,
-            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
-            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        previous_page_button.click(
-            fn=go_to_previous_search_page,
-            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
-            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-
-
-def create_prompt_view_tab():
-    with gr.TabItem("View Prompt Database"):
-        gr.Markdown("# View Prompt Database Entries")
-        with gr.Row():
-            with gr.Column():
-                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
-                page_number = gr.Number(value=1, label="Page Number", precision=0)
-                view_button = gr.Button("View Page")
-                next_page_button = gr.Button("Next Page")
-                previous_page_button = gr.Button("Previous Page")
-            with gr.Column():
-                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
-        results_display = gr.HTML()
-
-        # FIXME - SQL functions to be moved to DB_Manager
-        def view_database(page, entries_per_page):
-            offset = (page - 1) * entries_per_page
-            try:
-                with sqlite3.connect('prompts.db') as conn:
-                    cursor = conn.cursor()
-                    cursor.execute('''
-                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
-                        FROM Prompts p
-                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
-                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
-                        GROUP BY p.id
-                        ORDER BY p.name
-                        LIMIT ? OFFSET ?
-                    ''', (entries_per_page, offset))
-                    prompts = cursor.fetchall()
-
-                    cursor.execute('SELECT COUNT(*) FROM Prompts')
-                    total_prompts = cursor.fetchone()[0]
-
-                results = ""
-                for prompt in prompts:
-                    # Escape HTML special characters and replace newlines with <br> tags
-                    title = html.escape(prompt[0]).replace('\n', '<br>')
-                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
-                    system_prompt = html.escape(prompt[2] or '')
-                    user_prompt = html.escape(prompt[3] or '')
-                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
-
-                    results += f"""
-                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
-                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
-                            <div><strong>Title:</strong> {title}</div>
-                            <div><strong>Details:</strong> {details}</div>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>User Prompt:</strong>
-                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>System Prompt:</strong>
-                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>Keywords:</strong> {keywords}
-                        </div>
-                    </div>
-                    """
-
-                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
-                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
-
-                return results, pagination, total_pages
-            except sqlite3.Error as e:
-                return f"<p>Error fetching prompts: {e}</p>", "Error", 0
-
-        def update_page(page, entries_per_page):
-            results, pagination, total_pages = view_database(page, entries_per_page)
-            next_disabled = page >= total_pages
-            prev_disabled = page <= 1
-            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
-                interactive=not prev_disabled)
-
-        def go_to_next_page(current_page, entries_per_page):
-            next_page = current_page + 1
-            return update_page(next_page, entries_per_page)
-
-        def go_to_previous_page(current_page, entries_per_page):
-            previous_page = max(1, current_page - 1)
-            return update_page(previous_page, entries_per_page)
-
-        view_button.click(
-            fn=update_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        next_page_button.click(
-            fn=go_to_next_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        previous_page_button.click(
-            fn=go_to_previous_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-
-
-def create_prompt_search_tab():
-    with gr.TabItem("Search Prompts"):
-        gr.Markdown("# Search and View Prompt Details")
-        gr.Markdown("Currently has all of the https://github.com/danielmiessler/fabric prompts already available")
-        with gr.Row():
-            with gr.Column():
-                search_query_input = gr.Textbox(label="Search Prompts", placeholder="Enter your search query...")
-                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
-                page_number = gr.Number(value=1, label="Page Number", precision=0)
-            with gr.Column():
-                search_button = gr.Button("Search Prompts")
-                next_page_button = gr.Button("Next Page")
-                previous_page_button = gr.Button("Previous Page")
-                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
-        search_results_output = gr.HTML()
-
-        def search_and_display_prompts(query, page, entries_per_page):
-            offset = (page - 1) * entries_per_page
-            try:
-                # FIXME - SQL functions to be moved to DB_Manager
-                with sqlite3.connect('prompts.db') as conn:
-                    cursor = conn.cursor()
-                    cursor.execute('''
-                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
-                        FROM Prompts p
-                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
-                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
-                        WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
-                        GROUP BY p.id
-                        ORDER BY p.name
-                        LIMIT ? OFFSET ?
-                    ''', (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', entries_per_page, offset))
-                    prompts = cursor.fetchall()
-
-                    cursor.execute('''
-                        SELECT COUNT(DISTINCT p.id)
-                        FROM Prompts p
-                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
-                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
-                        WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
-                    ''', (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%'))
-                    total_prompts = cursor.fetchone()[0]
-
-                results = ""
-                for prompt in prompts:
-                    title = html.escape(prompt[0]).replace('\n', '<br>')
-                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
-                    system_prompt = html.escape(prompt[2] or '')
-                    user_prompt = html.escape(prompt[3] or '')
-                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
-
-                    results += f"""
-                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
-                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
-                            <div><strong>Title:</strong> {title}</div>
-                            <div><strong>Details:</strong> {details}</div>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>User Prompt:</strong>
-                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>System Prompt:</strong>
-                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
-                        </div>
-                        <div style="margin-top: 10px;">
-                            <strong>Keywords:</strong> {keywords}
-                        </div>
-                    </div>
-                    """
-
-                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
-                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
-
-                return results, pagination, total_pages
-            except sqlite3.Error as e:
-                return f"<p>Error searching prompts: {e}</p>", "Error", 0
-
-        def update_search_page(query, page, entries_per_page):
-            results, pagination, total_pages = search_and_display_prompts(query, page, entries_per_page)
-            next_disabled = page >= total_pages
-            prev_disabled = page <= 1
-            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(interactive=not prev_disabled)
-
-        def go_to_next_search_page(query, current_page, entries_per_page):
-            next_page = current_page + 1
-            return update_search_page(query, next_page, entries_per_page)
-
-        def go_to_previous_search_page(query, current_page, entries_per_page):
-            previous_page = max(1, current_page - 1)
-            return update_search_page(query, previous_page, entries_per_page)
-
-        search_button.click(
-            fn=update_search_page,
-            inputs=[search_query_input, page_number, entries_per_page],
-            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        next_page_button.click(
-            fn=go_to_next_search_page,
-            inputs=[search_query_input, page_number, entries_per_page],
-            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
-        )
-
-        previous_page_button.click(
-            fn=go_to_previous_search_page,
-            inputs=[search_query_input, page_number, entries_per_page],
-            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
-        )
+# Search_Tab.py
+# Description: This file contains the code for the search tab in the Gradio UI
+#
+# Imports
+import html
+import logging
+import sqlite3
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import view_database, search_and_display_items
+from App_Function_Libraries.DB.SQLite_DB import search_prompts
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_detailed_view
+from App_Function_Libraries.Utils.Utils import get_database_path
+#
+###################################################################################################
+#
+# Functions:
+
+logger = logging.getLogger()
+
+
+def create_search_tab():
+    with gr.TabItem("Search / Detailed View"):
+        with gr.Row():
+            with gr.Column():
+                gr.Markdown("# Search across all ingested items in the Database")
+                gr.Markdown(" by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
+                search_button = gr.Button("Search")
+                items_output = gr.Dropdown(label="Select Item", choices=[])
+                item_mapping = gr.State({})
+                prompt_summary_output = gr.HTML(label="Prompt & Summary", visible=True)
+
+                search_button.click(
+                    fn=update_dropdown,
+                    inputs=[search_query_input, search_type_input],
+                    outputs=[items_output, item_mapping]
+                )
+            with gr.Column():
+                content_output = gr.Markdown(label="Content", visible=True)
+                items_output.change(
+                    fn=update_detailed_view,
+                    inputs=[items_output, item_mapping],
+                    outputs=[prompt_summary_output, content_output]
+                )
+
+
+def display_search_results(query):
+    if not query.strip():
+        return "Please enter a search query."
+
+    results = search_prompts(query)
+
+    # Debugging: Print the results to the console to see what is being returned
+    print(f"Processed search results for query '{query}': {results}")
+
+    if results:
+        result_md = "## Search Results:\n"
+        for result in results:
+            # Debugging: Print each result to see its format
+            print(f"Result item: {result}")
+
+            if len(result) == 2:
+                name, details = result
+                result_md += f"**Title:** {name}\n\n**Description:** {details}\n\n---\n"
+
+            elif len(result) == 4:
+                name, details, system, user = result
+                result_md += f"**Title:** {name}\n\n"
+                result_md += f"**Description:** {details}\n\n"
+                result_md += f"**System Prompt:** {system}\n\n"
+                result_md += f"**User Prompt:** {user}\n\n"
+                result_md += "---\n"
+            else:
+                result_md += "Error: Unexpected result format.\n\n---\n"
+        return result_md
+    return "No results found."
+
+
+def create_viewing_tab():
+    with gr.TabItem("View Database"):
+        gr.Markdown("# View Database Entries")
+        with gr.Row():
+            with gr.Column():
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+            with gr.Column():
+                results_display = gr.HTML()
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+
+        def update_page(page, entries_per_page):
+            results, pagination, total_pages = view_database(page, entries_per_page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(interactive=not prev_disabled)
+
+        def go_to_next_page(current_page, entries_per_page):
+            next_page = current_page + 1
+            return update_page(next_page, entries_per_page)
+
+        def go_to_previous_page(current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return update_page(previous_page, entries_per_page)
+
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+def create_search_summaries_tab():
+    with gr.TabItem("Search/View Title+Summary "):
+        gr.Markdown("# Search across all ingested items in the Database and review their summaries")
+        gr.Markdown("Search by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
+                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
+                                             label="Search By")
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                char_count_input = gr.Number(value=5000, label="Amount of characters to display from the main content",
+                                             precision=0)
+            with gr.Column():
+                search_button = gr.Button("Search")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+        search_results_output = gr.HTML()
+
+
+        def update_search_page(query, search_type, page, entries_per_page, char_count):
+            # Ensure char_count is a positive integer
+            char_count = max(1, int(char_count)) if char_count else 5000
+            results, pagination, total_pages = search_and_display_items(query, search_type, page, entries_per_page, char_count)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
+                interactive=not prev_disabled)
+
+        def go_to_next_search_page(query, search_type, current_page, entries_per_page, char_count):
+            next_page = current_page + 1
+            return update_search_page(query, search_type, next_page, entries_per_page, char_count)
+
+        def go_to_previous_search_page(query, search_type, current_page, entries_per_page, char_count):
+            previous_page = max(1, current_page - 1)
+            return update_search_page(query, search_type, previous_page, entries_per_page, char_count)
+
+        search_button.click(
+            fn=update_search_page,
+            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_search_page,
+            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_search_page,
+            inputs=[search_query_input, search_type_input, page_number, entries_per_page, char_count_input],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+
+def create_prompt_view_tab():
+    with gr.TabItem("View Prompt Database"):
+        gr.Markdown("# View Prompt Database Entries")
+        with gr.Row():
+            with gr.Column():
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+            with gr.Column():
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+        results_display = gr.HTML()
+
+        # FIXME - SQL functions to be moved to DB_Manager
+        def view_database(page, entries_per_page):
+            offset = (page - 1) * entries_per_page
+            try:
+                with sqlite3.connect(get_database_path('prompts.db')) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute('''
+                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+                        FROM Prompts p
+                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                        GROUP BY p.id
+                        ORDER BY p.name
+                        LIMIT ? OFFSET ?
+                    ''', (entries_per_page, offset))
+                    prompts = cursor.fetchall()
+
+                    cursor.execute('SELECT COUNT(*) FROM Prompts')
+                    total_prompts = cursor.fetchone()[0]
+
+                results = ""
+                for prompt in prompts:
+                    # Escape HTML special characters and replace newlines with <br> tags
+                    title = html.escape(prompt[0]).replace('\n', '<br>')
+                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
+                    system_prompt = html.escape(prompt[2] or '')
+                    user_prompt = html.escape(prompt[3] or '')
+                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
+
+                    results += f"""
+                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
+                            <div><strong>Title:</strong> {title}</div>
+                            <div><strong>Details:</strong> {details}</div>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>User Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>System Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>Keywords:</strong> {keywords}
+                        </div>
+                    </div>
+                    """
+
+                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
+                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
+
+                return results, pagination, total_pages
+            except sqlite3.Error as e:
+                return f"<p>Error fetching prompts: {e}</p>", "Error", 0
+
+        def update_page(page, entries_per_page):
+            results, pagination, total_pages = view_database(page, entries_per_page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
+                interactive=not prev_disabled)
+
+        def go_to_next_page(current_page, entries_per_page):
+            next_page = current_page + 1
+            return update_page(next_page, entries_per_page)
+
+        def go_to_previous_page(current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return update_page(previous_page, entries_per_page)
+
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+
+def create_prompt_search_tab():
+    with gr.TabItem("Search Prompts"):
+        gr.Markdown("# Search and View Prompt Details")
+        gr.Markdown("Currently has all of the https://github.com/danielmiessler/fabric prompts already available")
+        with gr.Row():
+            with gr.Column():
+                search_query_input = gr.Textbox(label="Search Prompts", placeholder="Enter your search query...")
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+            with gr.Column():
+                search_button = gr.Button("Search Prompts")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+        search_results_output = gr.HTML()
+
+        # This is dirty and shouldn't be in the UI code, but it's a quick way to get the search working.
+        # FIXME - SQL functions to be moved to DB_Manager
+        def search_and_display_prompts(query, page, entries_per_page):
+            offset = (page - 1) * entries_per_page
+            try:
+                # FIXME - SQL functions to be moved to DB_Manager
+                with sqlite3.connect(get_database_path('prompts.db')) as conn:
+                    cursor = conn.cursor()
+                    cursor.execute('''
+                        SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+                        FROM Prompts p
+                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                        WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
+                        GROUP BY p.id
+                        ORDER BY p.name
+                        LIMIT ? OFFSET ?
+                    ''', (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', entries_per_page, offset))
+                    prompts = cursor.fetchall()
+
+                    cursor.execute('''
+                        SELECT COUNT(DISTINCT p.id)
+                        FROM Prompts p
+                        LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                        LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                        WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
+                    ''', (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%'))
+                    total_prompts = cursor.fetchone()[0]
+
+                results = ""
+                for prompt in prompts:
+                    title = html.escape(prompt[0]).replace('\n', '<br>')
+                    details = html.escape(prompt[1] or '').replace('\n', '<br>')
+                    system_prompt = html.escape(prompt[2] or '')
+                    user_prompt = html.escape(prompt[3] or '')
+                    keywords = html.escape(prompt[4] or '').replace('\n', '<br>')
+
+                    results += f"""
+                    <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+                        <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;">
+                            <div><strong>Title:</strong> {title}</div>
+                            <div><strong>Details:</strong> {details}</div>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>User Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{user_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>System Prompt:</strong>
+                            <pre style="white-space: pre-wrap; word-wrap: break-word;">{system_prompt}</pre>
+                        </div>
+                        <div style="margin-top: 10px;">
+                            <strong>Keywords:</strong> {keywords}
+                        </div>
+                    </div>
+                    """
+
+                total_pages = (total_prompts + entries_per_page - 1) // entries_per_page
+                pagination = f"Page {page} of {total_pages} (Total prompts: {total_prompts})"
+
+                return results, pagination, total_pages
+            except sqlite3.Error as e:
+                return f"<p>Error searching prompts: {e}</p>", "Error", 0
+
+        def update_search_page(query, page, entries_per_page):
+            results, pagination, total_pages = search_and_display_prompts(query, page, entries_per_page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(interactive=not prev_disabled)
+
+        def go_to_next_search_page(query, current_page, entries_per_page):
+            next_page = current_page + 1
+            return update_search_page(query, next_page, entries_per_page)
+
+        def go_to_previous_search_page(query, current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return update_search_page(query, previous_page, entries_per_page)
+
+        search_button.click(
+            fn=update_search_page,
+            inputs=[search_query_input, page_number, entries_per_page],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_search_page,
+            inputs=[search_query_input, page_number, entries_per_page],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_search_page,
+            inputs=[search_query_input, page_number, entries_per_page],
+            outputs=[search_results_output, pagination_info, page_number, next_page_button, previous_page_button]
+        )

App_Function_Libraries/Gradio_UI/Trash.py

@@ -1,134 +1,139 @@
 # Trash.py
-# Gradio UI for deleting items from the database
-import html
-import sqlite3
-
+# Gradio UI for managing trashed items in the database
+#
 # Imports
+from typing import Tuple, List
 
-# External Imports
 import gradio as gr
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import delete_prompt, empty_trash, get_trashed_items, user_delete_item
+from App_Function_Libraries.DB.DB_Manager import (
+    get_trashed_items, user_delete_item, empty_trash,
+    get_transcripts, fetch_item_details,
+    search_media_database, mark_as_trash,
+)
 
 
-def delete_item(media_id, force):
-    return user_delete_item(media_id, force)
+#
+############################################################################################################
+#
+# Functions:
+
 
 def list_trash():
     items = get_trashed_items()
     return "\n".join(
         [f"ID: {item['id']}, Title: {item['title']}, Trashed on: {item['trash_date']}" for item in items])
 
+
+def delete_item(media_id, force):
+    return user_delete_item(media_id, force)
+
+
 def empty_trash_ui(days):
     deleted, remaining = empty_trash(days)
     return f"Deleted {deleted} items. {remaining} items remain in trash."
 
-def create_view_trash_tab():
-    with gr.TabItem("View Trash"):
-        view_button = gr.Button("View Trash")
-        trash_list = gr.Textbox(label="Trashed Items")
-        view_button.click(list_trash, inputs=[], outputs=trash_list)
+
+def get_media_transcripts(media_id):
+    transcripts = get_transcripts(media_id)
+    return "\n\n".join([f"Transcript ID: {t[0]}\nModel: {t[1]}\nCreated: {t[3]}\n{t[2][:200]}..." for t in transcripts])
 
 
+def get_media_summaries(media_id):
+    _, summary, _ = fetch_item_details(media_id)
+    return summary if summary else "No summary available."
 
 
-def search_prompts_for_deletion(query):
+def get_media_prompts(media_id):
+    prompt, _, _ = fetch_item_details(media_id)
+    return prompt if prompt else "No prompt available."
+
+
+def search_and_mark_trash(search_query: str) -> Tuple[List[Tuple[int, str, str]], str]:
     try:
-        with sqlite3.connect('prompts.db') as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT id, name, details
-                FROM Prompts
-                WHERE name LIKE ? OR details LIKE ?
-                LIMIT 10
-            ''', (f'%{query}%', f'%{query}%'))
-            results = cursor.fetchall()
-
-            if not results:
-                return "No matching prompts found."
-
-            output = "<h3>Matching Prompts:</h3>"
-            for row in results:
-                output += f"<p><strong>ID:</strong> {row[0]} | <strong>Name:</strong> {html.escape(row[1])} | <strong>Details:</strong> {html.escape(row[2][:100])}...</p>"
-            return output
-    except sqlite3.Error as e:
-        return f"An error occurred while searching prompts: {e}"
-
-
-def search_media_for_deletion(query):
+        results = search_media_database(search_query)
+        if not results:
+            return [], "No items found matching the search query."
+        return results, "Search completed successfully."
+    except Exception as e:
+        return [], f"Error during search: {str(e)}"
+
+
+def mark_item_as_trash(media_id: int) -> str:
     try:
-        with sqlite3.connect('media.db') as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT id, title, description
-                FROM media
-                WHERE title LIKE ? OR description LIKE ?
-                LIMIT 10
-            ''', (f'%{query}%', f'%{query}%'))
-            results = cursor.fetchall()
-
-            if not results:
-                return "No matching media found."
-
-            output = "<h3>Matching Media:</h3>"
-            for row in results:
-                output += f"<p><strong>ID:</strong> {row[0]} | <strong>Title:</strong> {html.escape(row[1])} | <strong>Description:</strong> {html.escape(row[2][:100])}...</p>"
-            return output
-    except sqlite3.Error as e:
-        return f"An error occurred while searching media: {e}"
+        mark_as_trash(media_id)
+        return f"Item with ID {media_id} has been marked as trash."
+    except Exception as e:
+        return f"Error marking item as trash: {str(e)}"
 
-def create_delete_trash_tab():
-    with gr.TabItem("Delete DB Item"):
-        gr.Markdown("# Search and Delete Items from Databases")
-
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("## Search and Delete Prompts")
-                prompt_search_input = gr.Textbox(label="Search Prompts")
-                prompt_search_button = gr.Button("Search Prompts")
-                prompt_search_results = gr.HTML()
-                prompt_id_input = gr.Number(label="Prompt ID")
-                prompt_delete_button = gr.Button("Delete Prompt")
-                prompt_delete_output = gr.Textbox(label="Delete Result")
-
-            with gr.Column():
-                gr.Markdown("## Search and Delete Media")
-                media_search_input = gr.Textbox(label="Search Media")
-                media_search_button = gr.Button("Search Media")
-                media_search_results = gr.HTML()
-                media_id_input = gr.Number(label="Media ID")
-                media_force_checkbox = gr.Checkbox(label="Force Delete")
-                media_delete_button = gr.Button("Delete Media")
-                media_delete_output = gr.Textbox(label="Delete Result")
-
-        prompt_search_button.click(
-            search_prompts_for_deletion,
-            inputs=[prompt_search_input],
-            outputs=prompt_search_results
-        )
 
-        prompt_delete_button.click(
-            delete_prompt,
-            inputs=[prompt_id_input],
-            outputs=prompt_delete_output
+def create_search_and_mark_trash_tab():
+    with gr.TabItem("Search and Mark as Trash"):
+        gr.Markdown("# Search for Items and Mark as Trash")
+
+        search_input = gr.Textbox(label="Search Query")
+        search_button = gr.Button("Search")
+        search_results = gr.Dropdown(label="Search Results", choices=[], interactive=True)
+        search_status = gr.Textbox(label="Search Status")
+
+        mark_trash_button = gr.Button("Mark Selected Item as Trash")
+        mark_trash_status = gr.Textbox(label="Mark as Trash Status")
+
+        def update_search_results(query):
+            results, status = search_and_mark_trash(query)
+            choices = [f"{id}: {title} ({url})" for id, title, url in results]
+            return choices, status
+
+        search_button.click(
+            update_search_results,
+            inputs=[search_input],
+            outputs=[search_results, search_status]
         )
 
-        media_search_button.click(
-            search_media_for_deletion,
-            inputs=[media_search_input],
-            outputs=media_search_results
+        def mark_selected_as_trash(selected_item):
+            if selected_item:
+                media_id = int(selected_item.split(":")[0])
+                return mark_item_as_trash(media_id)
+            return "No item selected."
+
+        mark_trash_button.click(
+            mark_selected_as_trash,
+            inputs=[search_results],
+            outputs=[mark_trash_status]
         )
 
+
+def create_view_trash_tab():
+    with gr.TabItem("View Trash"):
+        view_button = gr.Button("View Trash")
+        trash_list = gr.Textbox(label="Trashed Items")
+        view_button.click(list_trash, inputs=[], outputs=trash_list)
+
+
+def create_delete_trash_tab():
+    with gr.TabItem("Delete DB Item"):
+        gr.Markdown("# Delete Items from Databases")
+
+        media_id_input = gr.Number(label="Media ID")
+        media_force_checkbox = gr.Checkbox(label="Force Delete")
+        media_delete_button = gr.Button("Delete Media")
+        media_delete_output = gr.Textbox(label="Delete Result")
+
         media_delete_button.click(
             delete_item,
             inputs=[media_id_input, media_force_checkbox],
             outputs=media_delete_output
         )
 
+
 def create_empty_trash_tab():
     with gr.TabItem("Empty Trash"):
         days_input = gr.Slider(minimum=15, maximum=90, step=5, label="Delete items older than (days)")
         empty_button = gr.Button("Empty Trash")
         empty_output = gr.Textbox(label="Result")
-        empty_button.click(empty_trash_ui, inputs=[days_input], outputs=empty_output)
+        empty_button.click(empty_trash_ui, inputs=[days_input], outputs=empty_output)
+
+#
+# End of File
+############################################################################################################

App_Function_Libraries/Gradio_UI/Video_transcription_tab.py

@@ -9,8 +9,6 @@ import os
 # External Imports
 import gradio as gr
 import yt_dlp
-
-from App_Function_Libraries.Confabulation_check import simplified_geval
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import load_preset_prompts, add_media_to_database
@@ -21,8 +19,9 @@ from App_Function_Libraries.Summarization_General_Lib import perform_transcripti
 from App_Function_Libraries.Utils.Utils import convert_to_seconds, safe_read_file, format_transcription, \
     create_download_directory, generate_unique_identifier, extract_text_from_segments
 from App_Function_Libraries.Video_DL_Ingestion_Lib import parse_and_expand_urls, extract_metadata, download_video
+from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import run_geval
 #
-################################################################################################################################################################
+#######################################################################################################################
 #
 # Functions:
 
@@ -108,7 +107,7 @@ def create_video_transcription_tab():
                 api_name_input = gr.Dropdown(
                     choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
                              "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace", "Custom-OpenAI-API"],
                     value=None, label="API Name (Mandatory)")
                 api_key_input = gr.Textbox(label="API Key (Mandatory)", placeholder="Enter your API key here",
                                            type="password")
@@ -467,7 +466,7 @@ def create_video_transcription_tab():
                     if confab_checkbox:
                         logging.info("Confabulation check enabled")
                         # Assuming result[1] contains the transcript and result[2] contains the summary
-                        confabulation_result = simplified_geval(result[1], result[2], api_name, api_key)
+                        confabulation_result = run_geval(result[1], result[2], api_key, api_name)
                         logging.info(f"Simplified G-Eval result: {confabulation_result}")
 
                     # Ensure that result is a tuple with 5 elements

App_Function_Libraries/Gradio_UI/View_tab.py

@@ -0,0 +1,170 @@
+# View_tab.py
+# Description: Gradio functions for the view tab
+#
+# Imports
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import (
+    search_media_database, mark_as_trash, get_specific_prompt, delete_specific_transcript,
+    delete_specific_summary, delete_specific_prompt, get_specific_transcript, get_specific_summary,
+    get_media_transcripts, get_media_summaries, get_media_prompts
+)
+#
+############################################################################################################
+#
+# Functions:
+
+# FIXME - add mark_as_trash ability to the UI
+
+
+# FIXME - Doesn't work. also need ot merge this tab wtih Edit Existing Items tab....
+def create_manage_items_tab():
+    with gr.TabItem("Edit/Manage DB Items"):
+        search_input = gr.Textbox(label="Search for Media (title or ID)")
+        search_button = gr.Button("Search")
+        media_selector = gr.Dropdown(label="Select Media", choices=[], interactive=True)
+
+        with gr.Accordion("Transcripts"):
+            get_transcripts_button = gr.Button("Get Transcripts")
+            transcript_selector = gr.Dropdown(label="Select Transcript", choices=[], interactive=True)
+            transcripts_output = gr.Textbox(label="Transcript Content", lines=10)
+            delete_transcript_button = gr.Button("Delete Selected Transcript")
+
+        with gr.Accordion("Summaries"):
+            get_summaries_button = gr.Button("Get Summaries")
+            summary_selector = gr.Dropdown(label="Select Summary", choices=[], interactive=True)
+            summaries_output = gr.Textbox(label="Summary Content", lines=5)
+            delete_summary_button = gr.Button("Delete Selected Summary")
+
+        with gr.Accordion("Prompts"):
+            get_prompts_button = gr.Button("Get Prompts")
+            prompt_selector = gr.Dropdown(label="Select Prompt", choices=[], interactive=True)
+            prompts_output = gr.Textbox(label="Prompt Content", lines=5)
+            delete_prompt_button = gr.Button("Delete Selected Prompt")
+
+        status_output = gr.Textbox(label="Status")
+
+        def search_media(query):
+            results = search_media_database(query)
+            choices = [f"{result[0]}: {result[1]}" for result in results]
+            return {"choices": choices, "value": None}
+
+        search_button.click(search_media, inputs=[search_input], outputs=[media_selector])
+
+        def get_transcripts(media_selection):
+            if not media_selection:
+                return {"choices": [], "value": None}
+            media_id = int(media_selection.split(":")[0])
+            transcripts = get_media_transcripts(media_id)
+            choices = [f"{t[0]}: {t[3]}" for t in transcripts]
+            return {"choices": choices, "value": None}
+
+        def display_transcript(transcript_selection):
+            if not transcript_selection:
+                return "No transcript selected."
+            transcript_id = int(transcript_selection.split(":")[0])
+            transcript = get_specific_transcript(transcript_id)
+            return transcript['content'] if 'content' in transcript else transcript.get('error', "Transcript not found.")
+
+        get_transcripts_button.click(
+            get_transcripts,
+            inputs=[media_selector],
+            outputs=[transcript_selector]
+        )
+        transcript_selector.change(
+            display_transcript,
+            inputs=[transcript_selector],
+            outputs=[transcripts_output]
+        )
+
+        def get_summaries(media_selection):
+            if not media_selection:
+                return {"choices": [], "value": None}
+            media_id = int(media_selection.split(":")[0])
+            summaries = get_media_summaries(media_id)
+            choices = [f"{s[0]}: {s[3]}" for s in summaries]
+            return {"choices": choices, "value": None}
+
+        def display_summary(summary_selection):
+            if not summary_selection:
+                return "No summary selected."
+            summary_id = int(summary_selection.split(":")[0])
+            summary = get_specific_summary(summary_id)
+            return summary['content'] if 'content' in summary else summary.get('error', "Summary not found.")
+
+        get_summaries_button.click(
+            get_summaries,
+            inputs=[media_selector],
+            outputs=[summary_selector]
+        )
+        summary_selector.change(
+            display_summary,
+            inputs=[summary_selector],
+            outputs=[summaries_output]
+        )
+
+        def get_prompts(media_selection):
+            if not media_selection:
+                return {"choices": [], "value": None}
+            media_id = int(media_selection.split(":")[0])
+            prompts = get_media_prompts(media_id)
+            choices = [f"{p[0]}: {p[3]}" for p in prompts]
+            return {"choices": choices, "value": None}
+
+        def display_prompt(prompt_selection):
+            if not prompt_selection:
+                return "No prompt selected."
+            prompt_id = int(prompt_selection.split(":")[0])
+            prompt = get_specific_prompt(prompt_id)
+            return prompt['content'] if 'content' in prompt else prompt.get('error', "Prompt not found.")
+
+        get_prompts_button.click(
+            get_prompts,
+            inputs=[media_selector],
+            outputs=[prompt_selector]
+        )
+        prompt_selector.change(
+            display_prompt,
+            inputs=[prompt_selector],
+            outputs=[prompts_output]
+        )
+
+        def delete_transcript(transcript_selection):
+            if not transcript_selection:
+                return "No transcript selected."
+            transcript_id = int(transcript_selection.split(":")[0])
+            result = delete_specific_transcript(transcript_id)
+            return result
+
+        def delete_summary(summary_selection):
+            if not summary_selection:
+                return "No summary selected."
+            summary_id = int(summary_selection.split(":")[0])
+            result = delete_specific_summary(summary_id)
+            return result
+
+        def delete_prompt(prompt_selection):
+            if not prompt_selection:
+                return "No prompt selected."
+            prompt_id = int(prompt_selection.split(":")[0])
+            result = delete_specific_prompt(prompt_id)
+            return result
+
+        delete_transcript_button.click(
+            delete_transcript,
+            inputs=[transcript_selector],
+            outputs=[status_output]
+        )
+        delete_summary_button.click(
+            delete_summary,
+            inputs=[summary_selector],
+            outputs=[status_output]
+        )
+        delete_prompt_button.click(
+            delete_prompt,
+            inputs=[prompt_selector],
+            outputs=[status_output]
+        )

App_Function_Libraries/Gradio_UI/Website_scraping_tab.py

@@ -1,113 +1,540 @@
 # Website_scraping_tab.py
 # Gradio UI for scraping websites
-
+#
 # Imports
+import asyncio
+import json
+import logging
+import os
+import random
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, List, Dict, Any
+from urllib.parse import urlparse, urljoin
+
 #
 # External Imports
 import gradio as gr
+from playwright.async_api import TimeoutError, async_playwright
+from playwright.sync_api import sync_playwright
 
+#
+# Local Imports
+from App_Function_Libraries.Article_Extractor_Lib import scrape_from_sitemap, scrape_by_url_level, scrape_article
 from App_Function_Libraries.Article_Summarization_Lib import scrape_and_summarize_multiple
 from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
 from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+from App_Function_Libraries.Summarization_General_Lib import summarize
 
 
-#
-# Local Imports
-#
 #
 ########################################################################################################################
 #
 # Functions:
 
+def get_url_depth(url: str) -> int:
+    return len(urlparse(url).path.strip('/').split('/'))
+
+
+def sync_recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay=1.0):
+    def run_async_scrape():
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(
+            recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay)
+        )
+
+    with ThreadPoolExecutor() as executor:
+        future = executor.submit(run_async_scrape)
+        return future.result()
+
+
+async def recursive_scrape(
+        base_url: str,
+        max_pages: int,
+        max_depth: int,
+        progress_callback: callable,
+        delay: float = 1.0,
+        resume_file: str = 'scrape_progress.json',
+        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+) -> List[Dict]:
+    async def save_progress():
+        temp_file = resume_file + ".tmp"
+        with open(temp_file, 'w') as f:
+            json.dump({
+                'visited': list(visited),
+                'to_visit': to_visit,
+                'scraped_articles': scraped_articles,
+                'pages_scraped': pages_scraped
+            }, f)
+        os.replace(temp_file, resume_file)  # Atomic replace
+
+    def is_valid_url(url: str) -> bool:
+        return url.startswith("http") and len(url) > 0
+
+    # Load progress if resume file exists
+    if os.path.exists(resume_file):
+        with open(resume_file, 'r') as f:
+            progress_data = json.load(f)
+            visited = set(progress_data['visited'])
+            to_visit = progress_data['to_visit']
+            scraped_articles = progress_data['scraped_articles']
+            pages_scraped = progress_data['pages_scraped']
+    else:
+        visited = set()
+        to_visit = [(base_url, 0)]  # (url, depth)
+        scraped_articles = []
+        pages_scraped = 0
+
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=user_agent)
+
+            try:
+                while to_visit and pages_scraped < max_pages:
+                    current_url, current_depth = to_visit.pop(0)
+
+                    if current_url in visited or current_depth > max_depth:
+                        continue
+
+                    visited.add(current_url)
+
+                    # Update progress
+                    progress_callback(f"Scraping page {pages_scraped + 1}/{max_pages}: {current_url}")
+
+                    try:
+                        await asyncio.sleep(random.uniform(delay * 0.8, delay * 1.2))
+
+                        # This function should be implemented to handle asynchronous scraping
+                        article_data = await scrape_article_async(context, current_url)
+
+                        if article_data and article_data['extraction_successful']:
+                            scraped_articles.append(article_data)
+                            pages_scraped += 1
+
+                        # If we haven't reached max depth, add child links to to_visit
+                        if current_depth < max_depth:
+                            page = await context.new_page()
+                            await page.goto(current_url)
+                            await page.wait_for_load_state("networkidle")
+
+                            links = await page.eval_on_selector_all('a[href]',
+                                                                    "(elements) => elements.map(el => el.href)")
+                            for link in links:
+                                child_url = urljoin(base_url, link)
+                                if is_valid_url(child_url) and child_url.startswith(
+                                        base_url) and child_url not in visited and should_scrape_url(child_url):
+                                    to_visit.append((child_url, current_depth + 1))
+
+                            await page.close()
+
+                    except Exception as e:
+                        logging.error(f"Error scraping {current_url}: {str(e)}")
+
+                    # Save progress periodically (e.g., every 10 pages)
+                    if pages_scraped % 10 == 0:
+                        await save_progress()
+
+            finally:
+                await browser.close()
+
+    finally:
+        # These statements are now guaranteed to be reached after the scraping is done
+        await save_progress()
+
+        # Remove the progress file when scraping is completed successfully
+        if os.path.exists(resume_file):
+            os.remove(resume_file)
+
+        # Final progress update
+        progress_callback(f"Scraping completed. Total pages scraped: {pages_scraped}")
+
+        return scraped_articles
+
+
+async def scrape_article_async(context, url: str) -> Dict[str, Any]:
+    page = await context.new_page()
+    try:
+        await page.goto(url)
+        await page.wait_for_load_state("networkidle")
+
+        title = await page.title()
+        content = await page.content()
+
+        return {
+            'url': url,
+            'title': title,
+            'content': content,
+            'extraction_successful': True
+        }
+    except Exception as e:
+        logging.error(f"Error scraping article {url}: {str(e)}")
+        return {
+            'url': url,
+            'extraction_successful': False,
+            'error': str(e)
+        }
+    finally:
+        await page.close()
+
+
+def scrape_article_sync(url: str) -> Dict[str, Any]:
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        try:
+            page.goto(url)
+            page.wait_for_load_state("networkidle")
+
+            title = page.title()
+            content = page.content()
+
+            return {
+                'url': url,
+                'title': title,
+                'content': content,
+                'extraction_successful': True
+            }
+        except Exception as e:
+            logging.error(f"Error scraping article {url}: {str(e)}")
+            return {
+                'url': url,
+                'extraction_successful': False,
+                'error': str(e)
+            }
+        finally:
+            browser.close()
+
+
+def should_scrape_url(url: str) -> bool:
+    parsed_url = urlparse(url)
+    path = parsed_url.path.lower()
+
+    # List of patterns to exclude
+    exclude_patterns = [
+        '/tag/', '/category/', '/author/', '/search/', '/page/',
+        'wp-content', 'wp-includes', 'wp-json', 'wp-admin',
+        'login', 'register', 'cart', 'checkout', 'account',
+        '.jpg', '.png', '.gif', '.pdf', '.zip'
+    ]
+
+    # Check if the URL contains any exclude patterns
+    if any(pattern in path for pattern in exclude_patterns):
+        return False
+
+    # Add more sophisticated checks here
+    # For example, you might want to only include URLs with certain patterns
+    include_patterns = ['/article/', '/post/', '/blog/']
+    if any(pattern in path for pattern in include_patterns):
+        return True
+
+    # By default, return True if no exclusion or inclusion rules matched
+    return True
+
+
+async def scrape_with_retry(url: str, max_retries: int = 3, retry_delay: float = 5.0):
+    for attempt in range(max_retries):
+        try:
+            return await scrape_article(url)
+        except TimeoutError:
+            if attempt < max_retries - 1:
+                logging.warning(f"Timeout error scraping {url}. Retrying in {retry_delay} seconds...")
+                await asyncio.sleep(retry_delay)
+            else:
+                logging.error(f"Failed to scrape {url} after {max_retries} attempts.")
+                return None
+        except Exception as e:
+            logging.error(f"Error scraping {url}: {str(e)}")
+            return None
+
 
 def create_website_scraping_tab():
     with gr.TabItem("Website Scraping"):
-        gr.Markdown("# Scrape Websites & Summarize Articles using a Headless Chrome Browser!")
+        gr.Markdown("# Scrape Websites & Summarize Articles")
         with gr.Row():
             with gr.Column():
-                url_input = gr.Textbox(label="Article URLs", placeholder="Enter article URLs here, one per line", lines=5)
-                custom_article_title_input = gr.Textbox(label="Custom Article Titles (Optional, one per line)",
-                                                        placeholder="Enter custom titles for the articles, one per line",
-                                                        lines=5)
+                scrape_method = gr.Radio(
+                    ["Individual URLs", "Sitemap", "URL Level", "Recursive Scraping"],
+                    label="Scraping Method",
+                    value="Individual URLs"
+                )
+                url_input = gr.Textbox(
+                    label="Article URLs or Base URL",
+                    placeholder="Enter article URLs here, one per line, or base URL for sitemap/URL level/recursive scraping",
+                    lines=5
+                )
+                url_level = gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    label="URL Level (for URL Level scraping)",
+                    value=2,
+                    visible=False
+                )
+                max_pages = gr.Slider(
+                    minimum=1,
+                    maximum=100,
+                    step=1,
+                    label="Maximum Pages to Scrape (for Recursive Scraping)",
+                    value=10,
+                    visible=False
+                )
+                max_depth = gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    label="Maximum Depth (for Recursive Scraping)",
+                    value=3,
+                    visible=False
+                )
+                custom_article_title_input = gr.Textbox(
+                    label="Custom Article Titles (Optional, one per line)",
+                    placeholder="Enter custom titles for the articles, one per line",
+                    lines=5
+                )
                 with gr.Row():
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
+                    summarize_checkbox = gr.Checkbox(label="Summarize Articles", value=False)
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt", value=False, visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt", value=False, visible=True)
                 with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
+                    temp_slider = gr.Slider(0.1, 2.0, 0.7, label="Temperature")
                 with gr.Row():
-                    website_custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=load_preset_prompts(),
+                        visible=False
+                    )
                 with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-""",
-                                                     lines=3,
-                                                     visible=False)
-
-                custom_prompt_checkbox.change(
-                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-                    inputs=[custom_prompt_checkbox],
-                    outputs=[website_custom_prompt_input, system_prompt_input]
-                )
-                preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
-                )
-
-                def update_prompts(preset_name):
-                    prompts = update_user_prompt(preset_name)
-                    return (
-                        gr.update(value=prompts["user_prompt"], visible=True),
-                        gr.update(value=prompts["system_prompt"], visible=True)
+                    website_custom_prompt_input = gr.Textbox(
+                        label="Custom Prompt",
+                        placeholder="Enter custom prompt here",
+                        lines=3,
+                        visible=False
+                    )
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(
+                        label="System Prompt",
+                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                                **Bulleted Note Creation Guidelines**
+                                
+                                **Headings**:
+                                - Based on referenced topics, not categories like quotes or terms
+                                - Surrounded by **bold** formatting 
+                                - Not listed as bullet points
+                                - No space between headings and list items underneath
+                                
+                                **Emphasis**:
+                                - **Important terms** set in bold font
+                                - **Text ending in a colon**: also bolded
+                                
+                                **Review**:
+                                - Ensure adherence to specified format
+                                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                                """,
+                        lines=3,
+                        visible=False
                     )
-
-                preset_prompt.change(
-                    update_prompts,
-                    inputs=preset_prompt,
-                    outputs=[website_custom_prompt_input, system_prompt_input]
-                )
 
                 api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"], value=None, label="API Name (Mandatory for Summarization)")
-                api_key_input = gr.Textbox(label="API Key (Mandatory if API Name is specified)",
-                                           placeholder="Enter your API key here; Ignore if using Local API or Built-in API", type="password")
-                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)",
-                                            value="default,no_keyword_set", visible=True)
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
+                             "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
+                             "Custom-OpenAI-API"],
+                    value=None,
+                    label="API Name (Mandatory for Summarization)"
+                )
+                api_key_input = gr.Textbox(
+                    label="API Key (Mandatory if API Name is specified)",
+                    placeholder="Enter your API key here; Ignore if using Local API or Built-in API",
+                    type="password"
+                )
+                keywords_input = gr.Textbox(
+                    label="Keywords",
+                    placeholder="Enter keywords here (comma-separated)",
+                    value="default,no_keyword_set",
+                    visible=True
+                )
 
                 scrape_button = gr.Button("Scrape and Summarize")
             with gr.Column():
+                progress_output = gr.Textbox(label="Progress", lines=3)
                 result_output = gr.Textbox(label="Result", lines=20)
 
-                scrape_button.click(
-                    fn=scrape_and_summarize_multiple,
-                    inputs=[url_input, website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
-                            custom_article_title_input, system_prompt_input],
-                    outputs=result_output
-                )
+        def update_ui_for_scrape_method(method):
+            url_level_update = gr.update(visible=(method == "URL Level"))
+            max_pages_update = gr.update(visible=(method == "Recursive Scraping"))
+            max_depth_update = gr.update(visible=(method == "Recursive Scraping"))
+            url_input_update = gr.update(
+                label="Article URLs" if method == "Individual URLs" else "Base URL",
+                placeholder="Enter article URLs here, one per line" if method == "Individual URLs" else "Enter the base URL for scraping"
+            )
+            return url_level_update, max_pages_update, max_depth_update, url_input_update
+
+        scrape_method.change(
+            fn=update_ui_for_scrape_method,
+            inputs=[scrape_method],
+            outputs=[url_level, max_pages, max_depth, url_input]
+        )
 
+        custom_prompt_checkbox.change(
+            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+            inputs=[custom_prompt_checkbox],
+            outputs=[website_custom_prompt_input, system_prompt_input]
+        )
+        preset_prompt_checkbox.change(
+            fn=lambda x: gr.update(visible=x),
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt]
+        )
 
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        preset_prompt.change(
+            update_prompts,
+            inputs=preset_prompt,
+            outputs=[website_custom_prompt_input, system_prompt_input]
+        )
+
+        async def scrape_and_summarize_wrapper(
+                scrape_method: str,
+                url_input: str,
+                url_level: Optional[int],
+                max_pages: int,
+                max_depth: int,
+                summarize_checkbox: bool,
+                custom_prompt: Optional[str],
+                api_name: Optional[str],
+                api_key: Optional[str],
+                keywords: str,
+                custom_titles: Optional[str],
+                system_prompt: Optional[str],
+                temperature: float = 0.7,
+                progress: gr.Progress = gr.Progress()
+        ) -> str:
+            try:
+                result: List[Dict[str, Any]] = []
+
+                if scrape_method == "Individual URLs":
+                    result = await scrape_and_summarize_multiple(url_input, custom_prompt, api_name, api_key, keywords,
+                                                                 custom_titles, system_prompt)
+                elif scrape_method == "Sitemap":
+                    result = scrape_from_sitemap(url_input)
+                elif scrape_method == "URL Level":
+                    if url_level is None:
+                        return convert_json_to_markdown(
+                            json.dumps({"error": "URL level is required for URL Level scraping."}))
+                    result = scrape_by_url_level(url_input, url_level)
+                elif scrape_method == "Recursive Scraping":
+                    result = await recursive_scrape(url_input, max_pages, max_depth, progress.update, delay=1.0)
+                else:
+                    return convert_json_to_markdown(json.dumps({"error": f"Unknown scraping method: {scrape_method}"}))
+
+                # Ensure result is always a list of dictionaries
+                if isinstance(result, dict):
+                    result = [result]
+                elif not isinstance(result, list):
+                    raise TypeError(f"Unexpected result type: {type(result)}")
+
+                if summarize_checkbox:
+                    total_articles = len(result)
+                    for i, article in enumerate(result):
+                        progress.update(f"Summarizing article {i + 1}/{total_articles}")
+                        summary = summarize(article['content'], custom_prompt, api_name, api_key, temperature,
+                                            system_prompt)
+                        article['summary'] = summary
+
+                # Concatenate all content
+                all_content = "\n\n".join(
+                    [f"# {article.get('title', 'Untitled')}\n\n{article.get('content', '')}\n\n" +
+                     (f"Summary: {article.get('summary', '')}" if summarize_checkbox else "")
+                     for article in result])
+
+                # Collect all unique URLs
+                all_urls = list(set(article.get('url', '') for article in result if article.get('url')))
+
+                # Structure the output for the entire website collection
+                website_collection = {
+                    "base_url": url_input,
+                    "scrape_method": scrape_method,
+                    "summarization_performed": summarize_checkbox,
+                    "api_used": api_name if summarize_checkbox else None,
+                    "keywords": keywords if summarize_checkbox else None,
+                    "url_level": url_level if scrape_method == "URL Level" else None,
+                    "max_pages": max_pages if scrape_method == "Recursive Scraping" else None,
+                    "max_depth": max_depth if scrape_method == "Recursive Scraping" else None,
+                    "total_articles_scraped": len(result),
+                    "urls_scraped": all_urls,
+                    "content": all_content
+                }
+
+                # Convert the JSON to markdown and return
+                return convert_json_to_markdown(json.dumps(website_collection, indent=2))
+            except Exception as e:
+                return convert_json_to_markdown(json.dumps({"error": f"An error occurred: {str(e)}"}))
+
+        # Update the scrape_button.click to include the temperature parameter
+        scrape_button.click(
+            fn=lambda *args: asyncio.run(scrape_and_summarize_wrapper(*args)),
+            inputs=[scrape_method, url_input, url_level, max_pages, max_depth, summarize_checkbox,
+                    website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
+                    custom_article_title_input, system_prompt_input, temp_slider],
+            outputs=[result_output]
+        )
+
+
+def convert_json_to_markdown(json_str: str) -> str:
+    """
+    Converts the JSON output from the scraping process into a markdown format.
+
+    Args:
+        json_str (str): JSON-formatted string containing the website collection data
+
+    Returns:
+        str: Markdown-formatted string of the website collection data
+    """
+    try:
+        # Parse the JSON string
+        data = json.loads(json_str)
+
+        # Check if there's an error in the JSON
+        if "error" in data:
+            return f"# Error\n\n{data['error']}"
+
+        # Start building the markdown string
+        markdown = f"# Website Collection: {data['base_url']}\n\n"
+
+        # Add metadata
+        markdown += "## Metadata\n\n"
+        markdown += f"- **Scrape Method:** {data['scrape_method']}\n"
+        markdown += f"- **API Used:** {data['api_used']}\n"
+        markdown += f"- **Keywords:** {data['keywords']}\n"
+        if data['url_level'] is not None:
+            markdown += f"- **URL Level:** {data['url_level']}\n"
+        markdown += f"- **Total Articles Scraped:** {data['total_articles_scraped']}\n\n"
+
+        # Add URLs scraped
+        markdown += "## URLs Scraped\n\n"
+        for url in data['urls_scraped']:
+            markdown += f"- {url}\n"
+        markdown += "\n"
+
+        # Add the content
+        markdown += "## Content\n\n"
+        markdown += data['content']
+
+        return markdown
+
+    except json.JSONDecodeError:
+        return "# Error\n\nInvalid JSON string provided."
+    except KeyError as e:
+        return f"# Error\n\nMissing key in JSON data: {str(e)}"
+    except Exception as e:
+        return f"# Error\n\nAn unexpected error occurred: {str(e)}"
+#
+# End of File
+########################################################################################################################