Updated packages, models, preparing for use with AWS (in background)

README.md

@@ -4,7 +4,7 @@ emoji: 🌍
 colorFrom: yellow
 colorTo: yellow
 sdk: gradio
-sdk_version: 3.50.2
+sdk_version: 5.8.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -1,9 +1,10 @@
 # Load in packages
 
 import os
+import socket
 
 from typing import Type
-from langchain_community.embeddings import HuggingFaceEmbeddings#, HuggingFaceInstructEmbeddings
+from langchain_huggingface.embeddings import HuggingFaceEmbeddings#, HuggingFaceInstructEmbeddings
 from langchain_community.vectorstores import FAISS
 import gradio as gr
 import pandas as pd
@@ -12,17 +13,37 @@ from transformers import AutoTokenizer
 import torch
 
 from llama_cpp import Llama
-from huggingface_hub import hf_hub_download  
+from huggingface_hub import hf_hub_download
+from chatfuncs.ingest import embed_faiss_save_to_zip
+from chatfuncs.helper_functions import get_or_create_env_var
+
+from chatfuncs.helper_functions import ensure_output_folder_exists, get_connection_params, output_folder, get_or_create_env_var, reveal_feedback_buttons, wipe_logs
+from chatfuncs.aws_functions import upload_file_to_s3
+#from chatfuncs.llm_api_call import llm_query
+from chatfuncs.auth import authenticate_user
 
 PandasDataFrame = Type[pd.DataFrame]
 
+from datetime import datetime
+today_rev = datetime.now().strftime("%Y%m%d")
+
+ensure_output_folder_exists()
+
+host_name = socket.gethostname()
+
+access_logs_data_folder = 'logs/' + today_rev + '/' + host_name + '/'
+feedback_data_folder = 'feedback/' + today_rev + '/' + host_name + '/'
+usage_data_folder = 'usage/' + today_rev + '/' + host_name + '/'
+
 # Disable cuda devices if necessary
 #os.environ['CUDA_VISIBLE_DEVICES'] = '-1' 
 
 #from chatfuncs.chatfuncs import *
 import chatfuncs.ingest as ing
 
-##  Load preset embeddings, vectorstore, and model
+###
+# Load preset embeddings, vectorstore, and model
+###
 
 embeddings_name = "BAAI/bge-base-en-v1.5"
 
@@ -68,7 +89,7 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
     if torch_device is None:
         torch_device = chatf.torch_device
 
-    if model_type == "Phi 3 Mini (larger, slow)":
+    if model_type == "Phi 3.5 Mini (larger, slow)":
         if torch_device == "cuda":
             gpu_config.update_gpu(gpu_layers)
             print("Loading with", gpu_config.n_gpu_layers, "model layers sent to GPU.")
@@ -84,8 +105,8 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
         try:
             model = Llama(
             model_path=hf_hub_download(
-            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF"),# "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), # "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
-            filename=os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")  #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf")#"mistral-7b-openorca.Q4_K_M.gguf"),
+            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3.5-mini-instruct-GGUF"),# "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), # "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
+            filename=os.environ.get("MODEL_FILE", "Phi-3.5-mini-instruct.Q4_K_M.gguf") #"Phi-3-mini-128k-instruct.Q4_K_M.gguf")  #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf")#"mistral-7b-openorca.Q4_K_M.gguf"),
         ),
         **vars(gpu_config) # change n_gpu_layers if you have more or less VRAM 
         )
@@ -95,17 +116,17 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
             print(e)
             model = Llama(
             model_path=hf_hub_download(
-            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #"QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #, "microsoft/Phi-3-mini-4k-instruct-gguf"),#"QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
-            filename=os.environ.get("MODEL_FILE", "Phi-3-mini-128k-instruct.Q4_K_M.gguf"), # "Phi-3-mini-128k-instruct.Q4_K_M.gguf") # , #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf"),#"mistral-7b-openorca.Q4_K_M.gguf"),
+            repo_id=os.environ.get("REPO_ID", "QuantFactory/Phi-3.5-mini-instruct-GGUF"), #"QuantFactory/Phi-3-mini-128k-instruct-GGUF"), #, "microsoft/Phi-3-mini-4k-instruct-gguf"),#"QuantFactory/Meta-Llama-3-8B-Instruct-GGUF-v2"), #"microsoft/Phi-3-mini-4k-instruct-gguf"),#"TheBloke/Mistral-7B-OpenOrca-GGUF"),
+            filename=os.environ.get("MODEL_FILE", "Phi-3.5-mini-instruct.Q4_K_M.gguf"), # "Phi-3-mini-128k-instruct.Q4_K_M.gguf") # , #"Meta-Llama-3-8B-Instruct-v2.Q6_K.gguf") #"Phi-3-mini-4k-instruct-q4.gguf"),#"mistral-7b-openorca.Q4_K_M.gguf"),
         ),
         **vars(cpu_config)
         )
 
         tokenizer = []
 
-    if model_type == "Flan Alpaca (small, fast)":
+    if model_type == "Qwen 2 0.5B (small, fast)":
         # Huggingface chat model
-        hf_checkpoint = 'declare-lab/flan-alpaca-large'#'declare-lab/flan-alpaca-base' # # #
+        hf_checkpoint = 'Qwen/Qwen2-0.5B-Instruct'# 'declare-lab/flan-alpaca-large'#'declare-lab/flan-alpaca-base' # # # 'Qwen/Qwen1.5-0.5B-Chat' #
         
         def create_hf_model(model_name):
 
@@ -113,14 +134,14 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
             
             if torch_device == "cuda":
                 if "flan" in model_name:
-                    model = AutoModelForSeq2SeqLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.float16)
+                    model = AutoModelForSeq2SeqLM.from_pretrained(model_name, device_map="auto")#, torch_dtype=torch.float16)
                 else:
-                    model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", torch_dtype=torch.float16)
+                    model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")#, torch_dtype=torch.float16)
             else:
                 if "flan" in model_name:
-                    model = AutoModelForSeq2SeqLM.from_pretrained(model_name, torch_dtype=torch.float16)
+                    model = AutoModelForSeq2SeqLM.from_pretrained(model_name)#, torch_dtype=torch.float16)
                 else: 
-                    model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype=torch.float16)
+                    model = AutoModelForCausalLM.from_pretrained(model_name)#, trust_remote_code=True)#, torch_dtype=torch.float16)
 
             tokenizer = AutoTokenizer.from_pretrained(model_name, model_max_length = chatf.context_length)
 
@@ -138,10 +159,10 @@ def load_model(model_type, gpu_layers, gpu_config=None, cpu_config=None, torch_d
     return model_type, load_confirmation, model_type
 
 # Both models are loaded on app initialisation so that users don't have to wait for the models to be downloaded
-model_type = "Phi 3 Mini (larger, slow)"
-load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
+#model_type = "Phi 3.5 Mini (larger, slow)"
+#load_model(model_type, chatf.gpu_layers, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
 
-model_type = "Flan Alpaca (small, fast)"
+model_type = "Qwen 2 0.5B (small, fast)"
 load_model(model_type, 0, chatf.gpu_config, chatf.cpu_config, chatf.torch_device)
 
 def docs_to_faiss_save(docs_out:PandasDataFrame, embeddings=embeddings):
@@ -152,25 +173,30 @@ def docs_to_faiss_save(docs_out:PandasDataFrame, embeddings=embeddings):
 
     vectorstore_func = FAISS.from_documents(documents=docs_out, embedding=embeddings)
 
-
     chatf.vectorstore = vectorstore_func
 
     out_message = "Document processing complete"
 
     return out_message, vectorstore_func
-
  # Gradio chat
 
-block = gr.Blocks(theme = gr.themes.Base())#css=".gradio-container {background-color: black}")
 
-with block:
+###
+# RUN UI
+###
+
+app = gr.Blocks(theme = gr.themes.Base())#css=".gradio-container {background-color: black}")
+
+with app:
     ingest_text = gr.State()
     ingest_metadata = gr.State()
     ingest_docs = gr.State()
 
     model_type_state = gr.State(model_type)
     embeddings_state = gr.State(chatf.embeddings)#globals()["embeddings"])
-    vectorstore_state = gr.State(chatf.vectorstore)#globals()["vectorstore"])  
+    vectorstore_state = gr.State(chatf.vectorstore)#globals()["vectorstore"]) 
+
+    relevant_query_state = gr.Checkbox(value=True, visible=False) 
 
     model_state = gr.State() # chatf.model (gives error)
     tokenizer_state = gr.State() # chatf.tokenizer (gives error)
@@ -178,9 +204,26 @@ with block:
     chat_history_state = gr.State()
     instruction_prompt_out = gr.State()
 
+    session_hash_state = gr.State()
+    s3_output_folder_state = gr.State()
+
+    session_hash_textbox = gr.Textbox(value="", visible=False)
+    s3_logs_output_textbox = gr.Textbox(label="S3 logs", visible=False)
+
+    access_logs_state = gr.State(access_logs_data_folder + 'dataset1.csv')
+    access_s3_logs_loc_state = gr.State(access_logs_data_folder)
+    usage_logs_state = gr.State(usage_data_folder + 'dataset1.csv')
+    usage_s3_logs_loc_state = gr.State(usage_data_folder)
+    feedback_logs_state = gr.State(feedback_data_folder + 'dataset1.csv')
+    feedback_s3_logs_loc_state = gr.State(feedback_data_folder)
+
     gr.Markdown("<h1><center>Lightweight PDF / web page QA bot</center></h1>")        
     
-    gr.Markdown("Chat with PDF, web page or (new) csv/Excel documents. The default is a small model (Flan Alpaca), that can only answer specific questions that are answered in the text. It cannot give overall impressions of, or summarise the document. The alternative (Phi 3 Mini (larger, slow)), can reason a little better, but is much slower (See Advanced tab).\n\nBy default the Lambeth Borough Plan '[Lambeth 2030 : Our Future, Our Lambeth](https://www.lambeth.gov.uk/better-fairer-lambeth/projects/lambeth-2030-our-future-our-lambeth)' is loaded. If you want to talk about another document or web page, please select from the second tab. If switching topic, please click the 'Clear chat' button.\n\nCaution: This is a public app. Please ensure that the document you upload is not sensitive is any way as other users may see it! Also, please note that LLM chatbots may give incomplete or incorrect information, so please use with care.")
+    gr.Markdown("Chat with PDF, web page or (new) csv/Excel documents. The default is a small model (Qwen 2 0.5B), that can only answer specific questions that are answered in the text. It cannot give overall impressions of, or summarise the document. The alternative (Phi 3.5 Mini (larger, slow)), can reason a little better, but is much slower (See Advanced tab).\n\nBy default the Lambeth Borough Plan '[Lambeth 2030 : Our Future, Our Lambeth](https://www.lambeth.gov.uk/better-fairer-lambeth/projects/lambeth-2030-our-future-our-lambeth)' is loaded. If you want to talk about another document or web page, please select from the second tab. If switching topic, please click the 'Clear chat' button.\n\nCaution: This is a public app. Please ensure that the document you upload is not sensitive is any way as other users may see it! Also, please note that LLM chatbots may give incomplete or incorrect information, so please use with care.")
+
+    with gr.Accordion(label="Use Gemini or AWS Claude model", open=False, visible=False):
+        api_model_choice = gr.Dropdown(value = "None", choices = ["gemini-1.5-flash-002", "gemini-1.5-pro-002", "anthropic.claude-3-haiku-20240307-v1:0", "anthropic.claude-3-sonnet-20240229-v1:0", "None"], label="LLM model to use", multiselect=False, interactive=True, visible=False)
+        in_api_key = gr.Textbox(value = "", label="Enter Gemini API key (only if using Google API models)", lines=1, type="password",interactive=True, visible=False)
 
     with gr.Row():
         current_source = gr.Textbox(label="Current data source(s)", value="Lambeth_2030-Our_Future_Our_Lambeth.pdf", scale = 10)
@@ -190,7 +233,7 @@ with block:
 
         with gr.Row():
             #chat_height = 500
-            chatbot = gr.Chatbot(avatar_images=('user.jfif', 'bot.jpg'),bubble_full_width = False, scale = 1) # , height=chat_height
+            chatbot = gr.Chatbot(avatar_images=('user.jfif', 'bot.jpg'),bubble_full_width = False, scale = 1, type='tuples') # , height=chat_height
             with gr.Accordion("Open this tab to see the source paragraphs used to generate the answer", open = False):
                 sources = gr.HTML(value = "Source paragraphs with the most relevant text will appear here") # , height=chat_height
 
@@ -210,7 +253,6 @@ with block:
                 "What is the vision statement for Lambeth?",
                 "What are the commitments for Lambeth?",
                 "What are the 2030 outcomes for Lambeth?"])
-
         
         current_topic = gr.Textbox(label="Feature currently disabled - Keywords related to current conversation topic.", placeholder="Keywords related to the conversation topic will appear here")      
 
@@ -230,14 +272,16 @@ with block:
             in_csv = gr.File(label="Upload CSV/Excel file", file_count="multiple", file_types=['.csv', '.xlsx'])
             in_text_column = gr.Textbox(label="Enter column name where text is stored")
             load_csv = gr.Button(value="Load in CSV/Excel file", variant="secondary", scale=0)
-        
-        ingest_embed_out = gr.Textbox(label="File/web page preparation progress")
+
+        with gr.Row():
+            ingest_embed_out = gr.Textbox(label="File/web page preparation progress")
+            file_out_box = gr.File(file_count='single', file_types=['.zip'])
 
     with gr.Tab("Advanced features"):
         out_passages = gr.Slider(minimum=1, value = 2, maximum=10, step=1, label="Choose number of passages to retrieve from the document. Numbers greater than 2 may lead to increased hallucinations or input text being truncated.")
         temp_slide = gr.Slider(minimum=0.1, value = 0.5, maximum=1, step=0.1, label="Choose temperature setting for response generation.")
         with gr.Row():
-            model_choice = gr.Radio(label="Choose a chat model", value="Flan Alpaca (small, fast)", choices = ["Flan Alpaca (small, fast)", "Phi 3 Mini (larger, slow)"])
+            model_choice = gr.Radio(label="Choose a chat model", value="Qwen 2 0.5B (small, fast)", choices = ["Qwen 2 0.5B (small, fast)", "Phi 3.5 Mini (larger, slow)"])
             change_model_button = gr.Button(value="Load model", scale=0)
         with gr.Accordion("Choose number of model layers to send to GPU (WARNING: please don't modify unless you are sure you have a GPU).", open = False):
             gpu_layer_choice = gr.Slider(label="Choose number of model layers to send to GPU.", value=0, minimum=0, maximum=100, step = 1, visible=True)
@@ -246,7 +290,7 @@ with block:
         
 
     gr.HTML(
-        "<center>This app is based on the models Flan Alpaca and Phi 3 Mini. It powered by Gradio, Transformers, and Llama.cpp.</a></center>"
+        "<center>This app is based on the models Qwen 2 0.5B and Phi 3.5 Mini. It powered by Gradio, Transformers, and Llama.cpp.</a></center>"
     )
 
     examples_set.change(fn=chatf.update_message, inputs=[examples_set], outputs=[message])
@@ -260,34 +304,34 @@ with block:
     # Load in a pdf
     load_pdf_click = load_pdf.click(ing.parse_file, inputs=[in_pdf], outputs=[ingest_text, current_source]).\
              then(ing.text_to_docs, inputs=[ingest_text], outputs=[ingest_docs]).\
-             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=[ingest_embed_out, vectorstore_state]).\
+             then(embed_faiss_save_to_zip, inputs=[ingest_docs], outputs=[ingest_embed_out, vectorstore_state, file_out_box]).\
              then(chatf.hide_block, outputs = [examples_set])
 
     # Load in a webpage
     load_web_click = load_web.click(ing.parse_html, inputs=[in_web, in_div], outputs=[ingest_text, ingest_metadata, current_source]).\
              then(ing.html_text_to_docs, inputs=[ingest_text, ingest_metadata], outputs=[ingest_docs]).\
-             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=[ingest_embed_out, vectorstore_state]).\
+             then(embed_faiss_save_to_zip, inputs=[ingest_docs], outputs=[ingest_embed_out, vectorstore_state, file_out_box]).\
              then(chatf.hide_block, outputs = [examples_set])
     
     # Load in a csv/excel file
     load_csv_click = load_csv.click(ing.parse_csv_or_excel, inputs=[in_csv, in_text_column], outputs=[ingest_text, current_source]).\
              then(ing.csv_excel_text_to_docs, inputs=[ingest_text, in_text_column], outputs=[ingest_docs]).\
-             then(docs_to_faiss_save, inputs=[ingest_docs], outputs=[ingest_embed_out, vectorstore_state]).\
+             then(embed_faiss_save_to_zip, inputs=[ingest_docs], outputs=[ingest_embed_out, vectorstore_state, file_out_box]).\
              then(chatf.hide_block, outputs = [examples_set])
 
     # Load in a webpage
 
     # Click/enter to send message action
-    response_click = submit.click(chatf.create_full_prompt, inputs=[message, chat_history_state, current_topic, vectorstore_state, embeddings_state, model_type_state, out_passages], outputs=[chat_history_state, sources, instruction_prompt_out], queue=False, api_name="retrieval").\
+    response_click = submit.click(chatf.create_full_prompt, inputs=[message, chat_history_state, current_topic, vectorstore_state, embeddings_state, model_type_state, out_passages, api_model_choice, in_api_key], outputs=[chat_history_state, sources, instruction_prompt_out, relevant_query_state], queue=False, api_name="retrieval").\
                 then(chatf.turn_off_interactivity, inputs=[message, chatbot], outputs=[message, chatbot], queue=False).\
-                then(chatf.produce_streaming_answer_chatbot, inputs=[chatbot, instruction_prompt_out, model_type_state, temp_slide], outputs=chatbot)
+                then(chatf.produce_streaming_answer_chatbot, inputs=[chatbot, instruction_prompt_out, model_type_state, temp_slide, relevant_query_state], outputs=chatbot)
     response_click.then(chatf.highlight_found_text, [chatbot, sources], [sources]).\
                 then(chatf.add_inputs_answer_to_history,[message, chatbot, current_topic], [chat_history_state, current_topic]).\
                 then(lambda: chatf.restore_interactivity(), None, [message], queue=False)
 
-    response_enter = message.submit(chatf.create_full_prompt, inputs=[message, chat_history_state, current_topic, vectorstore_state, embeddings_state, model_type_state, out_passages], outputs=[chat_history_state, sources, instruction_prompt_out], queue=False).\
+    response_enter = message.submit(chatf.create_full_prompt, inputs=[message, chat_history_state, current_topic, vectorstore_state, embeddings_state, model_type_state, out_passages, api_model_choice, in_api_key], outputs=[chat_history_state, sources, instruction_prompt_out, relevant_query_state], queue=False).\
                 then(chatf.turn_off_interactivity, inputs=[message, chatbot], outputs=[message, chatbot], queue=False).\
-                then(chatf.produce_streaming_answer_chatbot, [chatbot, instruction_prompt_out, model_type_state, temp_slide], chatbot)    
+                then(chatf.produce_streaming_answer_chatbot, [chatbot, instruction_prompt_out, model_type_state, temp_slide, relevant_query_state], chatbot)    
     response_enter.then(chatf.highlight_found_text, [chatbot, sources], [sources]).\
                 then(chatf.add_inputs_answer_to_history,[message, chatbot, current_topic], [chat_history_state, current_topic]).\
                 then(lambda: chatf.restore_interactivity(), None, [message], queue=False)
@@ -302,4 +346,24 @@ with block:
     # Thumbs up or thumbs down voting function
     chatbot.like(chatf.vote, [chat_history_state, instruction_prompt_out, model_type_state], None)
 
-block.queue().launch(debug=True)
+    ###
+    # LOGGING AND ON APP LOAD FUNCTIONS
+    ###    
+    app.load(get_connection_params, inputs=None, outputs=[session_hash_state, s3_output_folder_state, session_hash_textbox])
+
+    # Log usernames and times of access to file (to know who is using the app when running on AWS)
+    access_callback = gr.CSVLogger()
+    access_callback.setup([session_hash_textbox], access_logs_data_folder)
+
+    session_hash_textbox.change(lambda *args: access_callback.flag(list(args)), [session_hash_textbox], None, preprocess=False).\
+    then(fn = upload_file_to_s3, inputs=[access_logs_state, access_s3_logs_loc_state], outputs=[s3_logs_output_textbox])
+
+# Launch the Gradio app
+COGNITO_AUTH = get_or_create_env_var('COGNITO_AUTH', '0')
+print(f'The value of COGNITO_AUTH is {COGNITO_AUTH}')
+
+if __name__ == "__main__":
+    if os.environ['COGNITO_AUTH'] == "1":
+        app.queue().launch(show_error=True, auth=authenticate_user, max_file_size='50mb')
+    else:
+        app.queue().launch(show_error=True, inbrowser=True, max_file_size='50mb')

chatfuncs/auth.py

@@ -0,0 +1,48 @@
+
+import boto3
+from chatfuncs.helper_functions import get_or_create_env_var
+
+client_id = get_or_create_env_var('AWS_CLIENT_ID', '') # This client id is borrowed from async gradio app client
+print(f'The value of AWS_CLIENT_ID is {client_id}')
+
+user_pool_id = get_or_create_env_var('AWS_USER_POOL_ID', '')
+print(f'The value of AWS_USER_POOL_ID is {user_pool_id}')
+
+def authenticate_user(username, password, user_pool_id=user_pool_id, client_id=client_id):
+    """Authenticates a user against an AWS Cognito user pool.
+
+    Args:
+        user_pool_id (str): The ID of the Cognito user pool.
+        client_id (str): The ID of the Cognito user pool client.
+        username (str): The username of the user.
+        password (str): The password of the user.
+
+    Returns:
+        bool: True if the user is authenticated, False otherwise.
+    """
+
+    client = boto3.client('cognito-idp')  # Cognito Identity Provider client
+
+    try:
+        response = client.initiate_auth(
+            AuthFlow='USER_PASSWORD_AUTH',
+            AuthParameters={
+                'USERNAME': username,
+                'PASSWORD': password,
+            },
+            ClientId=client_id
+        )
+
+        # If successful, you'll receive an AuthenticationResult in the response
+        if response.get('AuthenticationResult'):
+            return True
+        else:
+            return False
+
+    except client.exceptions.NotAuthorizedException:
+        return False
+    except client.exceptions.UserNotFoundException:
+        return False
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return False

chatfuncs/aws_functions.py

@@ -0,0 +1,205 @@
+from typing import Type, List
+import pandas as pd
+import boto3
+import tempfile
+import os
+from chatfuncs.helper_functions import get_or_create_env_var
+
+PandasDataFrame = Type[pd.DataFrame]
+
+# Get AWS credentials if required
+bucket_name=""
+
+aws_var_val = get_or_create_env_var("RUN_AWS_FUNCTIONS", "1")
+print(f'The value of RUN_AWS_FUNCTIONS is {aws_var_val}')
+
+AWS_REGION = get_or_create_env_var('AWS_REGION', 'eu-west-2')
+print(f'The value of AWS_REGION is {AWS_REGION}')
+
+if aws_var_val == "1":
+    try:
+        bucket_name = os.environ['CONSULTATION_SUMMARY_BUCKET']
+        session = boto3.Session() # profile_name="default"
+    except Exception as e:
+        print(e)
+
+    def get_assumed_role_info():
+        sts_endpoint = 'https://sts.' + AWS_REGION + '.amazonaws.com'
+        sts = boto3.client('sts', region_name=AWS_REGION, endpoint_url=sts_endpoint)
+        response = sts.get_caller_identity()
+
+        # Extract ARN of the assumed role
+        assumed_role_arn = response['Arn']
+        
+        # Extract the name of the assumed role from the ARN
+        assumed_role_name = assumed_role_arn.split('/')[-1]
+        
+        return assumed_role_arn, assumed_role_name
+
+    try:
+        assumed_role_arn, assumed_role_name = get_assumed_role_info()
+
+        print("Assumed Role ARN:", assumed_role_arn)
+        print("Assumed Role Name:", assumed_role_name)
+
+    except Exception as e:
+        
+        print(e)
+
+# Download direct from S3 - requires login credentials
+def download_file_from_s3(bucket_name, key, local_file_path):
+
+    s3 = boto3.client('s3')
+    s3.download_file(bucket_name, key, local_file_path)
+    print(f"File downloaded from S3: s3://{bucket_name}/{key} to {local_file_path}")
+                         
+def download_folder_from_s3(bucket_name, s3_folder, local_folder):
+    """
+    Download all files from an S3 folder to a local folder.
+    """
+    s3 = boto3.client('s3')
+
+    # List objects in the specified S3 folder
+    response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
+
+    # Download each object
+    for obj in response.get('Contents', []):
+        # Extract object key and construct local file path
+        object_key = obj['Key']
+        local_file_path = os.path.join(local_folder, os.path.relpath(object_key, s3_folder))
+
+        # Create directories if necessary
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the object
+        try:
+            s3.download_file(bucket_name, object_key, local_file_path)
+            print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
+        except Exception as e:
+            print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
+
+def download_files_from_s3(bucket_name, s3_folder, local_folder, filenames):
+    """
+    Download specific files from an S3 folder to a local folder.
+    """
+    s3 = boto3.client('s3')
+
+    print("Trying to download file: ", filenames)
+
+    if filenames == '*':
+        # List all objects in the S3 folder
+        print("Trying to download all files in AWS folder: ", s3_folder)
+        response = s3.list_objects_v2(Bucket=bucket_name, Prefix=s3_folder)
+
+        print("Found files in AWS folder: ", response.get('Contents', []))
+
+        filenames = [obj['Key'].split('/')[-1] for obj in response.get('Contents', [])]
+
+        print("Found filenames in AWS folder: ", filenames)
+
+    for filename in filenames:
+        object_key = os.path.join(s3_folder, filename)
+        local_file_path = os.path.join(local_folder, filename)
+
+        # Create directories if necessary
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the object
+        try:
+            s3.download_file(bucket_name, object_key, local_file_path)
+            print(f"Downloaded 's3://{bucket_name}/{object_key}' to '{local_file_path}'")
+        except Exception as e:
+            print(f"Error downloading 's3://{bucket_name}/{object_key}':", e)
+
+def load_data_from_aws(in_aws_keyword_file, aws_password="", bucket_name=bucket_name):
+
+    temp_dir = tempfile.mkdtemp()
+    local_address_stub = temp_dir + '/doc-redaction/'
+    files = []
+
+    if not 'LAMBETH_BOROUGH_PLAN_PASSWORD' in os.environ:
+        out_message = "Can't verify password for dataset access. Do you have a valid AWS connection? Data not loaded."
+        return files, out_message
+    
+    if aws_password:
+        if "Lambeth borough plan" in in_aws_keyword_file and aws_password == os.environ['LAMBETH_BOROUGH_PLAN_PASSWORD']:
+
+            s3_folder_stub = 'example-data/lambeth-borough-plan/latest/'
+
+            local_folder_path = local_address_stub                
+
+            # Check if folder exists
+            if not os.path.exists(local_folder_path):
+                print(f"Folder {local_folder_path} does not exist! Making folder.")
+
+                os.mkdir(local_folder_path)
+
+            # Check if folder is empty
+            if len(os.listdir(local_folder_path)) == 0:
+                print(f"Folder {local_folder_path} is empty")
+                # Download data
+                download_files_from_s3(bucket_name, s3_folder_stub, local_folder_path, filenames='*')
+
+                print("AWS data downloaded")
+
+            else:
+                print(f"Folder {local_folder_path} is not empty")
+
+            #files = os.listdir(local_folder_stub)
+            #print(files)
+
+            files = [os.path.join(local_folder_path, f) for f in os.listdir(local_folder_path) if os.path.isfile(os.path.join(local_folder_path, f))]
+
+            out_message = "Data successfully loaded from AWS"
+            print(out_message)
+
+        else:
+            out_message = "Data not loaded from AWS"
+            print(out_message)
+    else:
+        out_message = "No password provided. Please ask the data team for access if you need this."
+        print(out_message)
+
+    return files, out_message
+
+def upload_file_to_s3(local_file_paths:List[str], s3_key:str, s3_bucket:str=bucket_name):
+    """
+    Uploads a file from local machine to Amazon S3.
+
+    Args:
+    - local_file_path: Local file path(s) of the file(s) to upload.
+    - s3_key: Key (path) to the file in the S3 bucket.
+    - s3_bucket: Name of the S3 bucket.
+
+    Returns:
+    - Message as variable/printed to console
+    """
+    final_out_message = []
+
+    s3_client = boto3.client('s3')
+
+    if isinstance(local_file_paths, str):
+        local_file_paths = [local_file_paths]
+
+    for file in local_file_paths:
+        try:
+            # Get file name off file path
+            file_name = os.path.basename(file)
+
+            s3_key_full = s3_key + file_name
+            print("S3 key: ", s3_key_full)
+
+            s3_client.upload_file(file, s3_bucket, s3_key_full)
+            out_message = "File " + file_name + " uploaded successfully!"
+            print(out_message)
+        
+        except Exception as e:
+            out_message = f"Error uploading file(s): {e}"
+            print(out_message)
+
+        final_out_message.append(out_message)
+        final_out_message_str = '\n'.join(final_out_message)
+
+    return final_out_message_str
+        
+    

chatfuncs/chatfuncs.py

@@ -28,20 +28,25 @@ from langchain.docstore.document import Document
 from nltk.corpus import stopwords
 from nltk.tokenize import RegexpTokenizer
 from nltk.stem import WordNetLemmatizer
+#from nltk.stem.snowball import SnowballStemmer
 from keybert import KeyBERT
 
 # For Name Entity Recognition model
 #from span_marker import SpanMarkerModel # Not currently used
 
+
 # For BM25 retrieval
-from gensim.corpora import Dictionary
-from gensim.models import TfidfModel, OkapiBM25Model
-from gensim.similarities import SparseMatrixSimilarity
+import bm25s
+import Stemmer
+
+#from gensim.corpora import Dictionary
+#from gensim.models import TfidfModel, OkapiBM25Model
+#from gensim.similarities import SparseMatrixSimilarity
 
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
-from chatfuncs.prompts import instruction_prompt_template_alpaca, instruction_prompt_mistral_orca, instruction_prompt_phi3, instruction_prompt_llama3
+from chatfuncs.prompts import instruction_prompt_template_alpaca, instruction_prompt_mistral_orca, instruction_prompt_phi3, instruction_prompt_llama3, instruction_prompt_qwen
 
 import gradio as gr
 
@@ -84,7 +89,7 @@ print("Running on device:", torch_device)
 threads = 8 #torch.get_num_threads()
 print("CPU threads:", threads)
 
-# Flan Alpaca (small, fast) Model parameters
+# Qwen 2 0.5B (small, fast) Model parameters
 temperature: float = 0.1
 top_k: int = 3
 top_p: float = 1
@@ -182,7 +187,7 @@ def docs_to_faiss_save(docs_out:PandasDataFrame, embeddings=embeddings):
 
 # Prompt functions
 
-def base_prompt_templates(model_type = "Flan Alpaca (small, fast)"):    
+def base_prompt_templates(model_type = "Qwen 2 0.5B (small, fast)"):    
   
     #EXAMPLE_PROMPT = PromptTemplate(
     #    template="\nCONTENT:\n\n{page_content}\n\nSOURCE: {source}\n\n",
@@ -196,9 +201,9 @@ def base_prompt_templates(model_type = "Flan Alpaca (small, fast)"):
 
 # The main prompt:  
 
-    if model_type == "Flan Alpaca (small, fast)":
-        INSTRUCTION_PROMPT=PromptTemplate(template=instruction_prompt_template_alpaca, input_variables=['question', 'summaries'])
-    elif model_type == "Phi 3 Mini (larger, slow)":
+    if model_type == "Qwen 2 0.5B (small, fast)":
+        INSTRUCTION_PROMPT=PromptTemplate(template=instruction_prompt_qwen, input_variables=['question', 'summaries'])
+    elif model_type == "Phi 3.5 Mini (larger, slow)":
         INSTRUCTION_PROMPT=PromptTemplate(template=instruction_prompt_phi3, input_variables=['question', 'summaries'])
 
     return INSTRUCTION_PROMPT, CONTENT_PROMPT
@@ -207,89 +212,175 @@ def write_out_metadata_as_string(metadata_in):
     metadata_string = [f"{'  '.join(f'{k}: {v}' for k, v in d.items() if k != 'page_section')}" for d in metadata_in] # ['metadata']
     return metadata_string
 
-def generate_expanded_prompt(inputs: Dict[str, str], instruction_prompt, content_prompt, extracted_memory, vectorstore, embeddings, out_passages = 2): # , 
-        
-        question =  inputs["question"]
-        chat_history = inputs["chat_history"]
+def generate_expanded_prompt(inputs: Dict[str, str], instruction_prompt, content_prompt, extracted_memory, vectorstore, embeddings, relevant_flag = True, out_passages = 2): # , 
         
+    question =  inputs["question"]
+    chat_history = inputs["chat_history"]
 
+    print("relevant_flag in generate_expanded_prompt:", relevant_flag)
+    
+    
+    if relevant_flag == True:
         new_question_kworded = adapt_q_from_chat_history(question, chat_history, extracted_memory) # new_question_keywords, 
-        
-       
-        docs_keep_as_doc, doc_df, docs_keep_out = hybrid_retrieval(new_question_kworded, vectorstore, embeddings, k_val = 25, out_passages = out_passages,
-                                                                          vec_score_cut_off = 0.85, vec_weight = 1, bm25_weight = 1, svm_weight = 1)#,
-                                                                          #vectorstore=globals()["vectorstore"], embeddings=globals()["embeddings"])
-        
-        #print(docs_keep_as_doc)
-        #print(doc_df)
-        if (not docs_keep_as_doc) | (doc_df.empty):
-            sorry_prompt = """Say 'Sorry, there is no relevant information to answer this question.'.
-RESPONSE:"""
-            return sorry_prompt, "No relevant sources found.", new_question_kworded
-        
-        # Expand the found passages to the neighbouring context
-        file_type = determine_file_type(doc_df['meta_url'][0])
+        docs_keep_as_doc, doc_df, docs_keep_out = hybrid_retrieval(new_question_kworded, vectorstore, embeddings, k_val = 25, out_passages = out_passages, vec_score_cut_off = 0.85, vec_weight = 1, bm25_weight = 1, svm_weight = 1)
+    else:
+        new_question_kworded = question
+        doc_df = pd.DataFrame()
+        docs_keep_as_doc = []
+        docs_keep_out = []
+    
+    if (not docs_keep_as_doc) | (doc_df.empty):
+        sorry_prompt = """Say 'Sorry, there is no relevant information to answer this question.'"""
+        return sorry_prompt, "No relevant sources found.", new_question_kworded
+    
+    # Expand the found passages to the neighbouring context
+    print("Doc_df columns:", doc_df.columns)
 
-        # Only expand passages if not tabular data
-        if (file_type != ".csv") & (file_type != ".xlsx"):
-            docs_keep_as_doc, doc_df = get_expanded_passages(vectorstore, docs_keep_out, width=3)
+    if 'meta_url' in doc_df.columns:
+        file_type = determine_file_type(doc_df['meta_url'][0])
+    else:
+        file_type = determine_file_type(doc_df['source'][0]) 
 
-        
- 
-        # Build up sources content to add to user display
-        doc_df['meta_clean'] = write_out_metadata_as_string(doc_df["metadata"]) # [f"<b>{'  '.join(f'{k}: {v}' for k, v in d.items() if k != 'page_section')}</b>" for d in doc_df['metadata']]
-        
-        # Remove meta text from the page content if it already exists there
-        doc_df['page_content_no_meta'] = doc_df.apply(lambda row: row['page_content'].replace(row['meta_clean'] + ". ", ""), axis=1)
-        doc_df['content_meta'] = doc_df['meta_clean'].astype(str) + ".<br><br>" + doc_df['page_content_no_meta'].astype(str)
+    # Only expand passages if not tabular data
+    if (file_type != ".csv") & (file_type != ".xlsx"):
+        docs_keep_as_doc, doc_df = get_expanded_passages(vectorstore, docs_keep_out, width=3)    
 
-        #modified_page_content = [f" Document {i+1} - {word}" for i, word in enumerate(doc_df['page_content'])]
-        modified_page_content = [f" Document {i+1} - {word}" for i, word in enumerate(doc_df['content_meta'])]
-        docs_content_string = '<br><br>'.join(modified_page_content)
+    # Build up sources content to add to user display
+    doc_df['meta_clean'] = write_out_metadata_as_string(doc_df["metadata"]) # [f"<b>{'  '.join(f'{k}: {v}' for k, v in d.items() if k != 'page_section')}</b>" for d in doc_df['metadata']]
+    
+    # Remove meta text from the page content if it already exists there
+    doc_df['page_content_no_meta'] = doc_df.apply(lambda row: row['page_content'].replace(row['meta_clean'] + ". ", ""), axis=1)
+    doc_df['content_meta'] = doc_df['meta_clean'].astype(str) + ".<br><br>" + doc_df['page_content_no_meta'].astype(str)
 
-        sources_docs_content_string = '<br><br>'.join(doc_df['content_meta'])#.replace("  "," ")#.strip()
-     
-        instruction_prompt_out = instruction_prompt.format(question=new_question_kworded, summaries=docs_content_string)
-        
-        print('Final prompt is: ')
-        print(instruction_prompt_out)
-                
-        return instruction_prompt_out, sources_docs_content_string, new_question_kworded
+    #modified_page_content = [f" Document {i+1} - {word}" for i, word in enumerate(doc_df['page_content'])]
+    modified_page_content = [f" Document {i+1} - {word}" for i, word in enumerate(doc_df['content_meta'])]
+    docs_content_string = '<br><br>'.join(modified_page_content)
 
-def create_full_prompt(user_input, history, extracted_memory, vectorstore, embeddings, model_type, out_passages):
+    sources_docs_content_string = '<br><br>'.join(doc_df['content_meta'])#.replace("  "," ")#.strip()
     
-    if not user_input.strip():
-        return history, "", "Respond with 'Please enter a question.' RESPONSE:"
+    instruction_prompt_out = instruction_prompt.format(question=new_question_kworded, summaries=docs_content_string)
+    
+    print('Final prompt is: ')
+    print(instruction_prompt_out)
+            
+    return instruction_prompt_out, sources_docs_content_string, new_question_kworded
 
+def create_full_prompt(user_input, history, extracted_memory, vectorstore, embeddings, model_type, out_passages, api_model_choice=None, api_key=None, relevant_flag = True):
+    
     #if chain_agent is None:
     #    history.append((user_input, "Please click the button to submit the Huggingface API key before using the chatbot (top right)"))
     #    return history, history, "", ""
     print("\n==== date/time: " + str(datetime.datetime.now()) + " ====")
-    print("User input: " + user_input)
     
-    history = history or []
     
+    history = history or []
+
+    if api_model_choice and api_model_choice != "None":
+         print("API model choice detected")
+         if api_key:
+            print("API key detected")
+            return history, "", None, relevant_flag       
+         else:
+            return history, "", None, relevant_flag
+         
     # Create instruction prompt
     instruction_prompt, content_prompt = base_prompt_templates(model_type=model_type)
+
+    if not user_input.strip():
+        user_input = "No user input found"
+        relevant_flag = False
+    else:
+        relevant_flag = True
+
+    print("User input:", user_input)
+   
     instruction_prompt_out, docs_content_string, new_question_kworded =\
                 generate_expanded_prompt({"question": user_input, "chat_history": history}, #vectorstore,
-                                    instruction_prompt, content_prompt, extracted_memory, vectorstore, embeddings, out_passages)
-    
+                                    instruction_prompt, content_prompt, extracted_memory, vectorstore, embeddings, relevant_flag, out_passages)
   
     history.append(user_input)
     
-    print("Output history is:")
-    print(history)
-
-    print("Final prompt to model is:")
-    print(instruction_prompt_out)
+    print("Output history is:", history)
+    print("Final prompt to model is:",instruction_prompt_out)
         
-    return history, docs_content_string, instruction_prompt_out
+    return history, docs_content_string, instruction_prompt_out, relevant_flag
 
 # Chat functions
+import boto3
+import json
+from chatfuncs.helper_functions import get_or_create_env_var
+
+# ResponseObject class for AWS Bedrock calls
+class ResponseObject:
+        def __init__(self, text, usage_metadata):
+            self.text = text
+            self.usage_metadata = usage_metadata
+
+max_tokens = 4096
+
+AWS_DEFAULT_REGION = get_or_create_env_var('AWS_DEFAULT_REGION', 'eu-west-2')
+print(f'The value of AWS_DEFAULT_REGION is {AWS_DEFAULT_REGION}')
 
-def produce_streaming_answer_chatbot(history, full_prompt, model_type,
+bedrock_runtime = boto3.client('bedrock-runtime', region_name=AWS_DEFAULT_REGION)
+
+def call_aws_claude(prompt: str, system_prompt: str, temperature: float, max_tokens: int, model_choice: str) -> ResponseObject:
+    """
+    This function sends a request to AWS Claude with the following parameters:
+    - prompt: The user's input prompt to be processed by the model.
+    - system_prompt: A system-defined prompt that provides context or instructions for the model.
+    - temperature: A value that controls the randomness of the model's output, with higher values resulting in more diverse responses.
+    - max_tokens: The maximum number of tokens (words or characters) in the model's response.
+    - model_choice: The specific model to use for processing the request.
+    
+    The function constructs the request configuration, invokes the model, extracts the response text, and returns a ResponseObject containing the text and metadata.
+    """
+
+    prompt_config = {
+        "anthropic_version": "bedrock-2023-05-31",
+        "max_tokens": max_tokens,
+        "top_p": 0.999,
+        "temperature":temperature,
+        "system": system_prompt,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                ],
+            }
+        ],
+    }
+
+    body = json.dumps(prompt_config)
+
+    modelId = model_choice
+    accept = "application/json"
+    contentType = "application/json"
+
+    request = bedrock_runtime.invoke_model(
+        body=body, modelId=modelId, accept=accept, contentType=contentType
+    )
+
+    # Extract text from request
+    response_body = json.loads(request.get("body").read())
+    text = response_body.get("content")[0].get("text")
+
+    response = ResponseObject(
+    text=text,
+    usage_metadata=request['ResponseMetadata']
+    )
+
+    # Now you can access both the text and metadata
+    #print("Text:", response.text)
+    print("Metadata:", response.usage_metadata)   
+    
+    return response
+
+def produce_streaming_answer_chatbot(history,
+            full_prompt,
+            model_type,
             temperature=temperature,
+            relevant_query_bool=True,
             max_new_tokens=max_new_tokens,
             sample=sample,
             repetition_penalty=repetition_penalty,
@@ -304,7 +395,16 @@ def produce_streaming_answer_chatbot(history, full_prompt, model_type,
 
     #    return history
 
-    if model_type == "Flan Alpaca (small, fast)": 
+    
+
+    if relevant_query_bool == False:
+        out_message = [("","No relevant query found. Please retry your question")]
+        history.append(out_message)
+
+        yield history
+        return
+
+    if model_type == "Qwen 2 0.5B (small, fast)": 
         # Get the model and tokenizer, and tokenize the user text.
         model_inputs = tokenizer(text=full_prompt, return_tensors="pt", return_attention_mask=False).to(torch_device)
         
@@ -322,7 +422,7 @@ def produce_streaming_answer_chatbot(history, full_prompt, model_type,
             top_k=top_k
         )
 
-        print(generate_kwargs)
+        #print(generate_kwargs)
 
         t = Thread(target=model.generate, kwargs=generate_kwargs)
         t.start()
@@ -350,7 +450,7 @@ def produce_streaming_answer_chatbot(history, full_prompt, model_type,
         print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
         print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
 
-    elif model_type == "Phi 3 Mini (larger, slow)":
+    elif model_type == "Phi 3.5 Mini (larger, slow)":
         #tokens = model.tokenize(full_prompt)
 
         gen_config = CtransGenGenerationConfig()
@@ -384,6 +484,33 @@ def produce_streaming_answer_chatbot(history, full_prompt, model_type,
         print(f'Tokens per secound: {NUM_TOKENS/time_generate}')
         print(f'Time per token: {(time_generate/NUM_TOKENS)*1000}ms')
 
+    elif model_type == "anthropic.claude-3-haiku-20240307-v1:0" or model_type == "anthropic.claude-3-sonnet-20240229-v1:0":
+        system_prompt = "You are answering questions from the user based on source material. Respond with short, factually correct answers."
+
+        try:
+            print("Calling AWS Claude model")
+            response = call_aws_claude(full_prompt, system_prompt, temperature, max_tokens, model_type)
+        except Exception as e:
+            # If fails, try again after 10 seconds in case there is a throttle limit
+            print(e)
+            try:
+                out_message = "API limit hit - waiting 30 seconds to retry."
+                print(out_message)
+
+                time.sleep(30)
+                response = call_aws_claude(full_prompt, system_prompt, temperature, max_tokens, model_type)
+            
+            except Exception as e:
+                print(e)
+                return "", history
+        # Update the conversation history with the new prompt and response
+        history.append({'role': 'user', 'parts': [full_prompt]})
+        history.append({'role': 'assistant', 'parts': [response.text]})
+        
+        # Print the updated conversation history
+        #print("conversation_history:", conversation_history)
+        
+        return response, history
 
 # Chat helper functions
 
@@ -507,7 +634,7 @@ def hybrid_retrieval(new_question_kworded, vectorstore, embeddings, k_val, out_p
                 docs_content = doc_df['page_content'].astype(str)
                 docs_url = doc_df['meta_url']
 
-                return docs_keep_as_doc, docs_content, docs_url
+                return docs_keep_as_doc, doc_df, docs_content, docs_url
             
             # Check for if more docs are removed than the desired output
             if out_passages > docs_keep_length: 
@@ -519,47 +646,58 @@ def hybrid_retrieval(new_question_kworded, vectorstore, embeddings, k_val, out_p
 
             print("Number of documents remaining: ", docs_keep_length)
             
-            # 2nd level check on retrieved docs with BM25
-
+            # 2nd level check using BM25s package to do keyword search on retrieved passages.
+            
             content_keep=[]
             for item in docs_keep:
                 content_keep.append(item[0].page_content)
 
-            corpus = corpus = [doc.lower().split() for doc in content_keep]
-            dictionary = Dictionary(corpus)
-            bm25_model = OkapiBM25Model(dictionary=dictionary)
-            bm25_corpus = bm25_model[list(map(dictionary.doc2bow, corpus))]
-            bm25_index = SparseMatrixSimilarity(bm25_corpus, num_docs=len(corpus), num_terms=len(dictionary),
-                                   normalize_queries=False, normalize_documents=False)
-            query = new_question_kworded.lower().split()
-            tfidf_model = TfidfModel(dictionary=dictionary, smartirs='bnn')  # Enforce binary weighting of queries
-            tfidf_query = tfidf_model[dictionary.doc2bow(query)]
-            similarities = np.array(bm25_index[tfidf_query])
-            #print(similarities)
-            temp = similarities.argsort()
-            ranks = np.arange(len(similarities))[temp.argsort()][::-1]
-
-            # Pair each index with its corresponding value
-            pairs = list(zip(ranks, docs_keep_as_doc))
-            # Sort the pairs by the indices
+            # Prepare Corpus (Tokenized & Optional Stemming)
+            corpus = [doc.lower() for doc in content_keep]
+            #stemmer = SnowballStemmer("english", ignore_stopwords=True)  # NLTK stemming not compatible
+            stemmer = Stemmer.Stemmer("english")
+            corpus_tokens = bm25s.tokenize(corpus, stopwords="en", stemmer=stemmer)
+
+            # Create and Index with BM25s
+            retriever = bm25s.BM25()
+            retriever.index(corpus_tokens)
+
+            # Query Processing (Stemming applied consistently if used above)
+            query_tokens = bm25s.tokenize(new_question_kworded.lower(), stemmer=stemmer)
+            results, scores = retriever.retrieve(query_tokens, corpus=corpus, k=len(corpus)) # Retrieve all docs
+
+            for i in range(results.shape[1]):
+                doc, score = results[0, i], scores[0, i]
+                print(f"Rank {i+1} (score: {score:.2f}): {doc}")
+
+            #print("BM25 results:", results)
+            #print("BM25 scores:", scores)
+
+            # Rank Calculation (Custom Logic for Your BM25 Score)
+            bm25_rank = list(range(1, len(results[0]) + 1))
+            #bm25_rank = results[0]#.tolist()[0]  # Since you have a single query
+            bm25_score = [(docs_keep_length / (rank + 1)) * bm25_weight for rank in bm25_rank] 
+            # +1 to avoid division by 0 for rank 0
+
+            # Result Ordering (Using the calculated ranks)
+            pairs = list(zip(bm25_rank, docs_keep_as_doc))
             pairs.sort()
-            # Extract the values in the new order
-            bm25_result = [value for ranks, value in pairs]
+            bm25_result = [value for rank, value in pairs]
             
-            bm25_rank=[]
-            bm25_score = []
-
-            for vec_item in docs_keep:
-                x = 0
-                for bm25_item in bm25_result:
-                    x = x + 1
-                    if bm25_item.page_content == vec_item[0].page_content:
-                        bm25_rank.append(x)
-                        bm25_score.append((docs_keep_length/x)*bm25_weight)
 
             # 3rd level check on retrieved docs with SVM retriever
+            # Check the type of the embeddings object
+            embeddings_type = type(embeddings)
+            print("Type of embeddings object:", embeddings_type)
 
-            svm_retriever = SVMRetriever.from_texts(content_keep, embeddings, k = k_val)
+
+            print("embeddings:", embeddings)
+
+            from langchain_huggingface import HuggingFaceEmbeddings
+            #hf_embeddings = HuggingFaceEmbeddings(**embeddings)
+            hf_embeddings = embeddings
+            
+            svm_retriever = SVMRetriever.from_texts(content_keep, hf_embeddings, k = k_val)
             svm_result = svm_retriever.invoke(new_question_kworded)
 
          
@@ -605,6 +743,10 @@ def hybrid_retrieval(new_question_kworded, vectorstore, embeddings, k_val, out_p
             # Make df of best options
             doc_df = create_doc_df(docs_keep_out)
 
+            print("doc_df:",doc_df)
+            print("docs_keep_as_doc:",docs_keep_as_doc)
+            print("docs_keep_out:", docs_keep_out)
+
             return docs_keep_as_doc, doc_df, docs_keep_out
 
 def get_expanded_passages(vectorstore, docs, width):

chatfuncs/helper_functions.py

@@ -0,0 +1,228 @@
+import os
+import gradio as gr
+import pandas as pd
+
+def get_or_create_env_var(var_name, default_value):
+    # Get the environment variable if it exists
+    value = os.environ.get(var_name)
+    
+    # If it doesn't exist, set it to the default value
+    if value is None:
+        os.environ[var_name] = default_value
+        value = default_value
+    
+    return value
+
+# Retrieving or setting output folder
+env_var_name = 'GRADIO_OUTPUT_FOLDER'
+default_value = 'output/'
+
+output_folder = get_or_create_env_var(env_var_name, default_value)
+print(f'The value of {env_var_name} is {output_folder}')
+
+def get_file_path_with_extension(file_path):
+    # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt")
+    basename = os.path.basename(file_path)
+    
+    # Return the basename with its extension
+    return basename
+
+def get_file_path_end(file_path):
+    # First, get the basename of the file (e.g., "example.txt" from "/path/to/example.txt")
+    basename = os.path.basename(file_path)
+    
+    # Then, split the basename and its extension and return only the basename without the extension
+    filename_without_extension, _ = os.path.splitext(basename)
+
+    #print(filename_without_extension)
+    
+    return filename_without_extension
+
+def detect_file_type(filename):
+    """Detect the file type based on its extension."""
+    if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):
+        return 'csv'
+    elif filename.endswith('.xlsx'):
+        return 'xlsx'
+    elif filename.endswith('.parquet'):
+        return 'parquet'
+    elif filename.endswith('.pdf'):
+        return 'pdf'
+    elif filename.endswith('.jpg'):
+        return 'jpg'
+    elif filename.endswith('.jpeg'):
+        return 'jpeg'
+    elif filename.endswith('.png'):
+        return 'png'
+    else:
+        raise ValueError("Unsupported file type.")
+
+def read_file(filename):
+    """Read the file based on its detected type."""
+    file_type = detect_file_type(filename)
+    
+    if file_type == 'csv':
+        return pd.read_csv(filename, low_memory=False)
+    elif file_type == 'xlsx':
+        return pd.read_excel(filename)
+    elif file_type == 'parquet':
+        return pd.read_parquet(filename)
+
+def ensure_output_folder_exists():
+    """Checks if the 'output/' folder exists, creates it if not."""
+
+    folder_name = "output/"
+
+    if not os.path.exists(folder_name):
+        # Create the folder if it doesn't exist
+        os.makedirs(folder_name)
+        print(f"Created the 'output/' folder.")
+    else:
+        print(f"The 'output/' folder already exists.")
+
+def put_columns_in_df(in_file):
+    new_choices = []
+    concat_choices = []
+    all_sheet_names = []
+    number_of_excel_files = 0
+    
+    for file in in_file:
+        file_name = file.name
+        file_type = detect_file_type(file_name)
+        #print("File type is:", file_type)
+
+        file_end = get_file_path_with_extension(file_name)
+
+        if file_type == 'xlsx':
+            number_of_excel_files += 1
+            new_choices = []
+            print("Running through all xlsx sheets")
+            anon_xlsx = pd.ExcelFile(file_name)
+            new_sheet_names = anon_xlsx.sheet_names
+            # Iterate through the sheet names
+            for sheet_name in new_sheet_names:
+                # Read each sheet into a DataFrame
+                df = pd.read_excel(file_name, sheet_name=sheet_name)
+
+                # Process the DataFrame (e.g., print its contents)
+                print(f"Sheet Name: {sheet_name}")
+                print(df.head())  # Print the first few rows
+
+                new_choices.extend(list(df.columns))
+
+            all_sheet_names.extend(new_sheet_names)
+
+        else:
+            df = read_file(file_name)
+            new_choices = list(df.columns)
+
+        concat_choices.extend(new_choices)
+        
+    # Drop duplicate columns
+    concat_choices = list(set(concat_choices))
+
+    if number_of_excel_files > 0:      
+        return gr.Dropdown(choices=concat_choices, value=concat_choices[0]), gr.Dropdown(choices=all_sheet_names, value=all_sheet_names[0], visible=True), file_end
+    else:
+        return gr.Dropdown(choices=concat_choices, value=concat_choices[0]), gr.Dropdown(visible=False), file_end
+
+# Following function is only relevant for locally-created executable files based on this app (when using pyinstaller it creates a _internal folder that contains tesseract and poppler. These need to be added to the system path to enable the app to run)
+def add_folder_to_path(folder_path: str):
+    '''
+    Check if a folder exists on your system. If so, get the absolute path and then add it to the system Path variable if it doesn't already exist.
+    '''
+
+    if os.path.exists(folder_path) and os.path.isdir(folder_path):
+        print(folder_path, "folder exists.")
+
+        # Resolve relative path to absolute path
+        absolute_path = os.path.abspath(folder_path)
+
+        current_path = os.environ['PATH']
+        if absolute_path not in current_path.split(os.pathsep):
+            full_path_extension = absolute_path + os.pathsep + current_path
+            os.environ['PATH'] = full_path_extension
+            #print(f"Updated PATH with: ", full_path_extension)
+        else:
+            print(f"Directory {folder_path} already exists in PATH.")
+    else:
+        print(f"Folder not found at {folder_path} - not added to PATH")
+
+# Upon running a process, the feedback buttons are revealed
+def reveal_feedback_buttons():
+    return gr.Radio(visible=True), gr.Textbox(visible=True), gr.Button(visible=True), gr.Markdown(visible=True)
+
+def wipe_logs(feedback_logs_loc, usage_logs_loc):
+    try:
+        os.remove(feedback_logs_loc)
+    except Exception as e:
+        print("Could not remove feedback logs file", e)
+    try:
+        os.remove(usage_logs_loc)
+    except Exception as e:
+        print("Could not remove usage logs file", e)
+    
+
+    
+
+async def get_connection_params(request: gr.Request):
+    base_folder = ""
+
+    if request:
+        #print("request user:", request.username)
+
+        #request_data = await request.json()  # Parse JSON body
+        #print("All request data:", request_data)
+        #context_value = request_data.get('context') 
+        #if 'context' in request_data:
+        #     print("Request context dictionary:", request_data['context'])
+
+        # print("Request headers dictionary:", request.headers)
+        # print("All host elements", request.client)           
+        # print("IP address:", request.client.host)
+        # print("Query parameters:", dict(request.query_params))
+        # To get the underlying FastAPI items you would need to use await and some fancy @ stuff for a live query: https://fastapi.tiangolo.com/vi/reference/request/
+        #print("Request dictionary to object:", request.request.body())
+        print("Session hash:", request.session_hash)
+
+        # Retrieving or setting CUSTOM_CLOUDFRONT_HEADER
+        CUSTOM_CLOUDFRONT_HEADER_var = get_or_create_env_var('CUSTOM_CLOUDFRONT_HEADER', '')
+        #print(f'The value of CUSTOM_CLOUDFRONT_HEADER is {CUSTOM_CLOUDFRONT_HEADER_var}')
+
+        # Retrieving or setting CUSTOM_CLOUDFRONT_HEADER_VALUE
+        CUSTOM_CLOUDFRONT_HEADER_VALUE_var = get_or_create_env_var('CUSTOM_CLOUDFRONT_HEADER_VALUE', '')
+        #print(f'The value of CUSTOM_CLOUDFRONT_HEADER_VALUE_var is {CUSTOM_CLOUDFRONT_HEADER_VALUE_var}')
+
+        if CUSTOM_CLOUDFRONT_HEADER_var and CUSTOM_CLOUDFRONT_HEADER_VALUE_var:
+            if CUSTOM_CLOUDFRONT_HEADER_var in request.headers:
+                supplied_cloudfront_custom_value = request.headers[CUSTOM_CLOUDFRONT_HEADER_var]
+                if supplied_cloudfront_custom_value == CUSTOM_CLOUDFRONT_HEADER_VALUE_var:
+                    print("Custom Cloudfront header found:", supplied_cloudfront_custom_value)
+                else:
+                    raise(ValueError, "Custom Cloudfront header value does not match expected value.")
+
+        # Get output save folder from 1 - username passed in from direct Cognito login, 2 - Cognito ID header passed through a Lambda authenticator, 3 - the session hash.
+
+        if request.username:
+            out_session_hash = request.username
+            base_folder = "user-files/"
+            print("Request username found:", out_session_hash)
+
+        elif 'x-cognito-id' in request.headers:
+            out_session_hash = request.headers['x-cognito-id']
+            base_folder = "user-files/"
+            print("Cognito ID found:", out_session_hash)
+
+        else:
+            out_session_hash = request.session_hash
+            base_folder = "temp-files/"
+            # print("Cognito ID not found. Using session hash as save folder:", out_session_hash)
+
+        output_folder = base_folder + out_session_hash + "/"
+        #if bucket_name:
+        #    print("S3 output folder is: " + "s3://" + bucket_name + "/" + output_folder)
+
+        return out_session_hash, output_folder, out_session_hash
+    else:
+        print("No session parameters found.")
+        return "",""

chatfuncs/ingest.py

@@ -573,7 +573,7 @@ def load_embeddings(model_name = "BAAI/bge-base-en-v1.5"):
 
     return embeddings_func
 
-def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding", model_name = "BAAI/bge-base-en-v1.5"):
+def embed_faiss_save_to_zip(docs_out, save_to="output", model_name = "BAAI/bge-base-en-v1.5"):
 
     load_embeddings(model_name=model_name)
 
@@ -582,7 +582,9 @@ def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding",
     print(f"> Total split documents: {len(docs_out)}")
 
     vectorstore = FAISS.from_documents(documents=docs_out, embedding=embeddings)
-        
+
+    if not Path(save_to).exists(): 
+        os.mkdir(save_to)
 
     if Path(save_to).exists():
         vectorstore.save_local(folder_path=save_to)
@@ -599,9 +601,13 @@ def embed_faiss_save_to_zip(docs_out, save_to="faiss_lambeth_census_embedding",
     os.remove(save_to + "/index.faiss")
     os.remove(save_to + "/index.pkl")
 
-    shutil.move(save_to + '.zip', save_to + "/" + save_to + '.zip')
+    save_zip_out = save_to + "/" + save_to + '.zip'
+
+    shutil.move(save_to + '.zip', save_zip_out)
+
+    out_message = "Document processing complete"
 
-    return vectorstore
+    return out_message, vectorstore, save_zip_out
 
 def docs_to_chroma_save(embeddings, docs_out:PandasDataFrame, save_to:str):
     print(f"> Total split documents: {len(docs_out)}")

chatfuncs/llm_api_call.py

@@ -0,0 +1,925 @@
+import os
+import google.generativeai as ai
+import pandas as pd
+import gradio as gr
+import markdown
+import time
+import boto3
+import json
+import string
+import re
+from gradio import Progress
+from typing import List, Tuple
+from io import StringIO
+
+from chatfuncs.prompts import prompt1, prompt2, prompt3, system_prompt, summarise_system_prompt, summarise_prompt
+from chatfuncs.helper_functions import output_folder, detect_file_type, get_file_path_end, read_file, get_or_create_env_var
+
+# ResponseObject class for AWS Bedrock calls
+class ResponseObject:
+        def __init__(self, text, usage_metadata):
+            self.text = text
+            self.usage_metadata = usage_metadata
+
+max_tokens = 4096
+
+
+AWS_DEFAULT_REGION = get_or_create_env_var('AWS_DEFAULT_REGION', 'eu-west-2')
+print(f'The value of AWS_DEFAULT_REGION is {AWS_DEFAULT_REGION}')
+
+bedrock_runtime = boto3.client('bedrock-runtime', region_name=AWS_DEFAULT_REGION)
+
+def normalise_string(text):
+    # Replace two or more dashes with a single dash
+    text = re.sub(r'-{2,}', '-', text)
+    
+    # Replace two or more spaces with a single space
+    text = re.sub(r'\s{2,}', ' ', text)
+    
+    return text
+
+def load_in_file(file_path: str, colname:str):
+    """
+    Loads in a tabular data file and returns data and file name.
+
+    Parameters:
+    - file_path (str): The path to the file to be processed.
+    """
+    file_type = detect_file_type(file_path)
+    print("File type is:", file_type)
+
+    out_file_part = get_file_path_end(file_path)
+    file_data = read_file(file_path)
+
+    file_data[colname].fillna("", inplace=True)
+
+    file_data[colname] = file_data[colname].astype(str).str.replace("\bnan\b", "", regex=True)  
+    
+
+    print(file_data[colname])
+
+    return file_data, out_file_part
+
+def load_in_data_file(file_paths:List[str], in_colnames:List[str], batch_size:int=50, ):
+    '''Load in data table, work out how many batches needed.'''
+
+    try:
+        file_data, file_name = load_in_file(file_paths[0], colname=in_colnames)
+        num_batches = (len(file_data) // batch_size) + 1
+
+    except Exception as e:
+        print(e)
+        file_data = pd.DataFrame()
+        file_name = ""
+        num_batches = 1  
+    
+    return file_data, file_name, num_batches
+
+def data_file_to_markdown_table(file_data:pd.DataFrame, file_name:str, chosen_cols: List[str], output_folder: str, batch_number: int, batch_size: int) -> Tuple[str, str, str]:
+    """
+    Processes a file by simplifying its content based on chosen columns and saves the result to a specified output folder.
+
+    Parameters:
+    - file_data (pd.DataFrame): Tabular data file with responses.
+    - file_name (str): File name with extension.
+    - chosen_cols (List[str]): A list of column names to include in the simplified file.
+    - output_folder (str): The directory where the simplified file will be saved.
+    - batch_number (int): The current batch number for processing.
+    - batch_size (int): The number of rows to process in each batch.
+
+    Returns:
+    - Tuple[str, str, str]: A tuple containing the path to the simplified CSV file, the simplified markdown table as a string, and the file path end (used for naming the output file).
+    """
+
+    #print("\nfile_data_in_markdown func:", file_data)
+    #print("\nBatch size in markdown func:", str(batch_size))
+    
+    normalised_simple_markdown_table = ""
+    simplified_csv_table_path = ""
+
+    # Simplify table to just responses column and the Response reference number
+    simple_file = file_data[[chosen_cols]].reset_index(names="Reference")
+    simple_file["Reference"] = simple_file["Reference"].astype(int) + 1
+    simple_file = simple_file.rename(columns={chosen_cols: "Response"})
+    simple_file["Response"] = simple_file["Response"].str.strip()
+    file_len = len(simple_file["Reference"])
+   
+
+     # Subset the data for the current batch
+    start_row = batch_number * batch_size
+    if start_row > file_len + 1:
+        print("Start row greater than file row length")
+        return simplified_csv_table_path, normalised_simple_markdown_table, file_name
+
+    if (start_row + batch_size) <= file_len + 1:
+        end_row = start_row + batch_size
+    else:
+        end_row = file_len + 1
+
+    simple_file = simple_file[start_row:end_row]  # Select the current batch
+
+    # Remove problematic characters including ASCII and various quote marks
+        # Remove problematic characters including control characters, special characters, and excessive leading/trailing whitespace
+    simple_file["Response"] = simple_file["Response"].str.replace(r'[\x00-\x1F\x7F]|["“”‘’<>]|\\', '', regex=True)  # Remove control and special characters
+    simple_file["Response"] = simple_file["Response"].str.strip()  # Remove leading and trailing whitespace
+    simple_file["Response"] = simple_file["Response"].str.replace(r'\s+', ' ', regex=True)  # Replace multiple spaces with a single space
+
+    # Remove blank and extremely short responses
+    simple_file = simple_file.loc[~(simple_file["Response"].isnull()) & ~(simple_file["Response"].str.len() < 5), :]
+
+    simplified_csv_table_path = output_folder + 'simple_markdown_table_' + file_name + '_row_' + str(start_row) + '_to_' + str(end_row) + '.csv'
+    simple_file.to_csv(simplified_csv_table_path, index=None)
+
+    simple_markdown_table = simple_file.to_markdown(index=None)
+
+    normalised_simple_markdown_table = normalise_string(simple_markdown_table)
+
+    return simplified_csv_table_path, normalised_simple_markdown_table, file_name, start_row, end_row
+
+def construct_gemini_generative_model(in_api_key: str, temperature: float, model_choice: str, system_prompt: str, max_tokens: int) -> Tuple[object, dict]:
+    """
+    Constructs a GenerativeModel for Gemini API calls.
+
+    Parameters:
+    - in_api_key (str): The API key for authentication.
+    - temperature (float): The temperature parameter for the model, controlling the randomness of the output.
+    - model_choice (str): The choice of model to use for generation.
+    - system_prompt (str): The system prompt to guide the generation.
+    - max_tokens (int): The maximum number of tokens to generate.
+
+    Returns:
+    - Tuple[object, dict]: A tuple containing the constructed GenerativeModel and its configuration.
+    """
+    # Construct a GenerativeModel
+    try:
+        if in_api_key:
+            #print("Getting API key from textbox")
+            api_key = in_api_key
+            ai.configure(api_key=api_key)
+        elif "GOOGLE_API_KEY" in os.environ:
+            #print("Searching for API key in environmental variables")
+            api_key = os.environ["GOOGLE_API_KEY"]
+            ai.configure(api_key=api_key)
+        else:
+            print("No API key foound")
+            raise gr.Error("No API key found.")
+    except Exception as e:
+        print(e)
+    
+    config = ai.GenerationConfig(temperature=temperature, max_output_tokens=max_tokens)
+
+    #model = ai.GenerativeModel.from_cached_content(cached_content=cache, generation_config=config)
+    model = ai.GenerativeModel(model_name='models/' + model_choice, system_instruction=system_prompt, generation_config=config)
+    
+    # Upload CSV file (replace with your actual file path)
+    #file_id = ai.upload_file(upload_file_path)
+
+    
+    # if file_type == 'xlsx':
+    #     print("Running through all xlsx sheets")
+    #     #anon_xlsx = pd.ExcelFile(upload_file_path)
+    #     if not in_excel_sheets:
+    #         out_message.append("No Excel sheets selected. Please select at least one to anonymise.")
+    #         continue
+
+    #     anon_xlsx = pd.ExcelFile(upload_file_path)                
+
+    #     # Create xlsx file:
+    #     anon_xlsx_export_file_name = output_folder + out_file_part + "_redacted.xlsx"
+
+
+    ### QUERYING LARGE LANGUAGE MODEL ###
+    # Prompt caching the table and system prompt. See here: https://ai.google.dev/gemini-api/docs/caching?lang=python
+    # Create a cache with a 5 minute TTL. ONLY FOR CACHES OF AT LEAST 32k TOKENS!
+    # cache = ai.caching.CachedContent.create(
+    # model='models/' + model_choice,
+    # display_name=out_file_part, # used to identify the cache
+    # system_instruction=system_prompt_with_table,
+    # ttl=datetime.timedelta(minutes=5),
+    # )
+
+    return model, config
+
+def call_aws_claude(prompt: str, system_prompt: str, temperature: float, max_tokens: int, model_choice: str) -> ResponseObject:
+    """
+    This function sends a request to AWS Claude with the following parameters:
+    - prompt: The user's input prompt to be processed by the model.
+    - system_prompt: A system-defined prompt that provides context or instructions for the model.
+    - temperature: A value that controls the randomness of the model's output, with higher values resulting in more diverse responses.
+    - max_tokens: The maximum number of tokens (words or characters) in the model's response.
+    - model_choice: The specific model to use for processing the request.
+    
+    The function constructs the request configuration, invokes the model, extracts the response text, and returns a ResponseObject containing the text and metadata.
+    """
+
+    prompt_config = {
+        "anthropic_version": "bedrock-2023-05-31",
+        "max_tokens": max_tokens,
+        "top_p": 0.999,
+        "temperature":temperature,
+        "system": system_prompt,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                ],
+            }
+        ],
+    }
+
+    body = json.dumps(prompt_config)
+
+    modelId = model_choice
+    accept = "application/json"
+    contentType = "application/json"
+
+    request = bedrock_runtime.invoke_model(
+        body=body, modelId=modelId, accept=accept, contentType=contentType
+    )
+
+    # Extract text from request
+    response_body = json.loads(request.get("body").read())
+    text = response_body.get("content")[0].get("text")
+
+    response = ResponseObject(
+    text=text,
+    usage_metadata=request['ResponseMetadata']
+    )
+
+    # Now you can access both the text and metadata
+    #print("Text:", response.text)
+    print("Metadata:", response.usage_metadata)   
+    
+    return response
+
+# Function to send a request and update history
+def send_request(prompt: str, conversation_history: List[dict], model: object, config: dict, model_choice: str, system_prompt: str, temperature: float, progress=Progress(track_tqdm=True)) -> Tuple[str, List[dict]]:
+    """
+    This function sends a request to a language model with the given prompt, conversation history, model configuration, model choice, system prompt, and temperature.
+    It constructs the full prompt by appending the new user prompt to the conversation history, generates a response from the model, and updates the conversation history with the new prompt and response.
+    If the model choice is specific to AWS Claude, it calls the `call_aws_claude` function; otherwise, it uses the `model.generate_content` method.
+    The function returns the response text and the updated conversation history.
+    """
+    # Constructing the full prompt from the conversation history
+    full_prompt = "Conversation history:\n"
+    
+    for entry in conversation_history:
+        role = entry['role'].capitalize()  # Assuming the history is stored with 'role' and 'parts'
+        message = ' '.join(entry['parts'])  # Combining all parts of the message
+        full_prompt += f"{role}: {message}\n"
+    
+    # Adding the new user prompt
+    full_prompt += f"\nUser: {prompt}"
+
+    # Print the full prompt for debugging purposes
+    #print("full_prompt:", full_prompt)
+
+    # Generate the model's response
+    if model_choice in ["gemini-1.5-flash-002", "gemini-1.5-pro-002"]:
+        try:
+            response = model.generate_content(contents=full_prompt, generation_config=config)
+        except Exception as e:
+            # If fails, try again after 10 seconds in case there is a throttle limit
+            print(e)
+            try:
+                print("Calling Gemini model")
+                out_message = "API limit hit - waiting 30 seconds to retry."
+                print(out_message)
+                progress(0.5, desc=out_message)
+                time.sleep(30)
+                response = model.generate_content(contents=full_prompt, generation_config=config)
+            except Exception as e:
+                print(e)
+                return "", conversation_history
+    else:
+        try:
+            print("Calling AWS Claude model")
+            response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
+        except Exception as e:
+            # If fails, try again after 10 seconds in case there is a throttle limit
+            print(e)
+            try:
+                out_message = "API limit hit - waiting 30 seconds to retry."
+                print(out_message)
+                progress(0.5, desc=out_message)
+                time.sleep(30)
+                response = call_aws_claude(prompt, system_prompt, temperature, max_tokens, model_choice)
+            
+            except Exception as e:
+                print(e)
+                return "", conversation_history
+
+    # Update the conversation history with the new prompt and response
+    conversation_history.append({'role': 'user', 'parts': [prompt]})
+    conversation_history.append({'role': 'assistant', 'parts': [response.text]})
+    
+    # Print the updated conversation history
+    #print("conversation_history:", conversation_history)
+    
+    return response, conversation_history
+
+def process_requests(prompts: List[str], system_prompt_with_table: str, conversation_history: List[dict], whole_conversation: List[str], whole_conversation_metadata: List[str], model: object, config: dict, model_choice: str, temperature: float, batch_no:int = 1, master:bool = False) -> Tuple[List[ResponseObject], List[dict], List[str], List[str]]:
+    """
+    Processes a list of prompts by sending them to the model, appending the responses to the conversation history, and updating the whole conversation and metadata.
+
+    Args:
+        prompts (List[str]): A list of prompts to be processed.
+        system_prompt_with_table (str): The system prompt including a table.
+        conversation_history (List[dict]): The history of the conversation.
+        whole_conversation (List[str]): The complete conversation including prompts and responses.
+        whole_conversation_metadata (List[str]): Metadata about the whole conversation.
+        model (object): The model to use for processing the prompts.
+        config (dict): Configuration for the model.
+        model_choice (str): The choice of model to use.        
+        temperature (float): The temperature parameter for the model.
+        batch_no (int): Batch number of the large language model request.
+        master (bool): Is this request for the master table.
+
+    Returns:
+        Tuple[List[ResponseObject], List[dict], List[str], List[str]]: A tuple containing the list of responses, the updated conversation history, the updated whole conversation, and the updated whole conversation metadata.
+    """
+    responses = []
+    for prompt in prompts:
+
+        response, conversation_history = send_request(prompt, conversation_history, model=model, config=config, model_choice=model_choice, system_prompt=system_prompt_with_table, temperature=temperature)
+        
+        #print(response.text)
+        print(response.usage_metadata)
+        responses.append(response)
+
+        # Create conversation txt object
+        whole_conversation.append(prompt)
+        whole_conversation.append(response.text)
+
+        # Create conversation metadata
+        if master == False:
+            whole_conversation_metadata.append(f"Query batch {batch_no} prompt {len(responses)} metadata:")
+        else:
+            whole_conversation_metadata.append(f"Query summary metadata:")
+
+        whole_conversation_metadata.append(str(response.usage_metadata))
+
+    return responses, conversation_history, whole_conversation, whole_conversation_metadata
+
+def replace_punctuation_with_underscore(input_string):
+    # Create a translation table where each punctuation character maps to '_'
+    translation_table = str.maketrans(string.punctuation, '_' * len(string.punctuation))
+    
+    # Translate the input string using the translation table
+    return input_string.translate(translation_table)
+
+def clean_markdown_table(text: str):
+    lines = text.splitlines()
+
+    # Remove any empty rows or rows with only pipes
+    cleaned_lines = [line for line in lines if not re.match(r'^\s*\|?\s*\|?\s*$', line)]
+
+    # Merge lines that belong to the same row (i.e., don't start with |)
+    merged_lines = []
+    buffer = ""
+    
+    for line in cleaned_lines:
+        if line.lstrip().startswith('|'):  # If line starts with |, it's a new row
+            if buffer:
+                merged_lines.append(buffer)  # Append the buffered content
+            buffer = line  # Start a new buffer with this row
+        else:
+            # Continuation of the previous row
+            buffer += ' ' + line.strip()  # Add content to the current buffer
+
+    # Don't forget to append the last buffer
+    if buffer:
+        merged_lines.append(buffer)
+
+    # Ensure consistent number of pipes in each row based on the header
+    header_pipes = merged_lines[0].count('|')  # Use the first row to count number of pipes
+    result = []
+
+    for line in merged_lines:
+        # Strip excessive whitespace around pipes
+        line = re.sub(r'\s*\|\s*', '|', line.strip())
+
+        # Replace numbers between pipes with commas and a space
+        line = re.sub(r'(?<=\|)(\s*\d+)(,\s*\d+)+(?=\|)', lambda m: ', '.join(m.group(0).split(',')), line)
+
+        # Replace groups of numbers separated by spaces with commas and a space
+        line = re.sub(r'(?<=\|)(\s*\d+)(\s+\d+)+(?=\|)', lambda m: ', '.join(m.group(0).split()), line)
+
+        # Fix inconsistent number of pipes by adjusting them to match the header
+        pipe_count = line.count('|')
+        if pipe_count < header_pipes:
+            line += '|' * (header_pipes - pipe_count)  # Add missing pipes
+        elif pipe_count > header_pipes:
+            # If too many pipes, split line and keep the first `header_pipes` columns
+            columns = line.split('|')[:header_pipes + 1]  # +1 to keep last pipe at the end
+            line = '|'.join(columns)
+
+        result.append(line)
+
+    # Join lines back into the cleaned markdown text
+    cleaned_text = '\n'.join(result)
+
+    return cleaned_text
+
+def write_llm_output_and_logs(responses: List[ResponseObject], whole_conversation: List[str], whole_conversation_metadata: List[str], out_file_part: str, latest_batch_completed: int, start_row:int, end_row:int, model_choice_clean: str, temperature: float, log_files_output_paths: List[str], existing_reference_df:pd.DataFrame, existing_topics_df:pd.DataFrame, first_run: bool = False) -> None:
+    """
+    Writes the output of the large language model requests and logs to files.
+
+    Parameters:
+    - responses (List[ResponseObject]): A list of ResponseObject instances containing the text and usage metadata of the responses.
+    - whole_conversation (List[str]): A list of strings representing the complete conversation including prompts and responses.
+    - whole_conversation_metadata (List[str]): A list of strings representing metadata about the whole conversation.
+    - out_file_part (str): The base part of the output file name.
+    - latest_batch_completed (int): The index of the current batch.
+    - start_row (int): Start row of the current batch.
+    - end_row (int): End row of the current batch.
+    - model_choice_clean (str): The cleaned model choice string.
+    - temperature (float): The temperature parameter used in the model.
+    - log_files_output_paths (List[str]): A list of paths to the log files.
+    - existing_reference_df (pd.DataFrame): The existing reference dataframe mapping response numbers to topics.
+    - existing_topics_df (pd.DataFrame): The existing unique topics dataframe
+    - first_run (bool): A boolean indicating if this is the first run through this function in this process. Defaults to False.
+    """
+    unique_topics_df_out_path = []
+    topic_table_out_path = "topic_table_error.csv"
+    reference_table_out_path = "reference_table_error.csv"
+    unique_topics_df_out_path = "unique_topic_table_error.csv"
+    topic_with_response_df = pd.DataFrame()
+    markdown_table = ""
+    out_reference_df = pd.DataFrame()
+    out_unique_topics_df = pd.DataFrame()
+    batch_out_file_part = "error"
+
+
+    # If there was an error in parsing, return boolean saying error
+    is_error = False
+
+    # Convert conversation to string and add to log outputs
+    whole_conversation_str = '\n'.join(whole_conversation)
+    whole_conversation_metadata_str = '\n'.join(whole_conversation_metadata)
+
+    start_row_reported = start_row + 1
+
+    # Save outputs for each batch. If master file created, label file as master
+    if first_run == True:
+        batch_out_file_part = f"{out_file_part}_batch_{latest_batch_completed + 1}"
+        batch_part = f"Rows {start_row_reported} to {end_row}: "
+    else:
+        batch_out_file_part = f"{out_file_part}_combined_batch_{latest_batch_completed + 1}"
+        batch_part = f"Rows {start_row_reported} to {end_row}: "
+
+    whole_conversation_path = output_folder + batch_out_file_part + "_full_conversation_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
+    whole_conversation_path_meta = output_folder + batch_out_file_part + "_metadata_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
+
+    # print("whole_conversation:", whole_conversation_str)
+
+    with open(whole_conversation_path, "w", encoding='utf-8', errors='replace') as f:
+        f.write(whole_conversation_str)
+
+    with open(whole_conversation_path_meta, "w", encoding='utf-8', errors='replace') as f:
+        f.write(whole_conversation_metadata_str)
+
+    log_files_output_paths.append(whole_conversation_path)
+    log_files_output_paths.append(whole_conversation_path_meta)
+
+    # Convert output table to markdown and then to a pandas dataframe to csv
+    # try:
+    cleaned_response = clean_markdown_table(responses[-1].text)
+    
+    markdown_table = markdown.markdown(cleaned_response, extensions=['tables'])
+
+    #print("markdown_table:", markdown_table)
+
+    # Remove <p> tags and make sure it has a valid HTML structure
+    html_table = re.sub(r'<p>(.*?)</p>', r'\1', markdown_table)
+    html_table = html_table.replace('<p>', '').replace('</p>', '').strip()
+
+    # Now ensure that the HTML structure is correct
+    if "<table>" not in html_table:
+        html_table = f"""
+        <table>
+            {html_table}
+        </table>
+        """
+
+    # print("Markdown table as HTML:", html_table)
+
+    html_buffer = StringIO(html_table)
+    #print("html_buffer:", html_buffer)
+
+    try:
+        topic_with_response_df = pd.read_html(html_buffer)[0]  # Assuming the first table in the HTML is the one you want
+    except Exception as e:
+        print("Error when trying to parse table:", e)
+        is_error = True
+        return topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, topic_with_response_df, markdown_table, out_reference_df, out_unique_topics_df, batch_out_file_part, is_error
+
+
+    # Rename columns to ensure consistent use of data frames later in code
+    topic_with_response_df.columns = ["General Topic", "Subtopic", "Sentiment", "Summary", "Response References"]
+
+    # Fill in NA rows with values from above (topics seem to be included only on one row):
+    topic_with_response_df = topic_with_response_df.ffill()
+
+    topic_table_out_path = output_folder + batch_out_file_part + "_topic_table_" + model_choice_clean + "_temp_" + str(temperature) + ".csv"
+
+    # Table to map references to topics
+    reference_data = []
+
+    # Iterate through each row in the original DataFrame
+    for index, row in topic_with_response_df.iterrows():
+        references = re.split(r',\s*|\s+', str(row.iloc[4]))  # Split the reference numbers
+        topic = row.iloc[0]
+        subtopic = row.iloc[1]
+        sentiment = row.iloc[2]
+        summary = row.iloc[3]
+
+        summary = batch_part + summary
+
+        # Create a new entry for each reference number
+        for ref in references:
+            reference_data.append({
+                'Response References': ref,
+                'General Topic': topic,
+                'Subtopic': subtopic,
+                'Sentiment': sentiment,
+                'Summary': summary,
+                "Start row of group": start_row_reported
+            })
+
+    # Create a new DataFrame from the reference data
+    new_reference_df = pd.DataFrame(reference_data)
+    
+    # Append on old reference data
+    out_reference_df = pd.concat([new_reference_df, existing_reference_df])
+
+    # Remove duplicate Response references for the same topic
+    out_reference_df.drop_duplicates(["Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
+
+    out_reference_df.sort_values(["Start row of group", "Response References", "General Topic", "Subtopic", "Sentiment"], inplace=True)
+
+
+    reference_counts = out_reference_df.groupby(["General Topic", "Subtopic", "Sentiment"]).agg({
+    'Response References': 'size',  # Count the number of references
+    'Summary': lambda x: '<br>'.join(
+        sorted(set(x), key=lambda summary: out_reference_df.loc[out_reference_df['Summary'] == summary, 'Start row of group'].min())
+    )
+    }).reset_index()
+
+    # Save the new DataFrame to CSV
+    reference_table_out_path = output_folder + batch_out_file_part + "_reference_table_" + model_choice_clean + "_temp_" + str(temperature) + ".csv"
+    
+    # Table of all unique topics with descriptions
+    new_unique_topics_df = topic_with_response_df[["General Topic", "Subtopic", "Sentiment"]] # , "Summary"
+    
+    # Join existing and new unique topics
+    out_unique_topics_df = pd.concat([new_unique_topics_df, existing_topics_df]).drop_duplicates(["Subtopic"]).drop(["Response References", "Summary"], axis = 1, errors="ignore")        
+
+    # Join the counts to existing_unique_topics_df
+    out_unique_topics_df = out_unique_topics_df.merge(reference_counts, how='left', on=["General Topic", "Subtopic", "Sentiment"]).sort_values("Response References", ascending=False)
+
+    unique_topics_df_out_path = output_folder + batch_out_file_part + "_unique_topics_" + model_choice_clean + "_temp_" + str(temperature) + ".csv"
+
+    # except Exception as e:
+    #     print("Error in write_llm_output_and_logs:")
+    #     print(e)
+
+    return topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, topic_with_response_df, markdown_table, out_reference_df, out_unique_topics_df, batch_out_file_part, is_error
+
+def llm_query(file_data:pd.DataFrame, existing_topics_w_references_table:pd.DataFrame, existing_reference_df:pd.DataFrame, existing_unique_topics_df:pd.DataFrame, display_table:str, file_name:str, num_batches:int, in_api_key:str, temperature:float, chosen_cols:List[str], model_choice:str, candidate_topics: List=[],latest_batch_completed:int=0, out_message:List=[], out_file_paths:List = [], log_files_output_paths:List = [], first_loop_state:bool=False, whole_conversation_metadata_str:str="", prompt1:str=prompt1, prompt2:str=prompt2, prompt3:str=prompt3, system_prompt:str=system_prompt, summarise_system_prompt:str=summarise_system_prompt, summarise_prompt:str=summarise_prompt, number_of_requests:int=1, batch_size:int=50, max_tokens:int=max_tokens, progress=Progress(track_tqdm=True)):
+
+    '''
+    Query an LLM (Gemini or AWS Anthropic-based) with up to three prompts about a table of open text data. Up to 'batch_size' rows will be queried at a time.
+
+    Parameters:
+    - file_data (pd.DataFrame): Pandas dataframe containing the consultation response data.
+    - existing_topics_w_references_table (pd.DataFrame): Pandas dataframe containing the latest master topic table that has been iterated through batches.
+    - existing_reference_df (pd.DataFrame): Pandas dataframe containing the list of Response reference numbers alongside the derived topics and subtopics.
+    - existing_unique_topics_df (pd.DataFrame): Pandas dataframe containing the unique list of topics, subtopics, sentiment and summaries until this point.
+    - display_table (str): Table for display in markdown format.
+    - file_name (str): File name of the data file.
+    - num_batches (int): Number of batches required to go through all the response rows.
+    - in_api_key (str): The API key for authentication.
+    - temperature (float): The temperature parameter for the model.
+    - chosen_cols (List[str]): A list of chosen columns to process.
+    - candidate_topics (List): A list of existing candidate topics submitted by the user.
+    - model_choice (str): The choice of model to use.
+    - latest_batch_completed (int): The index of the latest file completed.
+    - out_message (list): A list to store output messages.
+    - out_file_paths (list): A list to store output file paths.
+    - log_files_output_paths (list): A list to store log file output paths.
+    - first_loop_state (bool): A flag indicating the first loop state.
+    - whole_conversation_metadata_str (str): A string to store whole conversation metadata.
+    - prompt1 (str): The first prompt for the model.
+    - prompt2 (str): The second prompt for the model.
+    - prompt3 (str): The third prompt for the model.
+    - system_prompt (str): The system prompt for the model.
+    - summarise_system_prompt (str): The system prompt for the summary part of the model.
+    - summarise_prompt (str): The prompt for the model summary.
+    - number of requests (int): The number of prompts to send to the model.
+    - batch_size (int): The number of data rows to consider in each request.
+    - max_tokens (int): The maximum number of tokens for the model.
+    - progress (Progress): A progress tracker.
+    '''
+
+    tic = time.perf_counter()
+    model = ""
+    config = ""
+    final_time = 0.0
+    whole_conversation_metadata = []
+    all_topic_tables_df = []
+    all_markdown_topic_tables = []
+    is_error = False
+
+    # Reset output files on each run:
+    # out_file_paths = []
+
+    model_choice_clean = replace_punctuation_with_underscore(model_choice)
+
+    # If this is the first time around, set variables to 0/blank
+    if first_loop_state==True:
+        latest_batch_completed = 0
+        out_message = []
+        out_file_paths = []
+
+    print("latest_batch_completed:", str(latest_batch_completed))
+
+    if num_batches > 0:
+        progress_measure = round(latest_batch_completed / num_batches, 1)
+        progress(progress_measure, desc="Querying large language model")
+    else:
+        progress(0.1, desc="Querying large language model")
+
+    # Load file
+    # If out message or out_file_paths are blank, change to a list so it can be appended to
+    if isinstance(out_message, str):
+        out_message = [out_message]
+
+    if not out_file_paths:
+        out_file_paths = []
+
+    # Check if files and text exist
+    if file_data.empty:
+        out_message = "Please enter text or a file to redact."
+        return out_message, existing_topics_w_references_table, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_message
+    
+    if model_choice == "anthropic.claude-3-sonnet-20240229-v1:0" and file_data.shape[1] > 300:
+        out_message = "Your data has more than 300 rows, using the Sonnet model will be too expensive. Please choose the Haiku model instead."
+        return out_message, existing_topics_w_references_table, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, out_message
+        
+    # If we have already redacted the last file, return the input out_message and file list to the relevant components
+    if latest_batch_completed >= num_batches:
+        print("Last batch reached, returning batch:", str(latest_batch_completed))
+        # Set to a very high number so as not to mess with subsequent file processing by the user
+        latest_batch_completed = 999
+
+        toc = time.perf_counter()
+        final_time = toc - tic
+        #out_time = f"in {final_time} seconds."
+        #print(out_time)  
+
+        final_out_message = '\n'.join(out_message)
+        return display_table, existing_topics_w_references_table, existing_unique_topics_df, existing_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, final_out_message
+    
+    #for latest_batch_completed in range(num_batches):
+    reported_batch_no = latest_batch_completed + 1  
+    print("Running query batch", str(reported_batch_no))
+
+    # Call the function to prepare the input table
+    simplified_csv_table_path, normalised_simple_markdown_table, out_file_part, start_row, end_row = data_file_to_markdown_table(file_data, file_name, chosen_cols, output_folder, latest_batch_completed, batch_size)
+    log_files_output_paths.append(simplified_csv_table_path)
+
+
+    # Conversation history
+    conversation_history = []
+
+
+    # If this is the second batch, the master table will refer back to the current master table when assigning topics to the new table. Also runs if there is an existing list of topics supplied by the user
+    if latest_batch_completed >= 1 or candidate_topics:
+
+        #print("normalised_simple_markdown_table:", normalised_simple_markdown_table)
+
+        # Prepare Gemini models before query       
+        if model_choice in ["gemini-1.5-flash-002", "gemini-1.5-pro-002"]:
+            print("Using Gemini model:", model_choice)
+            model, config = construct_gemini_generative_model(in_api_key=in_api_key, temperature=temperature, model_choice=model_choice, system_prompt=summarise_system_prompt, max_tokens=max_tokens)
+        else:
+            print("Using AWS Bedrock model:", model_choice)
+
+        # Merge duplicate topics together to create a big merged summary table
+        #all_topic_tables_df_merged = existing_topics_w_references_table#pd.concat(all_topic_tables_df)
+
+        # Group by the first three columns and concatenate the fourth and fifth columns
+        # all_topic_tables_df_merged = existing_topics_w_references_table.groupby(["General Topic", "Subtopic", "Sentiment"], as_index=False).agg({
+        #     "Summary": '\n'.join,   # Concatenate the fourth column
+        #     "Response References": ', '.join  # Concatenate the fifth column
+        # })
+        # all_topic_tables_df_merged["Response References"] = ""
+        #all_topic_tables_df_merged["Summary"] = ""
+        #all_topic_tables_str = all_topic_tables_df_merged.to_markdown(index=None)
+
+        if candidate_topics:
+            # 'Zero shot topics' are those supplied by the user
+            zero_shot_topics = read_file(candidate_topics.name)
+            zero_shot_topics_series = zero_shot_topics.iloc[:, 0]
+            # Max 150 topics allowed
+            if len(zero_shot_topics_series) > 120:
+                print("Maximum 120 topics allowed to fit within large language model context limits.")
+                zero_shot_topics_series = zero_shot_topics_series.iloc[:120]
+
+            zero_shot_topics_list = list(zero_shot_topics_series)
+
+            print("Zero shot topics are:", zero_shot_topics_list)
+
+        #all_topic_tables_df_merged = existing_unique_topics_df
+        existing_unique_topics_df["Response References"] = ""
+
+        
+  
+
+        # Create the most up to date list of topics and subtopics.
+        # If there are candidate topics, but the existing_unique_topics_df hasn't yet been constructed, then create.
+        if candidate_topics and existing_unique_topics_df.empty:
+            existing_unique_topics_df = pd.DataFrame(data={'General Topic':'', 'Subtopic':zero_shot_topics_list, 'Sentiment':''})
+
+        # This part concatenates all zero shot and new topics together, so that for the next prompt the LLM will have the full list available
+        elif candidate_topics and not existing_unique_topics_df.empty:
+            zero_shot_topics_df = pd.DataFrame(data={'General Topic':'', 'Subtopic':zero_shot_topics_list, 'Sentiment':''})
+            existing_unique_topics_df = pd.concat([existing_unique_topics_df, zero_shot_topics_df]).drop_duplicates("Subtopic")
+
+            #print("Full topics list with zero shot_dropped:", existing_unique_topics_df)
+
+            existing_unique_topics_df.to_csv(output_folder + "Existing topics with zero shot dropped.csv")
+
+
+        unique_topics_markdown = existing_unique_topics_df[["General Topic", "Subtopic", "Sentiment"]].drop_duplicates(["General Topic", "Subtopic", "Sentiment"]).to_markdown(index=False)
+       
+        existing_unique_topics_df.to_csv(output_folder + f"{out_file_part}_master_all_topic_tables_df_merged_" + model_choice_clean + "_temp_" + str(temperature) + "_batch_" + str(latest_batch_completed) + ".csv")
+
+        # Format the summary prompt with the response table and topics
+        formatted_summary_prompt = summarise_prompt.format(response_table=normalised_simple_markdown_table, topics=unique_topics_markdown)
+
+        # Define the output file path for the formatted prompt
+        formatted_prompt_output_path = output_folder + out_file_part + "_full_prompt_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
+
+        # Write the formatted prompt to the specified file
+        try:
+            with open(formatted_prompt_output_path, "w", encoding='utf-8', errors='replace') as f:
+                f.write(formatted_summary_prompt)
+        except Exception as e:
+            print(f"Error writing prompt to file {formatted_prompt_output_path}: {e}")
+
+        summary_prompt_list = [formatted_summary_prompt]
+
+        print("master_summary_prompt_list:", summary_prompt_list[0])
+
+        summary_conversation_history = []
+        summary_whole_conversation = []
+
+        # Process requests to large language model
+        master_summary_response, summary_conversation_history, whole_summary_conversation, whole_conversation_metadata = process_requests(summary_prompt_list, summarise_system_prompt, summary_conversation_history, summary_whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, reported_batch_no, master = True)
+
+        print("master_summary_response:", master_summary_response[-1].text)
+        print("Whole conversation metadata:", whole_conversation_metadata)
+
+        new_topic_table_out_path, new_reference_table_out_path, new_unique_topics_df_out_path, new_topic_df, new_markdown_table, new_reference_df, new_unique_topics_df, master_batch_out_file_part, is_error =  write_llm_output_and_logs(master_summary_response, whole_summary_conversation, whole_conversation_metadata, out_file_part, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, first_run=False)
+
+        # If error in table parsing, leave function
+        if is_error == True:
+            final_message_out = "Could not complete summary, error in LLM output."
+            display_table, new_topic_df, new_unique_topics_df, new_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, final_message_out
+
+        # Write outputs to csv
+        ## Topics with references
+        new_topic_df.to_csv(new_topic_table_out_path, index=None)
+        log_files_output_paths.append(new_topic_table_out_path)
+
+        ## Reference table mapping response numbers to topics
+        new_reference_df.to_csv(new_reference_table_out_path, index=None)
+        log_files_output_paths.append(new_reference_table_out_path)
+
+        ## Unique topic list
+        new_unique_topics_df.to_csv(new_unique_topics_df_out_path, index=None)
+        out_file_paths.append(new_unique_topics_df_out_path)
+
+        all_topic_tables_df.append(new_topic_df)
+        all_markdown_topic_tables.append(new_markdown_table)
+
+        #display_table = master_summary_response[-1].text
+
+        # Show unique topics alongside document counts as output
+        display_table = new_unique_topics_df.to_markdown(index=False)
+
+        whole_conversation_metadata.append(whole_conversation_metadata_str)
+        whole_conversation_metadata_str = ' '.join(whole_conversation_metadata)
+
+
+        # Write final output to text file also
+        try:
+            new_final_table_output_path = output_folder + master_batch_out_file_part + "_full_final_response_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
+
+            with open(new_final_table_output_path, "w", encoding='utf-8', errors='replace') as f:
+                f.write(display_table)
+
+            log_files_output_paths.append(new_final_table_output_path)
+
+        except Exception as e:
+            print(e)
+
+    # If this is the first batch, run this
+    else:
+        #system_prompt_with_table = system_prompt + normalised_simple_markdown_table
+
+        # Prepare Gemini models before query       
+        if model_choice in ["gemini-1.5-flash-002", "gemini-1.5-pro-002"]:
+            print("Using Gemini model:", model_choice)
+            model, config = construct_gemini_generative_model(in_api_key=in_api_key, temperature=temperature, model_choice=model_choice, system_prompt=system_prompt, max_tokens=max_tokens)
+        else:
+            print("Using AWS Bedrock model:", model_choice)
+
+        formatted_prompt1 = prompt1.format(response_table=normalised_simple_markdown_table)
+
+        if prompt2: formatted_prompt2 = prompt2.format(response_table=normalised_simple_markdown_table)
+        else: formatted_prompt2 = prompt2
+        
+        if prompt3: formatted_prompt3 = prompt3.format(response_table=normalised_simple_markdown_table)
+        else: formatted_prompt3 = prompt3
+
+        batch_prompts = [formatted_prompt1, formatted_prompt2, formatted_prompt3][:number_of_requests]  # Adjust this list to send fewer requests 
+        
+        #whole_conversation = [system_prompt_with_table]
+
+        whole_conversation = [system_prompt] 
+
+        # Process requests to large language model
+        responses, conversation_history, whole_conversation, whole_conversation_metadata = process_requests(batch_prompts, system_prompt, conversation_history, whole_conversation, whole_conversation_metadata, model, config, model_choice, temperature, reported_batch_no)
+        
+        #print("Whole conversation metadata before:", whole_conversation_metadata)
+
+        print("responses:", responses[-1].text)
+        print("Whole conversation metadata:", whole_conversation_metadata)
+
+        topic_table_out_path, reference_table_out_path, unique_topics_df_out_path, topic_table_df, markdown_table, reference_df, new_unique_topics_df, batch_out_file_part, is_error =  write_llm_output_and_logs(responses, whole_conversation, whole_conversation_metadata, out_file_part, latest_batch_completed, start_row, end_row, model_choice_clean, temperature, log_files_output_paths, existing_reference_df, existing_unique_topics_df, first_run=True)
+
+        # If error in table parsing, leave function
+        if is_error == True:
+            display_table, new_topic_df, new_unique_topics_df, new_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, final_message_out
+        
+        
+        all_topic_tables_df.append(topic_table_df)
+
+        topic_table_df.to_csv(topic_table_out_path, index=None)
+        out_file_paths.append(topic_table_out_path)
+
+        reference_df.to_csv(reference_table_out_path, index=None)
+        log_files_output_paths.append(reference_table_out_path)
+
+        ## Unique topic list
+
+        new_unique_topics_df = pd.concat([new_unique_topics_df, existing_unique_topics_df]).drop_duplicates('Subtopic')
+
+        print("new_unique_topics_df:", new_unique_topics_df)
+
+        new_unique_topics_df.to_csv(unique_topics_df_out_path, index=None)
+        out_file_paths.append(unique_topics_df_out_path)
+        
+        all_markdown_topic_tables.append(markdown_table)
+
+        whole_conversation_metadata.append(whole_conversation_metadata_str)
+        whole_conversation_metadata_str = ' '.join(whole_conversation_metadata)
+        
+        # Write final output to text file also
+        try:
+            final_table_output_path = output_folder + batch_out_file_part + "_full_final_response_" + model_choice_clean + "_temp_" + str(temperature) + ".txt"
+
+            with open(final_table_output_path, "w", encoding='utf-8', errors='replace') as f:
+                f.write(responses[-1].text)
+
+            log_files_output_paths.append(final_table_output_path)
+
+        except Exception as e:
+            print(e)
+
+        display_table = responses[-1].text
+        new_topic_df = topic_table_df
+        new_reference_df = reference_df
+
+    # Increase latest file completed count unless we are at the last file
+    if latest_batch_completed != num_batches:
+        print("Completed file number:", str(latest_batch_completed))
+        latest_batch_completed += 1 
+
+    toc = time.perf_counter()
+    final_time = toc - tic
+    out_time = f"in {final_time:0.1f} seconds."
+    print(out_time)    
+    
+    out_message.append('All queries successfully completed in')
+
+    final_message_out = '\n'.join(out_message)
+    final_message_out = final_message_out + " " + out_time
+
+    final_message_out = final_message_out + "\n\nGo to to the LLM settings tab to see redaction logs. Please give feedback on the results below to help improve this app."    
+
+    return display_table, new_topic_df, new_unique_topics_df, new_reference_df, out_file_paths, out_file_paths, latest_batch_completed, log_files_output_paths, log_files_output_paths, whole_conversation_metadata_str, final_time, final_message_out

chatfuncs/prompts.py

@@ -64,4 +64,13 @@ You are an AI assistant that follows instruction extremely well. Help as much as
 Answer the QUESTION using information from the following CONTENT. Respond with short answers that directly answer the question.\n
 CONTENT: {summaries}\n
 QUESTION: {question}\n
-Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n"""
+Answer:<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n"""
+
+instruction_prompt_qwen = """<|im_start|>system\n
+You are an AI assistant that follows instruction extremely well. Help as much as you can.
+<|im_start|>user\n
+Answer the QUESTION using information from the following CONTENT. Respond with short answers that directly answer the question.
+CONTENT: {summaries}
+QUESTION: {question}\n
+Answer:<|im_end|>
+<|im_start|>assistant\n"""

requirements.txt

@@ -1,19 +1,22 @@
 langchain
 langchain-community
+langchain-huggingface
 beautifulsoup4
+google-generativeai==0.7.2
 pandas
-transformers==4.34.0
+transformers==4.41.2
+torch --extra-index-url https://download.pytorch.org/whl/cu121
 llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121
-torch \
---extra-index-url https://download.pytorch.org/whl/cu121
-sentence_transformers==2.2.2
+sentence_transformers==3.0.1
 faiss-cpu==1.7.4
 pypdf
 python-docx
 keybert
 span_marker
-gensim
-gradio==3.50.2
-gradio_client
+#gensim
+gradio
 nltk
-scipy<1.13
+bm25s
+PyStemmer
+scipy<1.13
+numpy==1.26.4

requirements_cpu.txt

@@ -1,18 +1,22 @@
 langchain
+langchain-huggingface
 langchain-community
 beautifulsoup4
+google-generativeai==0.7.2
 pandas
-transformers==4.34.0
+transformers==4.41.2
 llama-cpp-python --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-torch
-sentence_transformers==2.2.2
+torch==2.3.1
+sentence_transformers==3.0.1
 faiss-cpu==1.7.4
 pypdf
 python-docx
 keybert
 span_marker
-gensim
-gradio==3.50.2
-gradio_client
+#gensim
+gradio
 nltk
-scipy<1.13
+bm25s
+PyStemmer
+scipy<1.13
+numpy==1.26.4