Added clean data options, improved re-representation options and visualisation. General format changes

.gitignore

@@ -2,6 +2,8 @@
 *.ipynb
 *.npz
 *.csv
+*.xlsx
+*.xls
 *.pkl
 *.parquet
 *.png

app.py

@@ -1,516 +1,13 @@
-import os
-
 # Dendrograms will not work with the latest version of scipy (1.12.0), so installing the version prior to be safe
-os.system("pip install scipy==1.11.4")
+#os.system("pip install scipy==1.11.4")
 
 import gradio as gr
-from datetime import datetime
 import pandas as pd
 import numpy as np
-import time
 
-from sentence_transformers import SentenceTransformer
+from funcs.topic_core_funcs import pre_clean, extract_topics, reduce_outliers, represent_topics, visualise_topics, save_as_pytorch_model
+from funcs.helper_functions import dummy_function, initial_file_load
 from sklearn.feature_extraction.text import CountVectorizer
-from sklearn.pipeline import make_pipeline
-from sklearn.decomposition import TruncatedSVD
-from sklearn.feature_extraction.text import TfidfVectorizer
-import funcs.anonymiser as anon
-from umap import UMAP
-
-from torch import cuda, backends, version
-
-# Default seed, can be changed in number selection on options page
-random_seed = 42
-
-# Check for torch cuda
-# If you want to disable cuda for testing purposes
-#os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
-
-print("Is CUDA enabled? ", cuda.is_available())
-print("Is a CUDA device available on this computer?", backends.cudnn.enabled)
-if cuda.is_available():
-    torch_device = "gpu"
-    print("Cuda version installed is: ", version.cuda)
-    low_resource_mode = "No"
-    #os.system("nvidia-smi")
-else: 
-    torch_device =  "cpu"
-    low_resource_mode = "Yes"
-
-print("Device used is: ", torch_device)
-
-
-
-from bertopic import BERTopic
-
-
-today = datetime.now().strftime("%d%m%Y")
-today_rev = datetime.now().strftime("%Y%m%d")
-
-from funcs.helper_functions import dummy_function, initial_file_load, read_file, zip_folder, delete_files_in_folder, save_topic_outputs
-#from funcs.representation_model import representation_model
-from funcs.embeddings import make_or_load_embeddings
-
-# Log terminal output: https://github.com/gradio-app/gradio/issues/2362
-import sys
-
-class Logger:
-    def __init__(self, filename):
-        self.terminal = sys.stdout
-        self.log = open(filename, "w")
-
-    def write(self, message):
-        self.terminal.write(message)
-        self.log.write(message)
-        
-    def flush(self):
-        self.terminal.flush()
-        self.log.flush()
-        
-    def isatty(self):
-        return False    
-
-sys.stdout = Logger("output.log")
-
-def read_logs():
-    sys.stdout.flush()
-    with open("output.log", "r") as f:
-        return f.read()
-
-# Load embeddings
-embeddings_name = "BAAI/bge-small-en-v1.5" #"jinaai/jina-embeddings-v2-base-en"
-
-# Use of Jina deprecated - kept here for posterity
-# Pinning a Jina revision for security purposes: https://www.baseten.co/blog/pinning-ml-model-revisions-for-compatibility-and-security/
-# Save Jina model locally as described here: https://huggingface.co/jinaai/jina-embeddings-v2-base-en/discussions/29
-# local_embeddings_location = "model/jina/"
-#revision_choice = "b811f03af3d4d7ea72a7c25c802b21fc675a5d99"
-#revision_choice = "69d43700292701b06c24f43b96560566a4e5ad1f"
-
-# Model used for representing topics
-hf_model_name =  'second-state/stablelm-2-zephyr-1.6b-GGUF' #'TheBloke/phi-2-orange-GGUF' #'NousResearch/Nous-Capybara-7B-V1.9-GGUF'
-hf_model_file =   'stablelm-2-zephyr-1_6b-Q5_K_M.gguf' # 'phi-2-orange.Q5_K_M.gguf' #'Capybara-7B-V1.9-Q5_K_M.gguf'
-
-def extract_topics(data, in_files, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, data_file_name_no_ext, custom_labels_df, anonymise_drop, return_intermediate_files, embeddings_super_compress, low_resource_mode, save_topic_model, embeddings_out, zero_shot_similarity, random_seed, calc_probs, progress=gr.Progress(track_tqdm=True)):
-
-    progress(0, desc= "Loading data")
-
-    if calc_probs == "No":
-        calc_probs = False
-    elif calc_probs == "Yes":
-        print("Calculating all probabilities.")
-        calc_probs == True
-
-    if not in_colnames:
-        error_message = "Please enter one column name to use to find topics."
-        print(error_message)
-        return error_message, None, embeddings_out, data_file_name_no_ext, None, None
-
-    all_tic = time.perf_counter()
-
-    output_list = []
-    file_list = [string.name for string in in_files]
-
-    in_colnames_list_first = in_colnames[0]
-
-    docs = list(data[in_colnames_list_first].str.lower())
-    
-    if anonymise_drop == "Yes":
-        progress(0.1, desc= "Anonymising data")
-        anon_tic = time.perf_counter()
-        
-        data_anon_col, anonymisation_success = anon.anonymise_script(data, in_colnames_list_first, anon_strat="replace")
-        data[in_colnames_list_first] = data_anon_col[in_colnames_list_first]
-        anonymise_data_name = data_file_name_no_ext + "_anonymised_" + today_rev +  ".csv"
-        data.to_csv(anonymise_data_name)
-        output_list.append(anonymise_data_name)
-
-        print(anonymisation_success)
-
-        anon_toc = time.perf_counter()
-        time_out = f"Anonymising text took {anon_toc - anon_tic:0.1f} seconds"
-
-    # Check if embeddings are being loaded in 
-    progress(0.2, desc= "Loading/creating embeddings")
-
-    print("Low resource mode: ", low_resource_mode)
-
-    if low_resource_mode == "No":
-        print("Using high resource BGE transformer model")
-
-        embedding_model = SentenceTransformer(embeddings_name)
-
-        # Use of Jina now superseded by BGE, keeping this code just in case I consider reverting one day
-        #try:
-        #embedding_model = AutoModel.from_pretrained(embeddings_name, revision = revision_choice, trust_remote_code=True,device_map="auto") # For Jina
-        #except:
-        #     embedding_model = AutoModel.from_pretrained(embeddings_name)#, revision = revision_choice, trust_remote_code=True, device_map="auto", use_auth_token=os.environ["HF_TOKEN"])
-        #tokenizer = AutoTokenizer.from_pretrained(embeddings_name)
-        #embedding_model_pipe = pipeline("feature-extraction", model=embedding_model, tokenizer=tokenizer)
-
-        # UMAP model uses Bertopic defaults
-        umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', low_memory=False, random_state=random_seed)
-
-    elif low_resource_mode == "Yes":
-        print("Choosing low resource TF-IDF model.")
-
-        embedding_model_pipe = make_pipeline(
-                TfidfVectorizer(),
-                TruncatedSVD(100) # 100 # To be compatible with zero shot, this needs to be lower than number of suggested topics
-                )
-        embedding_model = embedding_model_pipe
-
-        umap_model = TruncatedSVD(n_components=5, random_state=random_seed)
-
-    embeddings_out = make_or_load_embeddings(docs, file_list, embeddings_out, embedding_model, embeddings_super_compress, low_resource_mode)
-
-    vectoriser_model = CountVectorizer(stop_words="english", ngram_range=(1, 2), min_df=0.1)
-
-    # Representation model not currently used in this function
-    #print("Create Keybert-like topic representations by default")
-    #from funcs.representation_model import create_representation_model, llm_config, chosen_start_tag
-    #representation_model = create_representation_model("No", llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode)  
-
- 
-    progress(0.3, desc= "Embeddings loaded. Creating BERTopic model")
-
-    if not candidate_topics:
-        
-        topic_model = BERTopic( embedding_model=embedding_model, #embedding_model_pipe, #for Jina
-                                vectorizer_model=vectoriser_model,
-                                umap_model=umap_model,
-                                min_topic_size = min_docs_slider,
-                                nr_topics = max_topics_slider,
-                                calculate_probabilities=calc_probs,
-                                #representation_model=representation_model,
-                                verbose = True)
-
-        assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
-
-        #print(assigned_topics)
-
-        # Replace original labels with Keybert labels
-        #if "KeyBERT" in topic_model.get_topic_info().columns:
-        #    keybert_labels = [f"{i+1}: {', '.join(entry[:5])}" for i, entry in enumerate(topic_model.get_topics(full=True)["KeyBERT"].values())]
-        #    topic_model.set_topic_labels(keybert_labels)
-
-
-    # Do this if you have pre-defined topics
-    else:
-        if low_resource_mode == "Yes":
-            error_message = "Zero shot topic modelling currently not compatible with low-resource embeddings. Please change this option to 'No' on the options tab and retry."
-            print(error_message)
-
-            return error_message, output_list, embeddings_out, data_file_name_no_ext, None, docs
-
-        zero_shot_topics = read_file(candidate_topics.name)
-        zero_shot_topics_lower = list(zero_shot_topics.iloc[:, 0].str.lower())
-
- 
-
-        topic_model = BERTopic( embedding_model=embedding_model, #embedding_model_pipe, # for Jina
-                                vectorizer_model=vectoriser_model,
-                                umap_model=umap_model,
-                                min_topic_size = min_docs_slider,
-                                nr_topics = max_topics_slider,
-                                zeroshot_topic_list = zero_shot_topics_lower,
-                                zeroshot_min_similarity = zero_shot_similarity, # 0.7
-                                calculate_probabilities=calc_probs,
-                                #representation_model=representation_model,
-                                verbose = True)
-        
-        assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
-
-        # For some reason, zero topic modelling exports assigned topics as a np.array instead of a list. Converting it back here.
-        if isinstance(assigned_topics, np.ndarray):
-            assigned_topics = assigned_topics.tolist()
-        #print(assigned_topics.tolist())
-
-         # Zero shot modelling is a model merge, which wipes the c_tf_idf part of the resulting model completely. To get hierarchical modelling to work, we need to recreate this part of the model with the CountVectorizer options used to create the initial model. Since with zero shot, we are merging two models that have exactly the same set of documents, the vocubulary should be the same, and so recreating the cf_tf_idf component in this way shouldn't be a problem. Discussion here, and below based on Maarten's suggested code: https://github.com/MaartenGr/BERTopic/issues/1700
-
-        doc_dets = topic_model.get_document_info(docs)
-
-        documents_per_topic = doc_dets.groupby(['Topic'], as_index=False).agg({'Document': ' '.join})
-
-        # Assign CountVectorizer to merged model
-
-        topic_model.vectorizer_model = vectoriser_model
-
-        # Re-calculate c-TF-IDF
-        c_tf_idf, _ = topic_model._c_tf_idf(documents_per_topic)
-        topic_model.c_tf_idf_ = c_tf_idf
-
-        # Replace original labels with Keybert labels
-        #if "KeyBERT" in topic_model.get_topic_info().columns:
-        #    print(topic_model.get_topics(full=True)["KeyBERT"].values())
-        #    keybert_labels = [f"{i+1}: {', '.join(entry[:5])}" for i, entry in enumerate(topic_model.get_topics(full=True)["KeyBERT"].values())]
-        #    topic_model.set_topic_labels(keybert_labels) 
-
-    if not assigned_topics:
-    # Handle the empty array case
-        return "No topics found.", output_list, embeddings_out, data_file_name_no_ext, topic_model, docs
-    
-    else: 
-        print("Topic model created.")
-
-    # Replace current topic labels if new ones loaded in
-    if not custom_labels_df.empty:
-        #custom_label_list = list(custom_labels_df.iloc[:,0])
-        custom_label_list = [label.replace("\n", "") for label in custom_labels_df.iloc[:,0]]
-
-        topic_model.set_topic_labels(custom_label_list)
-        #topic_model.update_topics(docs, topics=assigned_topics, vectorizer_model=vectoriser_model)
-
-        
-    print("Custom topics: ", topic_model.custom_labels_)
-
-    # Outputs
-    output_list, output_text = save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, save_topic_model)
-
-     # If you want to save your embedding files
-    if return_intermediate_files == "Yes":
-        print("Saving embeddings to file")
-        if low_resource_mode == "Yes":
-            embeddings_file_name = data_file_name_no_ext + '_' + 'tfidf_embeddings.npz'
-        else:
-            if embeddings_super_compress == "No":
-                embeddings_file_name = data_file_name_no_ext + '_' + 'bge_embeddings.npz'
-            else:
-                embeddings_file_name = data_file_name_no_ext + '_' + 'bge_embeddings_compress.npz'
-
-        np.savez_compressed(embeddings_file_name, embeddings_out)
-
-        output_list.append(embeddings_file_name)
-
-    all_toc = time.perf_counter()
-    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds."
-    print(time_out)
-
-    return output_text, output_list, embeddings_out, data_file_name_no_ext, topic_model, docs
-
-def reduce_outliers(topic_model, docs, embeddings_out, data_file_name_no_ext, save_topic_model, progress=gr.Progress(track_tqdm=True)):
-
-    progress(0, desc= "Preparing data")
-
-    output_list = []
-
-    all_tic = time.perf_counter()
-
-    assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
-
-    if isinstance(assigned_topics, np.ndarray):
-        assigned_topics = assigned_topics.tolist()
-
-    #progress(0.2, desc= "Loading in representation model")
-    #print("Create LLM topic labels:", create_llm_topic_labels)
-    #from funcs.representation_model import create_representation_model, llm_config, chosen_start_tag
-    #representation_model = create_representation_model(create_llm_topic_labels, llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode)  
-
-    # Reduce outliers if required, then update representation
-    progress(0.2, desc= "Reducing outliers")
-    print("Reducing outliers.")
-    # Calculate the c-TF-IDF representation for each outlier document and find the best matching c-TF-IDF topic representation using cosine similarity.
-    assigned_topics = topic_model.reduce_outliers(docs, assigned_topics, strategy="embeddings")
-    # Then, update the topics to the ones that considered the new data
-
-    print("Finished reducing outliers.")
-
-    progress(0.7, desc= "Replacing topic names with LLMs if necessary")
-    #print("Create LLM topic labels:", "No")
-    #vectoriser_model = CountVectorizer(stop_words="english", ngram_range=(1, 2), min_df=0.1)
-    #representation_model = create_representation_model("No", llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode) 
-    #topic_model.update_topics(docs, topics=assigned_topics, vectorizer_model=vectoriser_model, representation_model=representation_model)
-
-    topic_dets = topic_model.get_topic_info()
-
-    # Replace original labels with LLM labels
-    if "LLM" in topic_model.get_topic_info().columns:
-        llm_labels = [label[0][0].split("\n")[0] for label in topic_model.get_topics(full=True)["LLM"].values()]
-        topic_model.set_topic_labels(llm_labels)
-    else:
-        topic_model.set_topic_labels(list(topic_dets["Name"]))
-
-    # Outputs   
-    progress(0.9, desc= "Saving to file")
-    output_list, output_text = save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, save_topic_model)
-
-    all_toc = time.perf_counter()
-    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds"
-    print(time_out)
-    
-    return output_text, output_list, topic_model
-
-def represent_topics(topic_model, docs, embeddings_out, data_file_name_no_ext, low_resource_mode, save_topic_model, progress=gr.Progress(track_tqdm=True)):
-    #from funcs.prompts import capybara_prompt, capybara_start, open_hermes_prompt, open_hermes_start, stablelm_prompt, stablelm_start
-    from funcs.representation_model import create_representation_model, llm_config, chosen_start_tag
-
-    output_list = []
-
-    all_tic = time.perf_counter()
-
-    vectoriser_model = CountVectorizer(stop_words="english", ngram_range=(1, 2), min_df=0.1)
-
-    assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
-
-    topic_dets = topic_model.get_topic_info()
-
-    progress(0.1, desc= "Loading LLM model")
-    print("Create LLM topic labels:", "Yes")
-    representation_model = create_representation_model("Yes", llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode)  
-
-    topic_model.update_topics(docs, topics=assigned_topics, vectorizer_model=vectoriser_model, representation_model=representation_model)
-
-    # Replace original labels with LLM labels
-    if "LLM" in topic_model.get_topic_info().columns:
-        llm_labels = [label[0][0].split("\n")[0] for label in topic_model.get_topics(full=True)["LLM"].values()]
-        topic_model.set_topic_labels(llm_labels)
-
-        label_list_file_name = data_file_name_no_ext + '_llm_topic_list_' + today_rev + '.csv'
-
-        llm_labels_df = pd.DataFrame(data={"Label":llm_labels})
-        llm_labels_df.to_csv(label_list_file_name, index=None)
-        #with open(label_list_file_name, 'w') as file:
-        #    file.write(f"Label\n")
-        #    for item in llm_labels:
-        #        file.write(f"{item}\n")
-        output_list.append(label_list_file_name)
-    else:
-        topic_model.set_topic_labels(list(topic_dets["Name"]))
-
-    # Outputs
-    progress(0.8, desc= "Saving outputs")
-    output_list, output_text = save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, save_topic_model)
-
-    all_toc = time.perf_counter()
-    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds"
-    print(time_out)
-
-    return output_text, output_list, topic_model
-
-def visualise_topics(topic_model, data, data_file_name_no_ext, low_resource_mode,  embeddings_out, in_label, in_colnames, legend_label, sample_prop, visualisation_type_radio, random_seed, progress=gr.Progress()):
-
-    progress(0, desc= "Preparing data for visualisation")
-
-    output_list = []
-    vis_tic = time.perf_counter()
-
-    from funcs.bertopic_vis_documents import visualize_documents_custom, visualize_hierarchical_documents_custom, visualize_barchart_custom
-
-    if not visualisation_type_radio:
-        return "Please choose a visualisation type above.", output_list, None, None
-
-    # Get topic labels
-    if in_label:
-       in_label_list_first = in_label[0]
-    else:
-       return "Label column not found. Please enter this above.", output_list, None, None
-    
-    # Get docs
-    if in_colnames:
-        in_colnames_list_first = in_colnames[0]
-    else:
-        return "Label column not found. Please enter this on the data load tab.", output_list, None, None
-    
-    docs = list(data[in_colnames_list_first].str.lower())
-
-    # Make sure format of input series is good
-    data[in_label_list_first] = data[in_label_list_first].fillna('').astype(str)
-    label_list = list(data[in_label_list_first])
-
-    topic_dets = topic_model.get_topic_info()
-
-    # Replace original labels with another representation if specified
-    if legend_label:
-        topic_dets = topic_model.get_topics(full=True)
-        if legend_label in topic_dets:
-            labels = [topic_dets[legend_label].values()]
-            labels = [str(v) for v in labels]
-            topic_model.set_topic_labels(labels)
-
-    # Pre-reduce embeddings for visualisation purposes
-    if low_resource_mode == "No":
-        reduced_embeddings = UMAP(n_neighbors=15, n_components=2, min_dist=0.0, metric='cosine', random_state=random_seed).fit_transform(embeddings_out)
-    else:
-        reduced_embeddings = TruncatedSVD(2, random_state=random_seed).fit_transform(embeddings_out)
-
-    progress(0.5, desc= "Creating visualisation (this can take a while)")
-    # Visualise the topics:
-    
-    print("Creating visualisation")
-
-    # "Topic document graph", "Hierarchical view"
-
-    if visualisation_type_radio == "Topic document graph":
-        topics_vis = visualize_documents_custom(topic_model, docs, hover_labels = label_list, reduced_embeddings=reduced_embeddings, hide_annotations=True, hide_document_hover=False, custom_labels=True, sample = sample_prop, width= 1200, height = 750)
-
-        topics_vis_name = data_file_name_no_ext + '_' + 'vis_topic_docs_' + today_rev + '.html'
-        topics_vis.write_html(topics_vis_name)
-        output_list.append(topics_vis_name)
-
-        topics_vis_2 = visualize_barchart_custom(topic_model, top_n_topics = 12, custom_labels=True, width= 300, height = 250)
-
-        topics_vis_2_name = data_file_name_no_ext + '_' + 'vis_barchart_' + today_rev + '.html'
-        topics_vis_2.write_html(topics_vis_2_name)
-        output_list.append(topics_vis_2_name)
-
-    elif visualisation_type_radio == "Hierarchical view":
-       
-        # Check that original topics are retained
-        #new_topic_dets = topic_model.get_topic_info()
-        #new_topic_dets.to_csv("new_topic_dets.csv")
-
-        #from funcs.bertopic_hierarchical_topics_mod import hierarchical_topics_mod
-
-        hierarchical_topics = topic_model.hierarchical_topics(docs)
-
-        # Save new hierarchical topic model to file
-        hierarchical_topics_name = data_file_name_no_ext + '_' + 'vis_hierarchy_topics_' + today_rev + '.csv'
-        hierarchical_topics.to_csv(hierarchical_topics_name)
-        output_list.append(hierarchical_topics_name)
-
-        #hierarchical_topics = hierarchical_topics_mod(topic_model, docs)
-        topics_vis = visualize_hierarchical_documents_custom(topic_model, docs, label_list, hierarchical_topics, reduced_embeddings=reduced_embeddings, sample = sample_prop, hide_document_hover= False, custom_labels=True, width= 1200, height = 750)
-        #topics_vis = topic_model.visualize_hierarchical_documents(docs, hierarchical_topics, reduced_embeddings=reduced_embeddings, sample = sample_prop, hide_document_hover= False, custom_labels=True, width= 1200, height = 750)
-        topics_vis_2 = topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics, width= 1200, height = 750)
-
-        topics_vis_name = data_file_name_no_ext + '_' + 'vis_hierarchy_topic_doc_' + today_rev + '.html'
-        topics_vis.write_html(topics_vis_name)
-        output_list.append(topics_vis_name)
-
-        topics_vis_2_name = data_file_name_no_ext + '_' + 'vis_hierarchy_' + today_rev + '.html'
-        topics_vis_2.write_html(topics_vis_2_name)
-        output_list.append(topics_vis_2_name)
-
-    all_toc = time.perf_counter()
-    time_out = f"Creating visualisation took {all_toc - vis_tic:0.1f} seconds"
-    print(time_out)
-
-    return time_out, output_list, topics_vis, topics_vis_2
-
-def save_as_pytorch_model(topic_model, data_file_name_no_ext , progress=gr.Progress()):
-
-    if not topic_model:
-        return "No Pytorch model found.", None
-
-    progress(0, desc= "Saving topic model in Pytorch format")
-
-    output_list = []
-
-
-    topic_model_save_name_folder = "output_model/" + data_file_name_no_ext + "_topics_" + today_rev# + ".safetensors"
-    topic_model_save_name_zip = topic_model_save_name_folder + ".zip"
-
-    # Clear folder before replacing files
-    delete_files_in_folder(topic_model_save_name_folder)
-
-    topic_model.save(topic_model_save_name_folder, serialization='pytorch', save_embedding_model=True, save_ctfidf=False)
-
-    # Zip file example
-    
-    zip_folder(topic_model_save_name_folder, topic_model_save_name_zip)
-    output_list.append(topic_model_save_name_zip)
-
-    return "Model saved in Pytorch format.", output_list
 
 # Gradio app
 
@@ -524,6 +21,7 @@ with block:
     docs_state = gr.State()
     data_file_name_no_ext_state = gr.State()
     label_list_state = gr.State(pd.DataFrame())
+    vectoriser_state = gr.State(CountVectorizer(stop_words="english", ngram_range=(1, 2), min_df=0.1, max_df=0.95))
  
     gr.Markdown(
     """
@@ -539,6 +37,13 @@ with block:
             with gr.Row():
                 in_colnames = gr.Dropdown(choices=["Choose a column"], multiselect = True, label="Select column to find topics (first will be chosen if multiple selected).")                
 
+        with gr.Accordion("Clean data", open = False):
+            with gr.Row():
+                clean_text = gr.Dropdown(value = "No", choices=["Yes", "No"], multiselect=False, label="Clean data - remove html, numbers with > 2 digits, emails, postcodes (UK).")
+                drop_duplicate_text = gr.Dropdown(value = "No", choices=["Yes", "No"], multiselect=False, label="Remove duplicate text, drop < 10 char strings. May make previous embedding files incompatible due to differing lengths.")
+                anonymise_drop = gr.Dropdown(value = "No", choices=["Yes", "No"], multiselect=False, label="Anonymise data on file load. Personal details are redacted - not 100% effective!")                
+            clean_btn = gr.Button("Clean data")
+
         with gr.Accordion("I have my own list of topics (zero shot topic modelling).", open = False):
             candidate_topics = gr.File(label="Input topics from file (csv). File should have at least one column with a header and topic keywords in cells below. Topics will be taken from the first column of the file. Currently not compatible with low-resource embeddings.")
             zero_shot_similarity = gr.Slider(minimum = 0.5, maximum = 1, value = 0.65, step = 0.001, label = "Minimum similarity value for document to be assigned to zero-shot topic.")
@@ -548,20 +53,20 @@ with block:
             max_topics_slider = gr.Slider(minimum = 2, maximum = 500, value = 10, step = 1, label = "Maximum number of topics")
 
         with gr.Row():
-            topics_btn = gr.Button("Extract topics")
+            topics_btn = gr.Button("Extract topics", variant="primary")
             
         with gr.Row():
             output_single_text = gr.Textbox(label="Output topics")
             output_file = gr.File(label="Output file")
 
         with gr.Accordion("Post processing options.", open = True):
+            with gr.Row():
+                representation_type =  gr.Dropdown(label = "Method for generating new topic labels", value="Default", choices=["Default", "MMR", "KeyBERT", "LLM"]) 
+                represent_llm_btn = gr.Button("Change topic labels")
             with gr.Row():
                 reduce_outliers_btn = gr.Button("Reduce outliers")
-                represent_llm_btn = gr.Button("Generate topic labels with LLMs")
                 save_pytorch_btn = gr.Button("Save model in Pytorch format")
-
-        #logs = gr.Textbox(label="Processing logs.")
-
+                
     with gr.Tab("Visualise"):
         with gr.Row():
             visualisation_type_radio = gr.Radio(label="Visualisation type", choices=["Topic document graph", "Hierarchical view"])
@@ -575,40 +80,39 @@ with block:
             out_plot_file = gr.File(label="Output plots to file", file_count="multiple")
         plot = gr.Plot(label="Visualise your topics here.")
         plot_2 = gr.Plot(label="Visualise your topics here.")
-
     
     with gr.Tab("Options"):
         with gr.Accordion("Data load and processing options", open = True):
             with gr.Row():
-                anonymise_drop = gr.Dropdown(value = "No", choices=["Yes", "No"], multiselect=False, label="Anonymise data on file load. Names and other details are replaced with tags e.g. '<person>'.")
-                embedding_super_compress = gr.Dropdown(label = "Round embeddings to three dp for smaller files with less accuracy.", value="No", choices=["Yes", "No"])
                 seed_number = gr.Number(label="Random seed to use for dimensionality reduction.", minimum=0, step=1, value=42, precision=0)
-                calc_probs = gr.Dropdown(label="Calculate all topic probabilities (i.e. a separate document prob. value for each topic)", value="No", choices=["Yes", "No"])
+                calc_probs = gr.Dropdown(label="Calculate all topic probabilities", value="No", choices=["Yes", "No"])
             with gr.Row():
                 low_resource_mode_opt = gr.Dropdown(label = "Use low resource embeddings and processing.", value="No", choices=["Yes", "No"])
+                embedding_super_compress = gr.Dropdown(label = "Round embeddings to three dp for smaller files with less accuracy.", value="No", choices=["Yes", "No"])
+            with gr.Row(): 
                 return_intermediate_files = gr.Dropdown(label = "Return intermediate processing files from file preparation.", value="Yes", choices=["Yes", "No"])
-                save_topic_model = gr.Dropdown(label = "Save topic model to file.", value="Yes", choices=["Yes", "No"])
+                save_topic_model = gr.Dropdown(label = "Save topic model to BERTopic format pkl file.", value="No", choices=["Yes", "No"])
 
-    # Update column names dropdown when file uploaded
+    # Load in data. Update column names dropdown when file uploaded
     in_files.upload(fn=initial_file_load, inputs=[in_files], outputs=[in_colnames, in_label, data_state, output_single_text, topic_model_state, embeddings_state, data_file_name_no_ext_state, label_list_state])    
     in_colnames.change(dummy_function, in_colnames, None)
 
-    topics_btn.click(fn=extract_topics, inputs=[data_state, in_files, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, data_file_name_no_ext_state, label_list_state, anonymise_drop, return_intermediate_files, embedding_super_compress, low_resource_mode_opt, save_topic_model, embeddings_state, zero_shot_similarity, seed_number, calc_probs], outputs=[output_single_text, output_file, embeddings_state, data_file_name_no_ext_state, topic_model_state, docs_state], api_name="topics")
+    # Clean data
+    clean_btn.click(fn=pre_clean, inputs=[data_state, in_colnames, data_file_name_no_ext_state, clean_text, drop_duplicate_text, anonymise_drop], outputs=[output_single_text, output_file, data_state, data_file_name_no_ext_state], api_name="clean")
+
+    # Extract topics
+    topics_btn.click(fn=extract_topics, inputs=[data_state, in_files, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, data_file_name_no_ext_state, label_list_state, return_intermediate_files, embedding_super_compress, low_resource_mode_opt, save_topic_model, embeddings_state, zero_shot_similarity, seed_number, calc_probs, vectoriser_state], outputs=[output_single_text, output_file, embeddings_state, data_file_name_no_ext_state, topic_model_state, docs_state, vectoriser_state], api_name="topics")
 
+    # Reduce outliers
     reduce_outliers_btn.click(fn=reduce_outliers, inputs=[topic_model_state, docs_state, embeddings_state, data_file_name_no_ext_state, save_topic_model], outputs=[output_single_text, output_file, topic_model_state], api_name="reduce_outliers")
 
-    represent_llm_btn.click(fn=represent_topics, inputs=[topic_model_state, docs_state, embeddings_state, data_file_name_no_ext_state, low_resource_mode_opt, save_topic_model], outputs=[output_single_text, output_file, topic_model_state], api_name="represent_llm")
+    # Re-represent topic labels
+    represent_llm_btn.click(fn=represent_topics, inputs=[topic_model_state, docs_state, data_file_name_no_ext_state, low_resource_mode_opt, save_topic_model, representation_type, vectoriser_state], outputs=[output_single_text, output_file, topic_model_state], api_name="represent_llm")
 
+    # Save in Pytorch format
     save_pytorch_btn.click(fn=save_as_pytorch_model, inputs=[topic_model_state, data_file_name_no_ext_state], outputs=[output_single_text, output_file])
 
+    # Visualise topics
     plot_btn.click(fn=visualise_topics, inputs=[topic_model_state, data_state, data_file_name_no_ext_state, low_resource_mode_opt, embeddings_state, in_label, in_colnames, legend_label, sample_slide, visualisation_type_radio, seed_number], outputs=[vis_output_single_text, out_plot_file, plot, plot_2], api_name="plot")
 
-    #block.load(read_logs, None, logs, every=5)
-
-block.queue().launch(debug=True)#, server_name="0.0.0.0", ssl_verify=False, server_port=7860)
-
-
-
-
-
-
+block.queue().launch(debug=True)#, server_name="0.0.0.0", ssl_verify=False, server_port=7860)

funcs/anonymiser.py

@@ -1,5 +1,6 @@
 from spacy.cli import download
 import spacy
+from funcs.presidio_analyzer_custom import analyze_dict
 spacy.prefer_gpu()
 
 def spacy_model_installed(model_name):
@@ -21,7 +22,7 @@ def spacy_model_installed(model_name):
 model_name = "en_core_web_sm"
 spacy_model_installed(model_name)
 
-spacy.load(model_name)
+#spacy.load(model_name)
 # Need to overwrite version of gradio present in Huggingface spaces as it doesn't have like buttons/avatars (Oct 2023)
 #os.system("pip uninstall -y gradio")
 #os.system("pip install gradio==3.50.0")
@@ -33,11 +34,10 @@ import base64
 import time
 
 import pandas as pd
-import gradio as gr
 
 from faker import Faker
 
-from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine
+from presidio_analyzer import AnalyzerEngine, BatchAnalyzerEngine, PatternRecognizer
 from presidio_anonymizer import AnonymizerEngine, BatchAnonymizerEngine
 from presidio_anonymizer.entities import OperatorConfig
 
@@ -159,10 +159,24 @@ def read_file(filename):
 
 def anonymise_script(df, chosen_col, anon_strat):
 
+    #print(df.shape)
+
+    #df_chosen_col_mask = (df[chosen_col].isnull()) | (df[chosen_col].str.strip() == "")
+    #print("Length of input series blank at start is: ", df_chosen_col_mask.value_counts())
+
     # DataFrame to dict
     df_dict = pd.DataFrame(data={chosen_col:df[chosen_col].astype(str)}).to_dict(orient="list")
 
+    
+
     analyzer = AnalyzerEngine()
+
+    # Add titles to analyzer list
+    titles_recognizer = PatternRecognizer(supported_entity="TITLE",
+                                      deny_list=["Mr","Mrs","Miss", "Ms", "mr", "mrs", "miss", "ms"])
+
+    analyzer.registry.add_recognizer(titles_recognizer)
+
     batch_analyzer = BatchAnalyzerEngine(analyzer_engine=analyzer)
 
     anonymizer = AnonymizerEngine()
@@ -171,12 +185,13 @@ def anonymise_script(df, chosen_col, anon_strat):
 
     print("Identifying personal data")
     analyse_tic = time.perf_counter()
-    analyzer_results = batch_analyzer.analyze_dict(df_dict, language="en")
+    #analyzer_results = batch_analyzer.analyze_dict(df_dict, language="en")
+    analyzer_results = analyze_dict(batch_analyzer, df_dict, language="en")
     #print(analyzer_results)
     analyzer_results = list(analyzer_results)
 
     analyse_toc = time.perf_counter()
-    analyse_time_out = f"Cleaning the text took {analyse_toc - analyse_tic:0.1f} seconds."
+    analyse_time_out = f"Analysing the text took {analyse_toc - analyse_tic:0.1f} seconds."
     print(analyse_time_out)
 
     # Generate a 128-bit AES key. Then encode the key using base64 to get a string representation
@@ -206,15 +221,34 @@ def anonymise_script(df, chosen_col, anon_strat):
     if anon_strat == "encrypt": chosen_mask_config = people_encrypt_config
     elif anon_strat == "fake_first_name": chosen_mask_config = fake_first_name_config
 
-    # I think in general people will want to keep date / times
-    keep_date_config = eval('{"DATE_TIME": OperatorConfig("keep")}')
+    # I think in general people will want to keep date / times - NOT FOR TOPIC MODELLING
+    #keep_date_config = eval('{"DATE_TIME": OperatorConfig("keep")}')
 
-    combined_config = {**chosen_mask_config, **keep_date_config}
+    #combined_config = {**chosen_mask_config, **keep_date_config}
+    combined_config = {**chosen_mask_config}#, **keep_date_config}
     combined_config
 
+    print("Anonymising personal data")
     anonymizer_results = batch_anonymizer.anonymize_dict(analyzer_results, operators=combined_config)
 
-    scrubbed_df = pd.DataFrame(anonymizer_results)
+    #print(anonymizer_results)
+
+    scrubbed_df = pd.DataFrame(data={chosen_col:anonymizer_results[chosen_col]})
+
+    scrubbed_series = scrubbed_df[chosen_col]
+
+    #print(scrubbed_series[0:6])
+
+    #print("Length of output series is: ", len(scrubbed_series))
+    #print("Length of input series at end is: ", len(df[chosen_col]))
+
+    
+    #scrubbed_values_mask = (scrubbed_series.isnull()) | (scrubbed_series.str.strip() == "")
+    #df_chosen_col_mask = (df[chosen_col].isnull()) | (df[chosen_col].str.strip() == "")
+
+    #print("Length of input series blank at end is: ", df_chosen_col_mask.value_counts())
+    #print("Length of output series blank is: ", scrubbed_values_mask.value_counts())
+    
 
     # Create reporting message
     out_message = "Successfully anonymised"
@@ -222,7 +256,7 @@ def anonymise_script(df, chosen_col, anon_strat):
     if anon_strat == "encrypt":
         out_message = out_message + ". Your decryption key is " + key_string + "."
     
-    return scrubbed_df, out_message
+    return scrubbed_series, out_message
 
 def do_anonymise(in_file, anon_strat, chosen_cols):
     

funcs/clean_funcs.py

@@ -0,0 +1,107 @@
+import re
+import string
+import polars as pl
+import gradio as gr
+
+# Adding custom words to the stopwords
+custom_words = []
+my_stop_words = custom_words
+
+# #### Some of my cleaning functions
+email_start_pattern_regex = r'.*importance:|.*subject:'
+email_end_pattern_regex = r'kind regards.*|many thanks.*|sincerely.*'
+html_pattern_regex = r'<.*?>|&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-f]{1,6});|\xa0|&nbsp;'
+email_pattern_regex = r'\S*@\S*\s?'
+num_pattern_regex = r'[0-9]+'
+nums_three_more_regex = r'\b[0-9]{3,}\b|\b[0-9]+\s[0-9]+\b'
+postcode_pattern_regex = r'(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9][A-Z]{2})|((GIR ?0A{2})\b$)|(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]? ?[0-9]{1}?)$)|(\b(?:[A-Z][A-HJ-Y]?[0-9][0-9A-Z]?)\b$)'
+warning_pattern_regex = r'caution: this email originated from outside of the organization. do not click links or open attachments unless you recognize the sender and know the content is safe.'
+nbsp_pattern_regex = r'&nbsp;'
+
+# Pre-compiling the regular expressions for efficiency (not actually used)
+# email_start_pattern = re.compile(email_start_pattern_regex)
+# email_end_pattern = re.compile(email_end_pattern_regex)
+# html_pattern = re.compile(html_pattern_regex)
+# email_pattern = re.compile(email_end_pattern_regex)
+# num_pattern = re.compile(num_pattern_regex)
+# nums_three_more_regex_pattern = re.compile(nums_three_more_regex)
+# postcode_pattern = re.compile(postcode_pattern_regex)
+# warning_pattern = re.compile(warning_pattern_regex)
+# nbsp_pattern = re.compile(nbsp_pattern_regex)
+
+def initial_clean(texts , progress=gr.Progress()):
+    texts = pl.Series(texts)
+    text = texts.str.replace_all(html_pattern_regex, '')
+    text = text.str.replace_all(email_pattern_regex, '')
+    text = text.str.replace_all(nums_three_more_regex, '')
+    text = text.str.replace_all(postcode_pattern_regex, '')
+
+    text = text.to_list()
+    
+    return text
+
+def remove_hyphens(text_text):
+    return re.sub(r'(\w+)-(\w+)-?(\w)?', r'\1 \2 \3', text_text)
+
+
+def remove_characters_after_tokenization(tokens):
+    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))
+    filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens])
+    return filtered_tokens
+
+def convert_to_lowercase(tokens):
+    return [token.lower() for token in tokens if token.isalpha()]
+
+def remove_short_tokens(tokens):
+    return [token for token in tokens if len(token) > 3]
+
+
+def remove_dups_text(data_samples_ready, data_samples_clean, data_samples):
+   # Identify duplicates in the data: https://stackoverflow.com/questions/44191465/efficiently-identify-duplicates-in-large-list-500-000
+    # Only identifies the second duplicate
+
+    seen = set()
+    dups = []
+
+    for i, doi in enumerate(data_samples_ready):
+        if doi not in seen:
+            seen.add(doi)
+        else:
+            dups.append(i) 
+    #data_samples_ready[dupes[0:]]
+    
+    # To see a specific duplicated value you know the position of
+    #matching = [s for s in data_samples_ready if data_samples_ready[83] in s]
+    #matching
+    
+    # Remove duplicates only (keep first instance)
+    #data_samples_ready = list( dict.fromkeys(data_samples_ready) ) # This way would keep one version of the duplicates
+    
+    ### Remove all duplicates including original instance
+    
+    # Identify ALL duplicates including initial values
+    # https://stackoverflow.com/questions/11236006/identify-duplicate-values-in-a-list-in-python
+
+    from collections import defaultdict
+    D = defaultdict(list)
+    for i,item in enumerate(data_samples_ready):
+        D[item].append(i)
+    D = {k:v for k,v in D.items() if len(v)>1}
+    
+    # https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-a-list-of-lists
+    L = list(D.values())
+    flat_list_dups = [item for sublist in L for item in sublist]
+
+    # https://stackoverflow.com/questions/11303225/how-to-remove-multiple-indexes-from-a-list-at-the-same-time
+    for index in sorted(flat_list_dups, reverse=True):
+        del data_samples_ready[index]
+        del data_samples_clean[index]
+        del data_samples[index]
+    
+    # Remove blanks
+    data_samples_ready = [i for i in data_samples_ready if i]
+    data_samples_clean = [i for i in data_samples_clean if i]
+    data_samples = [i for i in data_samples if i]
+    
+    return data_samples_ready, data_samples_clean, flat_list_dups, data_samples
+

funcs/helper_functions.py

@@ -1,3 +1,4 @@
+import sys
 import os
 import zipfile
 import re
@@ -12,6 +13,30 @@ from datetime import datetime
 today = datetime.now().strftime("%d%m%Y")
 today_rev = datetime.now().strftime("%Y%m%d")
 
+# Log terminal output: https://github.com/gradio-app/gradio/issues/2362
+class Logger:
+    def __init__(self, filename):
+        self.terminal = sys.stdout
+        self.log = open(filename, "w")
+
+    def write(self, message):
+        self.terminal.write(message)
+        self.log.write(message)
+        
+    def flush(self):
+        self.terminal.flush()
+        self.log.flush()
+        
+    def isatty(self):
+        return False    
+
+sys.stdout = Logger("output.log")
+
+def read_logs():
+    sys.stdout.flush()
+    with open("output.log", "r") as f:
+        return f.read()
+
 
 def detect_file_type(filename):
     """Detect the file type based on its extension."""
@@ -189,7 +214,10 @@ def save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, sa
         doc_dets.to_csv(doc_det_output_name)
         output_list.append(doc_det_output_name)
 
-        topics_text_out_str = str(topic_dets["Name"])
+        if "CustomName" in topic_dets.columns:
+            topics_text_out_str = str(topic_dets["CustomName"])
+        else:
+            topics_text_out_str = str(topic_dets["Name"])
         output_text = "Topics: " + topics_text_out_str
     
         # Save topic model to file

funcs/presidio_analyzer_custom.py

@@ -0,0 +1,114 @@
+import gradio as gr
+from typing import List, Iterable, Dict, Union, Any, Optional, Iterator, Tuple
+from tqdm import tqdm
+
+from presidio_analyzer import DictAnalyzerResult, RecognizerResult, AnalyzerEngine
+from presidio_analyzer.nlp_engine import NlpArtifacts
+
+def analyze_iterator_custom(
+        self,
+        texts: Iterable[Union[str, bool, float, int]],
+        language: str,
+        list_length:int,
+        progress=gr.Progress(),
+        **kwargs,
+    ) -> List[List[RecognizerResult]]:
+        """
+        Analyze an iterable of strings.
+
+        :param texts: An list containing strings to be analyzed.
+        :param language: Input language
+        :param list_length: Length of the input list.
+        :param kwargs: Additional parameters for the `AnalyzerEngine.analyze` method.
+        """
+
+        # validate types
+        texts = self._validate_types(texts)
+
+        # Process the texts as batch for improved performance
+        nlp_artifacts_batch: Iterator[
+            Tuple[str, NlpArtifacts]
+        ] = self.analyzer_engine.nlp_engine.process_batch(
+            texts=texts, language=language
+        )
+
+        
+
+        list_results = []
+        for text, nlp_artifacts in progress.tqdm(nlp_artifacts_batch, total = list_length, desc = "Analysing text for personal information", unit = "rows"):
+            results = self.analyzer_engine.analyze(
+                text=str(text), nlp_artifacts=nlp_artifacts, language=language, **kwargs
+            )
+
+            list_results.append(results)
+
+        return list_results
+
+def analyze_dict(
+        self,
+        input_dict: Dict[str, Union[Any, Iterable[Any]]],
+        language: str,
+        keys_to_skip: Optional[List[str]] = None,
+        **kwargs,
+    ) -> Iterator[DictAnalyzerResult]:
+        """
+        Analyze a dictionary of keys (strings) and values/iterable of values.
+
+        Non-string values are returned as is.
+
+        :param input_dict: The input dictionary for analysis
+        :param language: Input language
+        :param keys_to_skip: Keys to ignore during analysis
+        :param kwargs: Additional keyword arguments
+        for the `AnalyzerEngine.analyze` method.
+        Use this to pass arguments to the analyze method,
+        such as `ad_hoc_recognizers`, `context`, `return_decision_process`.
+        See `AnalyzerEngine.analyze` for the full list.
+        """
+
+        context = []
+        if "context" in kwargs:
+            context = kwargs["context"]
+            del kwargs["context"]
+
+        if not keys_to_skip:
+            keys_to_skip = []
+
+            
+        for key, value in input_dict.items():
+            if not value or key in keys_to_skip:
+                yield DictAnalyzerResult(key=key, value=value, recognizer_results=[])
+                continue  # skip this key as requested
+
+            # Add the key as an additional context
+            specific_context = context[:]
+            specific_context.append(key)
+
+            if type(value) in (str, int, bool, float):
+                results: List[RecognizerResult] = self.analyzer_engine.analyze(
+                    text=str(value), language=language, context=[key], **kwargs
+                )
+            elif isinstance(value, dict):
+                new_keys_to_skip = self._get_nested_keys_to_skip(key, keys_to_skip)
+                results = self.analyze_dict(
+                    input_dict=value,
+                    language=language,
+                    context=specific_context,
+                    keys_to_skip=new_keys_to_skip,
+                    **kwargs,
+                )
+            elif isinstance(value, Iterable):
+                # Recursively iterate nested dicts
+                list_length = len(value)
+
+                results: List[List[RecognizerResult]] = analyze_iterator_custom(self,
+                    texts=value,
+                    language=language,
+                    context=specific_context,
+                    list_length=list_length,
+                    **kwargs,
+                )
+            else:
+                raise ValueError(f"type {type(value)} is unsupported.")
+
+            yield DictAnalyzerResult(key=key, value=value, recognizer_results=results)

funcs/representation_model.py

@@ -1,13 +1,11 @@
 import os
-#from ctransformers import AutoModelForCausalLM
-#from transformers import AutoTokenizer, pipeline
 from bertopic.representation import LlamaCPP
 from llama_cpp import Llama
 from pydantic import BaseModel
 import torch.cuda
-from huggingface_hub import hf_hub_download, HfFolder
+from huggingface_hub import hf_hub_download
 
-from bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, TextGeneration
+from bertopic.representation import KeyBERTInspired, MaximalMarginalRelevance, BaseRepresentation
 from funcs.prompts import capybara_prompt, capybara_start, open_hermes_prompt, open_hermes_start, stablelm_prompt, stablelm_start
 
 random_seed = 42
@@ -16,8 +14,6 @@ chosen_prompt = open_hermes_prompt # stablelm_prompt
 chosen_start_tag =  open_hermes_start # stablelm_start
 
 
-
-
 # Currently set n_gpu_layers to 0 even with cuda due to persistent bugs in implementation with cuda
 if torch.cuda.is_available():
     torch_device = "gpu"
@@ -46,7 +42,7 @@ reset: bool = True
 stream: bool = False
 n_threads: int = n_threads
 n_batch:int = 256
-n_ctx:int = 4096
+n_ctx:int = 8192 #4096. # Set to 8192 just to avoid any exceeded context window issues
 sample:bool = True
 trust_remote_code:bool =True
 
@@ -90,7 +86,9 @@ llm_config = LLamacppInitConfigGpu(last_n_tokens_size=last_n_tokens_size,
 # KeyBERT
 keybert = KeyBERTInspired(random_state=random_seed)
 # MMR
-mmr = MaximalMarginalRelevance(diversity=0.2)
+mmr = MaximalMarginalRelevance(diversity=0.5)
+
+base_rep = BaseRepresentation()
 
 # Find model file
 def find_model_file(hf_model_name, hf_model_file, search_folder):
@@ -135,15 +133,12 @@ def find_model_file(hf_model_name, hf_model_file, search_folder):
         return found_file
 
 
-def create_representation_model(create_llm_topic_labels, llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode):
+def create_representation_model(representation_type, llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode):
 
-    if create_llm_topic_labels == "Yes":
+    if representation_type == "LLM":
+        print("Generating LLM representation")
         # Use llama.cpp to load in model
 
-        # This was for testing on systems without a HF_HOME env variable
-        #os.unsetenv("HF_HOME") 
-
-        #if "HF_HOME" in os.environ:
         #    del os.environ["HF_HOME"]     
 
         # Check for HF_HOME environment variable and supply a default value if it's not found (typical location for huggingface models)
@@ -161,22 +156,27 @@ def create_representation_model(create_llm_topic_labels, llm_config, hf_model_na
 
         found_file = find_model_file(hf_model_name, hf_model_file,  hf_home_value)
 
-        llm = Llama(model_path=found_file, stop=chosen_start_tag, n_gpu_layers=llm_config.n_gpu_layers, n_ctx=llm_config.n_ctx) #**llm_config.model_dump())# 
+        llm = Llama(model_path=found_file, stop=chosen_start_tag, n_gpu_layers=llm_config.n_gpu_layers, n_ctx=llm_config.n_ctx, rope_freq_scale=0.5) #**llm_config.model_dump())# 
         #print(llm.n_gpu_layers)
         llm_model = LlamaCPP(llm, prompt=chosen_prompt)#, **gen_config.model_dump())
 
         # All representation models
         representation_model = {
-        "KeyBERT": keybert,
         "LLM": llm_model
         }
 
-    elif create_llm_topic_labels == "No":
-        if low_resource_mode == "Yes":
-            #representation_model = {"mmr": mmr}
-            representation_model = {"KeyBERT": keybert}
-        else:
-            representation_model = {"KeyBERT": keybert}
+    elif representation_type == "KeyBERT":
+        print("Generating KeyBERT representation")
+        #representation_model = {"mmr": mmr}
+        representation_model = {"KeyBERT": keybert}
+
+    elif representation_type == "MMR":
+        print("Generating MMR representation")
+        representation_model = {"MMR": mmr}
+
+    else:
+        print("Generating default representation type")
+        representation_model = {"Default":base_rep}
 
     # Deprecated example using CTransformers. This package is not really used anymore
     #model = AutoModelForCausalLM.from_pretrained('NousResearch/Nous-Capybara-7B-V1.9-GGUF', model_type='mistral', model_file='Capybara-7B-V1.9-Q5_K_M.gguf', hf=True, **vars(llm_config))

funcs/topic_core_funcs.py

@@ -0,0 +1,500 @@
+# Dendrograms will not work with the latest version of scipy (1.12.0), so installing the version prior to be safe
+#os.system("pip install scipy==1.11.4")
+
+import gradio as gr
+from datetime import datetime
+import pandas as pd
+import numpy as np
+import time
+from bertopic import BERTopic
+
+from funcs.clean_funcs import initial_clean
+from funcs.helper_functions import read_file, zip_folder, delete_files_in_folder, save_topic_outputs
+from funcs.embeddings import make_or_load_embeddings
+
+from sentence_transformers import SentenceTransformer
+from sklearn.pipeline import make_pipeline
+from sklearn.decomposition import TruncatedSVD
+from sklearn.feature_extraction.text import TfidfVectorizer
+import funcs.anonymiser as anon
+from umap import UMAP
+
+from torch import cuda, backends, version
+
+# Default seed, can be changed in number selection on options page
+random_seed = 42
+
+# Check for torch cuda
+# If you want to disable cuda for testing purposes
+#os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+
+print("Is CUDA enabled? ", cuda.is_available())
+print("Is a CUDA device available on this computer?", backends.cudnn.enabled)
+if cuda.is_available():
+    torch_device = "gpu"
+    print("Cuda version installed is: ", version.cuda)
+    low_resource_mode = "No"
+    #os.system("nvidia-smi")
+else: 
+    torch_device =  "cpu"
+    low_resource_mode = "Yes"
+
+print("Device used is: ", torch_device)
+
+today = datetime.now().strftime("%d%m%Y")
+today_rev = datetime.now().strftime("%Y%m%d")
+
+# Load embeddings
+embeddings_name = "BAAI/bge-small-en-v1.5" #"jinaai/jina-embeddings-v2-base-en"
+
+# LLM model used for representing topics
+hf_model_name =  'second-state/stablelm-2-zephyr-1.6b-GGUF' #'TheBloke/phi-2-orange-GGUF' #'NousResearch/Nous-Capybara-7B-V1.9-GGUF'
+hf_model_file =   'stablelm-2-zephyr-1_6b-Q5_K_M.gguf' # 'phi-2-orange.Q5_K_M.gguf' #'Capybara-7B-V1.9-Q5_K_M.gguf'
+
+def pre_clean(data, in_colnames, data_file_name_no_ext, clean_text, drop_duplicate_text, anonymise_drop, progress=gr.Progress(track_tqdm=True)):
+    
+    output_text = ""
+    output_list = []
+
+    progress(0, desc = "Cleaning data")
+
+    if not in_colnames:
+        error_message = "Please enter one column name to use for cleaning and finding topics."
+        print(error_message)
+        return error_message, None, data_file_name_no_ext, None, None
+
+    all_tic = time.perf_counter()
+
+    output_list = []
+    #file_list = [string.name for string in in_files]
+
+    in_colnames_list_first = in_colnames[0]
+
+    if clean_text == "Yes":
+        clean_tic = time.perf_counter()
+        print("Starting data clean.")
+
+        data_file_name_no_ext = data_file_name_no_ext + "_clean"
+
+        data[in_colnames_list_first] = initial_clean(data[in_colnames_list_first])
+
+        clean_toc = time.perf_counter()
+        clean_time_out = f"Cleaning the text took {clean_toc - clean_tic:0.1f} seconds."
+        print(clean_time_out)
+
+    if drop_duplicate_text == "Yes":
+        progress(0.3, desc= "Drop duplicates - remove short texts")
+
+        data_file_name_no_ext = data_file_name_no_ext + "_dedup"
+
+        #print("Removing duplicates and short entries from data")
+        #print("Data shape before: ", data.shape)
+        data[in_colnames_list_first] = data[in_colnames_list_first].str.strip()
+        data = data[data[in_colnames_list_first].str.len() >= 10]
+        data = data.drop_duplicates(subset = in_colnames_list_first).dropna(subset= in_colnames_list_first).reset_index()
+        
+        #print("Data shape after duplicate/null removal: ", data.shape)
+
+    if anonymise_drop == "Yes":
+        progress(0.6, desc= "Anonymising data")
+
+        data_file_name_no_ext = data_file_name_no_ext + "_anon"
+
+        anon_tic = time.perf_counter()
+        
+        data_anon_col, anonymisation_success = anon.anonymise_script(data, in_colnames_list_first, anon_strat="redact")
+
+        data[in_colnames_list_first] = data_anon_col
+
+        print(anonymisation_success)
+
+        anon_toc = time.perf_counter()
+        time_out = f"Anonymising text took {anon_toc - anon_tic:0.1f} seconds"
+
+    out_data_name = data_file_name_no_ext + "_" + today_rev +  ".csv"
+    data.to_csv(out_data_name)
+    output_list.append(out_data_name)
+
+    all_toc = time.perf_counter()
+    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds."
+    print(time_out)
+
+    output_text = "Data clean completed."
+    
+    return output_text, output_list, data, data_file_name_no_ext
+
+def extract_topics(data, in_files, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, data_file_name_no_ext, custom_labels_df, return_intermediate_files, embeddings_super_compress, low_resource_mode, save_topic_model, embeddings_out, zero_shot_similarity, random_seed, calc_probs, vectoriser_state, progress=gr.Progress(track_tqdm=True)):
+
+    all_tic = time.perf_counter()
+
+    progress(0, desc= "Loading data")
+
+    output_list = []
+    file_list = [string.name for string in in_files]
+
+    if calc_probs == "No":
+        calc_probs = False
+
+    elif calc_probs == "Yes":
+        print("Calculating all probabilities.")
+        calc_probs = True
+
+    if not in_colnames:
+        error_message = "Please enter one column name to use for cleaning and finding topics."
+        print(error_message)
+        return error_message, None, data_file_name_no_ext, embeddings_out, None, None
+
+    
+
+    in_colnames_list_first = in_colnames[0]
+
+    docs = list(data[in_colnames_list_first])
+
+    # Check if embeddings are being loaded in 
+    progress(0.2, desc= "Loading/creating embeddings")
+
+    print("Low resource mode: ", low_resource_mode)
+
+    if low_resource_mode == "No":
+        print("Using high resource BGE transformer model")
+
+        embedding_model = SentenceTransformer(embeddings_name)
+
+        # UMAP model uses Bertopic defaults
+        umap_model = UMAP(n_neighbors=15, n_components=5, min_dist=0.0, metric='cosine', low_memory=False, random_state=random_seed)
+
+    elif low_resource_mode == "Yes":
+        print("Choosing low resource TF-IDF model.")
+
+        embedding_model_pipe = make_pipeline(
+                TfidfVectorizer(),
+                TruncatedSVD(100)
+                )
+        embedding_model = embedding_model_pipe
+
+        umap_model = TruncatedSVD(n_components=5, random_state=random_seed)
+
+    embeddings_out = make_or_load_embeddings(docs, file_list, embeddings_out, embedding_model, embeddings_super_compress, low_resource_mode)
+
+    # This is saved as a Gradio state object
+    vectoriser_model = vectoriser_state
+ 
+    progress(0.3, desc= "Embeddings loaded. Creating BERTopic model")
+
+    fail_error_message = "Topic model creation failed. If you have a small dataset, try reducing minimum documents per topic."
+
+    if not candidate_topics:
+        
+        try:
+
+            topic_model = BERTopic( embedding_model=embedding_model,
+                                    vectorizer_model=vectoriser_model,
+                                    umap_model=umap_model,
+                                    min_topic_size = min_docs_slider,
+                                    nr_topics = max_topics_slider,
+                                    calculate_probabilities=calc_probs,
+                                    verbose = True)
+
+            assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
+
+        except:
+            print(fail_error_message)
+
+            return fail_error_message, output_list, embeddings_out, data_file_name_no_ext, None, docs, vectoriser_model
+    
+
+    # Do this if you have pre-defined topics
+    else:
+        if low_resource_mode == "Yes":
+            error_message = "Zero shot topic modelling currently not compatible with low-resource embeddings. Please change this option to 'No' on the options tab and retry."
+            print(error_message)
+
+            return error_message, output_list, embeddings_out, data_file_name_no_ext, None, docs
+
+        zero_shot_topics = read_file(candidate_topics.name)
+        zero_shot_topics_lower = list(zero_shot_topics.iloc[:, 0].str.lower())
+
+ 
+        try:
+            topic_model = BERTopic( embedding_model=embedding_model, #embedding_model_pipe, # for Jina
+                                    vectorizer_model=vectoriser_model,
+                                    umap_model=umap_model,
+                                    min_topic_size = min_docs_slider,
+                                    nr_topics = max_topics_slider,
+                                    zeroshot_topic_list = zero_shot_topics_lower,
+                                    zeroshot_min_similarity = zero_shot_similarity, # 0.7
+                                    calculate_probabilities=calc_probs,
+                                    verbose = True)
+            
+            assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
+
+        except:
+            print(fail_error_message)
+
+            return fail_error_message, output_list, embeddings_out, data_file_name_no_ext, None, docs, vectoriser_model
+
+        # For some reason, zero topic modelling exports assigned topics as a np.array instead of a list. Converting it back here.
+        if isinstance(assigned_topics, np.ndarray):
+            assigned_topics = assigned_topics.tolist()
+
+         # Zero shot modelling is a model merge, which wipes the c_tf_idf part of the resulting model completely. To get hierarchical modelling to work, we need to recreate this part of the model with the CountVectorizer options used to create the initial model. Since with zero shot, we are merging two models that have exactly the same set of documents, the vocubulary should be the same, and so recreating the cf_tf_idf component in this way shouldn't be a problem. Discussion here, and below based on Maarten's suggested code: https://github.com/MaartenGr/BERTopic/issues/1700
+
+        doc_dets = topic_model.get_document_info(docs)
+
+        documents_per_topic = doc_dets.groupby(['Topic'], as_index=False).agg({'Document': ' '.join})
+
+        # Assign CountVectorizer to merged model
+
+        topic_model.vectorizer_model = vectoriser_model
+
+        # Re-calculate c-TF-IDF
+        c_tf_idf, _ = topic_model._c_tf_idf(documents_per_topic)
+        topic_model.c_tf_idf_ = c_tf_idf
+
+    if not assigned_topics:
+    # Handle the empty array case
+        return "No topics found.", output_list, embeddings_out, data_file_name_no_ext, topic_model, docs
+    
+    else: 
+        print("Topic model created.")
+
+    # Replace current topic labels if new ones loaded in
+    if not custom_labels_df.empty:
+        #custom_label_list = list(custom_labels_df.iloc[:,0])
+        custom_label_list = [label.replace("\n", "") for label in custom_labels_df.iloc[:,0]]
+
+        topic_model.set_topic_labels(custom_label_list)
+        
+    print("Custom topics: ", topic_model.custom_labels_)
+
+    # Outputs
+    output_list, output_text = save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, save_topic_model)
+
+     # If you want to save your embedding files
+    if return_intermediate_files == "Yes":
+        print("Saving embeddings to file")
+        if low_resource_mode == "Yes":
+            embeddings_file_name = data_file_name_no_ext + '_' + 'tfidf_embeddings.npz'
+        else:
+            if embeddings_super_compress == "No":
+                embeddings_file_name = data_file_name_no_ext + '_' + 'bge_embeddings.npz'
+            else:
+                embeddings_file_name = data_file_name_no_ext + '_' + 'bge_embeddings_compress.npz'
+
+        np.savez_compressed(embeddings_file_name, embeddings_out)
+
+        output_list.append(embeddings_file_name)
+
+    all_toc = time.perf_counter()
+    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds."
+    print(time_out)
+
+    return output_text, output_list, embeddings_out, data_file_name_no_ext, topic_model, docs, vectoriser_model
+
+def reduce_outliers(topic_model, docs, embeddings_out, data_file_name_no_ext, save_topic_model, progress=gr.Progress(track_tqdm=True)):
+
+    progress(0, desc= "Preparing data")
+
+    output_list = []
+
+    all_tic = time.perf_counter()
+
+    assigned_topics, probs = topic_model.fit_transform(docs, embeddings_out)
+
+    if isinstance(assigned_topics, np.ndarray):
+        assigned_topics = assigned_topics.tolist()
+
+
+    # Reduce outliers if required, then update representation
+    progress(0.2, desc= "Reducing outliers")
+    print("Reducing outliers.")
+    # Calculate the c-TF-IDF representation for each outlier document and find the best matching c-TF-IDF topic representation using cosine similarity.
+    assigned_topics = topic_model.reduce_outliers(docs, assigned_topics, strategy="embeddings")
+    # Then, update the topics to the ones that considered the new data
+
+    print("Finished reducing outliers.")
+
+    progress(0.7, desc= "Replacing topic names with LLMs if necessary")
+
+    topic_dets = topic_model.get_topic_info()
+
+    # Replace original labels with LLM labels
+    if "LLM" in topic_model.get_topic_info().columns:
+        llm_labels = [label[0][0].split("\n")[0] for label in topic_model.get_topics(full=True)["LLM"].values()]
+        topic_model.set_topic_labels(llm_labels)
+    else:
+        topic_model.set_topic_labels(list(topic_dets["Name"]))
+
+    # Outputs   
+    progress(0.9, desc= "Saving to file")
+    output_list, output_text = save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, save_topic_model)
+
+    all_toc = time.perf_counter()
+    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds"
+    print(time_out)
+    
+    return output_text, output_list, topic_model
+
+def represent_topics(topic_model, docs, data_file_name_no_ext, low_resource_mode, save_topic_model, representation_type, vectoriser_model, progress=gr.Progress(track_tqdm=True)):
+    from funcs.representation_model import create_representation_model, llm_config, chosen_start_tag
+
+    output_list = []
+
+    all_tic = time.perf_counter()
+
+    progress(0.1, desc= "Loading model and creating new representation")
+
+    representation_model = create_representation_model(representation_type, llm_config, hf_model_name, hf_model_file, chosen_start_tag, low_resource_mode)  
+
+    progress(0.6, desc= "Updating existing topics")
+    topic_model.update_topics(docs, vectorizer_model=vectoriser_model, representation_model=representation_model)
+
+    topic_dets = topic_model.get_topic_info()
+
+    # Replace original labels with LLM labels
+    if representation_type == "LLM":
+        llm_labels = [label[0].split("\n")[0] for label in topic_dets["LLM"]]
+        topic_model.set_topic_labels(llm_labels)
+
+        label_list_file_name = data_file_name_no_ext + '_llm_topic_list_' + today_rev + '.csv'
+
+        llm_labels_df = pd.DataFrame(data={"Label":llm_labels})
+        llm_labels_df.to_csv(label_list_file_name, index=None)
+
+        output_list.append(label_list_file_name)
+    else:
+        new_topic_labels = topic_model.generate_topic_labels(nr_words=3, separator=", ", aspect = representation_type)
+
+        topic_model.set_topic_labels(new_topic_labels)#list(topic_dets[representation_type]))
+        #topic_model.set_topic_labels(list(topic_dets["Name"]))
+
+    # Outputs
+    progress(0.8, desc= "Saving outputs")
+    output_list, output_text = save_topic_outputs(topic_model, data_file_name_no_ext, output_list, docs, save_topic_model)
+
+    all_toc = time.perf_counter()
+    time_out = f"All processes took {all_toc - all_tic:0.1f} seconds"
+    print(time_out)
+
+    return output_text, output_list, topic_model
+
+def visualise_topics(topic_model, data, data_file_name_no_ext, low_resource_mode,  embeddings_out, in_label, in_colnames, legend_label, sample_prop, visualisation_type_radio, random_seed,  progress=gr.Progress(track_tqdm=True)):
+
+    progress(0, desc= "Preparing data for visualisation")
+
+    output_list = []
+    vis_tic = time.perf_counter()
+
+    from funcs.bertopic_vis_documents import visualize_documents_custom, visualize_hierarchical_documents_custom, visualize_barchart_custom
+
+    if not visualisation_type_radio:
+        return "Please choose a visualisation type above.", output_list, None, None
+
+    # Get topic labels
+    if in_label:
+       in_label_list_first = in_label[0]
+    else:
+       return "Label column not found. Please enter this above.", output_list, None, None
+    
+    # Get docs
+    if in_colnames:
+        in_colnames_list_first = in_colnames[0]
+    else:
+        return "Label column not found. Please enter this on the data load tab.", output_list, None, None
+    
+    docs = list(data[in_colnames_list_first].str.lower())
+
+    # Make sure format of input series is good
+    data[in_label_list_first] = data[in_label_list_first].fillna('').astype(str)
+    label_list = list(data[in_label_list_first])
+
+    topic_dets = topic_model.get_topic_info()
+
+    # Replace original labels with another representation if specified
+    if legend_label:
+        topic_dets = topic_model.get_topics(full=True)
+        if legend_label in topic_dets:
+            labels = [topic_dets[legend_label].values()]
+            labels = [str(v) for v in labels]
+            topic_model.set_topic_labels(labels)
+
+    # Pre-reduce embeddings for visualisation purposes
+    if low_resource_mode == "No":
+        reduced_embeddings = UMAP(n_neighbors=15, n_components=2, min_dist=0.0, metric='cosine', random_state=random_seed).fit_transform(embeddings_out)
+    else:
+        reduced_embeddings = TruncatedSVD(2, random_state=random_seed).fit_transform(embeddings_out)
+
+    progress(0.5, desc= "Creating visualisation (this can take a while)")
+    # Visualise the topics:
+    
+    print("Creating visualisation")
+
+    # "Topic document graph", "Hierarchical view"
+
+    if visualisation_type_radio == "Topic document graph":
+        topics_vis = visualize_documents_custom(topic_model, docs, hover_labels = label_list, reduced_embeddings=reduced_embeddings, hide_annotations=True, hide_document_hover=False, custom_labels=True, sample = sample_prop, width= 1200, height = 750)
+
+        topics_vis_name = data_file_name_no_ext + '_' + 'vis_topic_docs_' + today_rev + '.html'
+        topics_vis.write_html(topics_vis_name)
+        output_list.append(topics_vis_name)
+
+        topics_vis_2 = topic_model.visualize_heatmap(custom_labels=True, width= 1200, height = 1200)
+
+        topics_vis_2_name = data_file_name_no_ext + '_' + 'vis_heatmap_' + today_rev + '.html'
+        topics_vis_2.write_html(topics_vis_2_name)
+        output_list.append(topics_vis_2_name)
+
+    elif visualisation_type_radio == "Hierarchical view":
+
+        hierarchical_topics = topic_model.hierarchical_topics(docs)
+
+        # Save new hierarchical topic model to file
+        hierarchical_topics_name = data_file_name_no_ext + '_' + 'vis_hierarchy_topics_' + today_rev + '.csv'
+        hierarchical_topics.to_csv(hierarchical_topics_name)
+        output_list.append(hierarchical_topics_name)
+
+        try:
+            topics_vis = visualize_hierarchical_documents_custom(topic_model, docs, label_list, hierarchical_topics, reduced_embeddings=reduced_embeddings, sample = sample_prop, hide_document_hover= False, custom_labels=True, width= 1200, height = 750)
+            topics_vis_2 = topic_model.visualize_hierarchy(hierarchical_topics=hierarchical_topics, width= 1200, height = 750)
+        except:
+            error_message = "Visualisation preparation failed. Perhaps you need more topics to create the full hierarchy (more than 10)?"
+            return error_message, output_list, None, None
+
+        topics_vis_name = data_file_name_no_ext + '_' + 'vis_hierarchy_topic_doc_' + today_rev + '.html'
+        topics_vis.write_html(topics_vis_name)
+        output_list.append(topics_vis_name)
+
+        topics_vis_2_name = data_file_name_no_ext + '_' + 'vis_hierarchy_' + today_rev + '.html'
+        topics_vis_2.write_html(topics_vis_2_name)
+        output_list.append(topics_vis_2_name)
+
+    all_toc = time.perf_counter()
+    time_out = f"Creating visualisation took {all_toc - vis_tic:0.1f} seconds"
+    print(time_out)
+
+    return time_out, output_list, topics_vis, topics_vis_2
+
+def save_as_pytorch_model(topic_model, data_file_name_no_ext , progress=gr.Progress(track_tqdm=True)):
+
+    if not topic_model:
+        return "No Pytorch model found.", None
+
+    progress(0, desc= "Saving topic model in Pytorch format")
+
+    output_list = []
+
+
+    topic_model_save_name_folder = "output_model/" + data_file_name_no_ext + "_topics_" + today_rev# + ".safetensors"
+    topic_model_save_name_zip = topic_model_save_name_folder + ".zip"
+
+    # Clear folder before replacing files
+    delete_files_in_folder(topic_model_save_name_folder)
+
+    topic_model.save(topic_model_save_name_folder, serialization='pytorch', save_embedding_model=True, save_ctfidf=False)
+
+    # Zip file example
+    
+    zip_folder(topic_model_save_name_folder, topic_model_save_name_zip)
+    output_list.append(topic_model_save_name_zip)
+
+    return "Model saved in Pytorch format.", output_list

requirements.txt

@@ -2,11 +2,14 @@ gradio==3.50.0
 transformers==4.37.1
 accelerate==0.26.1
 torch==2.1.2
-llama-cpp-python==0.2.33
+llama-cpp-python==0.2.36
 bertopic==0.16.0
 spacy==3.7.2
+en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1.tar.gz
 pyarrow==14.0.2
+openpyxl==3.1.2
 Faker==22.2.0
 presidio_analyzer==2.2.351
 presidio_anonymizer==2.2.351
 scipy==1.11.4
+polars==0.20.6