Spaces:
Running
Running
Added controls for saving topic models and visualisation. Removed custom UMAP layer
Browse files
app.py
CHANGED
@@ -79,7 +79,7 @@ hf_model_name = 'TheBloke/phi-2-orange-GGUF' #'NousResearch/Nous-Capybara-7B-V1
|
|
79 |
hf_model_file = 'phi-2-orange.Q5_K_M.gguf' #'Capybara-7B-V1.9-Q5_K_M.gguf' # 'stablelm-2-zephyr-1_6b-Q5_K_M.gguf'
|
80 |
|
81 |
|
82 |
-
def extract_topics(in_files, in_file, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, in_label, anonymise_drop, return_intermediate_files, embeddings_super_compress, low_resource_mode, create_llm_topic_labels):
|
83 |
|
84 |
output_list = []
|
85 |
file_list = [string.name for string in in_file]
|
@@ -149,11 +149,11 @@ def extract_topics(in_files, in_file, min_docs_slider, in_colnames, max_topics_s
|
|
149 |
|
150 |
|
151 |
if not candidate_topics:
|
152 |
-
umap_model = UMAP(n_neighbors=15, n_components=5, random_state=random_seed)
|
153 |
|
154 |
topic_model = BERTopic( embedding_model=embedding_model_pipe,
|
155 |
vectorizer_model=vectoriser_model,
|
156 |
-
umap_model=umap_model,
|
157 |
min_topic_size= min_docs_slider,
|
158 |
nr_topics = max_topics_slider,
|
159 |
representation_model=representation_model,
|
@@ -177,11 +177,11 @@ def extract_topics(in_files, in_file, min_docs_slider, in_colnames, max_topics_s
|
|
177 |
umap_neighbours = len(zero_shot_topics_lower)
|
178 |
else: umap_neighbours = 15
|
179 |
|
180 |
-
umap_model = UMAP(n_neighbors=umap_neighbours, n_components=5, random_state=random_seed)
|
181 |
|
182 |
topic_model = BERTopic( embedding_model=embedding_model_pipe,
|
183 |
vectorizer_model=vectoriser_model,
|
184 |
-
umap_model=umap_model,
|
185 |
min_topic_size = min_docs_slider,
|
186 |
nr_topics = max_topics_slider,
|
187 |
zeroshot_topic_list = zero_shot_topics_lower,
|
@@ -228,19 +228,20 @@ def extract_topics(in_files, in_file, min_docs_slider, in_colnames, max_topics_s
|
|
228 |
topics_text_out_str = str(topic_dets["Name"])
|
229 |
output_text = "Topics: " + topics_text_out_str
|
230 |
|
231 |
-
#
|
232 |
-
|
233 |
-
|
|
|
234 |
|
235 |
-
|
236 |
-
|
237 |
|
238 |
-
|
239 |
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
|
245 |
if return_intermediate_files == "Yes":
|
246 |
print("Saving embeddings to file")
|
@@ -249,11 +250,14 @@ def extract_topics(in_files, in_file, min_docs_slider, in_colnames, max_topics_s
|
|
249 |
|
250 |
output_list.append(semantic_search_file_name)
|
251 |
|
252 |
-
|
253 |
-
|
254 |
-
|
|
|
|
|
|
|
255 |
|
256 |
-
return output_text, output_list,
|
257 |
|
258 |
# , topic_model_save_name
|
259 |
|
@@ -301,14 +305,16 @@ with block:
|
|
301 |
return_intermediate_files = gr.Dropdown(label = "Return intermediate processing files from file preparation. Files can be loaded in to save processing time in future.", value="Yes", choices=["Yes", "No"])
|
302 |
embedding_super_compress = gr.Dropdown(label = "Round embeddings to three dp for smaller files with less accuracy.", value="No", choices=["Yes", "No"])
|
303 |
with gr.Row():
|
304 |
-
low_resource_mode_opt = gr.Dropdown(label = "Use low resource embeddings model
|
305 |
create_llm_topic_labels = gr.Dropdown(label = "Create LLM-generated topic labels.", value="No", choices=["Yes", "No"])
|
|
|
|
|
306 |
|
307 |
# Update column names dropdown when file uploaded
|
308 |
in_files.upload(fn=put_columns_in_df, inputs=[in_files], outputs=[in_colnames, in_label, data_state])
|
309 |
in_colnames.change(dummy_function, in_colnames, None)
|
310 |
|
311 |
-
topics_btn.click(fn=extract_topics, inputs=[data_state, in_files, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, in_label, anonymise_drop, return_intermediate_files, embedding_super_compress, low_resource_mode_opt, create_llm_topic_labels], outputs=[output_single_text, output_file, plot], api_name="topics")
|
312 |
|
313 |
block.queue().launch(debug=True)#, server_name="0.0.0.0", ssl_verify=False, server_port=7860)
|
314 |
|
|
|
79 |
hf_model_file = 'phi-2-orange.Q5_K_M.gguf' #'Capybara-7B-V1.9-Q5_K_M.gguf' # 'stablelm-2-zephyr-1_6b-Q5_K_M.gguf'
|
80 |
|
81 |
|
82 |
+
def extract_topics(in_files, in_file, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, in_label, anonymise_drop, return_intermediate_files, embeddings_super_compress, low_resource_mode, create_llm_topic_labels, save_topic_model, visualise_topics):
|
83 |
|
84 |
output_list = []
|
85 |
file_list = [string.name for string in in_file]
|
|
|
149 |
|
150 |
|
151 |
if not candidate_topics:
|
152 |
+
#umap_model = UMAP(n_neighbors=15, n_components=5, random_state=random_seed)
|
153 |
|
154 |
topic_model = BERTopic( embedding_model=embedding_model_pipe,
|
155 |
vectorizer_model=vectoriser_model,
|
156 |
+
#umap_model=umap_model,
|
157 |
min_topic_size= min_docs_slider,
|
158 |
nr_topics = max_topics_slider,
|
159 |
representation_model=representation_model,
|
|
|
177 |
umap_neighbours = len(zero_shot_topics_lower)
|
178 |
else: umap_neighbours = 15
|
179 |
|
180 |
+
#umap_model = UMAP(n_neighbors=umap_neighbours, n_components=5, random_state=random_seed)
|
181 |
|
182 |
topic_model = BERTopic( embedding_model=embedding_model_pipe,
|
183 |
vectorizer_model=vectoriser_model,
|
184 |
+
#umap_model=umap_model,
|
185 |
min_topic_size = min_docs_slider,
|
186 |
nr_topics = max_topics_slider,
|
187 |
zeroshot_topic_list = zero_shot_topics_lower,
|
|
|
228 |
topics_text_out_str = str(topic_dets["Name"])
|
229 |
output_text = "Topics: " + topics_text_out_str
|
230 |
|
231 |
+
# Save topic model to file
|
232 |
+
if save_topic_model == "Yes":
|
233 |
+
topic_model_save_name_folder = "output_model/" + data_file_name_no_ext + "_topics_" + today_rev# + ".safetensors"
|
234 |
+
topic_model_save_name_zip = topic_model_save_name_folder + ".zip"
|
235 |
|
236 |
+
# Clear folder before replacing files
|
237 |
+
delete_files_in_folder(topic_model_save_name_folder)
|
238 |
|
239 |
+
topic_model.save(topic_model_save_name_folder, serialization='pytorch', save_embedding_model=True, save_ctfidf=False)
|
240 |
|
241 |
+
# Zip file example
|
242 |
+
|
243 |
+
zip_folder(topic_model_save_name_folder, topic_model_save_name_zip)
|
244 |
+
output_list.append(topic_model_save_name_zip)
|
245 |
|
246 |
if return_intermediate_files == "Yes":
|
247 |
print("Saving embeddings to file")
|
|
|
250 |
|
251 |
output_list.append(semantic_search_file_name)
|
252 |
|
253 |
+
if visualise_topics == "Yes":
|
254 |
+
# Visualise the topics:
|
255 |
+
print("Creating visualisation")
|
256 |
+
topics_vis = topic_model.visualize_documents(label_col, reduced_embeddings=reduced_embeddings, hide_annotations=True, hide_document_hover=False, custom_labels=True)
|
257 |
+
|
258 |
+
return output_text, output_list, topics_vis
|
259 |
|
260 |
+
return output_text, output_list, None
|
261 |
|
262 |
# , topic_model_save_name
|
263 |
|
|
|
305 |
return_intermediate_files = gr.Dropdown(label = "Return intermediate processing files from file preparation. Files can be loaded in to save processing time in future.", value="Yes", choices=["Yes", "No"])
|
306 |
embedding_super_compress = gr.Dropdown(label = "Round embeddings to three dp for smaller files with less accuracy.", value="No", choices=["Yes", "No"])
|
307 |
with gr.Row():
|
308 |
+
low_resource_mode_opt = gr.Dropdown(label = "Use low resource embeddings model.", value="No", choices=["Yes", "No"])
|
309 |
create_llm_topic_labels = gr.Dropdown(label = "Create LLM-generated topic labels.", value="No", choices=["Yes", "No"])
|
310 |
+
save_topic_model = gr.Dropdown(label = "Save topic model to file.", value="Yes", choices=["Yes", "No"])
|
311 |
+
visualise_topics = gr.Dropdown(label = "Create a visualisation to map topics.", value="Yes", choices=["Yes", "No"])
|
312 |
|
313 |
# Update column names dropdown when file uploaded
|
314 |
in_files.upload(fn=put_columns_in_df, inputs=[in_files], outputs=[in_colnames, in_label, data_state])
|
315 |
in_colnames.change(dummy_function, in_colnames, None)
|
316 |
|
317 |
+
topics_btn.click(fn=extract_topics, inputs=[data_state, in_files, min_docs_slider, in_colnames, max_topics_slider, candidate_topics, in_label, anonymise_drop, return_intermediate_files, embedding_super_compress, low_resource_mode_opt, create_llm_topic_labels, save_topic_model, visualise_topics], outputs=[output_single_text, output_file, plot], api_name="topics")
|
318 |
|
319 |
block.queue().launch(debug=True)#, server_name="0.0.0.0", ssl_verify=False, server_port=7860)
|
320 |
|