Changed all intermediate file outputs to save to output folder

app.py

@@ -12,11 +12,6 @@ from search_funcs.helper_functions import display_info, initial_data_load, put_c
 from search_funcs.spacy_search_funcs import spacy_fuzzy_search
 from search_funcs.aws_functions import load_data_from_aws
 
-#from fastapi import FastAPI
-#app = FastAPI()
-
-
-
 # Attempt to delete temporary files generated by previous use of the app (as the files can be very big!). Only setup to work for local runs in Windows
 temp_folder_path = get_temp_folder_path()
 empty_folder(temp_folder_path)
@@ -104,8 +99,7 @@ depends on factors such as the type of documents or queries. Information taken f
     **Thematic/semantic search**
 
     This search type enables you to search for broader themes (e.g. happiness, nature) and the search will pick out text passages that relate to these themes even if they don't contain the exact words. 1. Load in data file (ideally a file with '_cleaned' at the end of the name, a pkl.gz file), with (optionally) the 'embeddings... .npz' file in the same folder to save loading time. 2. Select the field in your data to search. 3. Wait for the data file to be prepared for search. 4. Enter the search term in the 'Enter semantic search query here' box below and press Enter/click on 'Start semantic search'. 4. Your search results will be saved in a csv file and will be presented in the 'File output' area below.
-    """)
-       
+    """)       
 
         with gr.Row():
             current_source_semantic = gr.Textbox(label="Current data source(s)", value="None")

search_funcs/bm25_functions.py

@@ -220,12 +220,12 @@ class BM25:
 		return list(indices), docs, list(scores)
 
 	def save(self, filename):
-		with open(f"{filename}.pkl", "wb") as fsave:
+		with open(f"{output_folder}{filename}.pkl", "wb") as fsave:
 			pickle.dump(self, fsave, protocol=pickle.HIGHEST_PROTOCOL)
 
 	@staticmethod
 	def load(filename):
-		with open(f"{filename}.pkl", "rb") as fsave:
+		with open(f"{output_folder}{filename}.pkl", "rb") as fsave:
 			return pickle.load(fsave)
 
 # These following functions are my own work
@@ -432,9 +432,9 @@ def prepare_bm25(corpus, in_file, text_column, search_index, clean, return_inter
 		progress(0.8, desc = "Saving search index to file")
 
 		if clean == "Yes":
-			bm25_search_file_name = data_file_name_no_ext + '_cleaned_search_index.pkl.gz'
+			bm25_search_file_name = output_folder + data_file_name_no_ext + '_cleaned_search_index.pkl.gz'
 		else:
-			bm25_search_file_name = data_file_name_no_ext + '_search_index.pkl.gz'
+			bm25_search_file_name = output_folder + data_file_name_no_ext + '_search_index.pkl.gz'
 			#np.savez_compressed(bm25_search_file_name, bm25)
 
 		with gzip.open(bm25_search_file_name, 'wb') as file:

search_funcs/helper_functions.py

@@ -231,11 +231,6 @@ def put_columns_in_join_df(in_file):
     return gr.Dropdown(choices=concat_choices), new_df, out_message
 
 
-    """
-    A dummy function that exists just so that dropdown updates work correctly.
-    """
-    return None    
-
 def display_info(info_component):
     gr.Info(info_component)
 

search_funcs/semantic_ingest_functions.py

@@ -68,9 +68,7 @@ def parse_file_not_used(file_paths, text_column='text'):
     file_names = []
 
     for file_path in file_paths:
-        #print(file_path.name)
-        #file = open(file_path.name, 'r')
-        #print(file)
+
         file_extension = detect_file_type(file_path.name)
         if file_extension in extension_to_parser:
             parsed_contents[file_path.name] = extension_to_parser[file_extension](file_path.name)
@@ -222,19 +220,11 @@ def csv_excel_text_to_docs(df, in_file, text_column, clean = "No", return_interm
     if "prepared_docs" in data_file_name:
         print("Loading in documents from file.")
 
-        #print(df[0:5])
-        #section_series = df.iloc[:,0]
-        #section_series = "{" + section_series + "}"
-
         doc_sections = df
 
-        #print(doc_sections[0])
-
         # Convert each element in the Series to a Document instance
-        #doc_sections = section_series.apply(lambda x: Document(**x))
 
         return doc_sections, "Finished preparing documents", output_list
-    #    df = document_to_dataframe(df.iloc[:,0])
 
     ingest_tic = time.perf_counter()
 
@@ -248,16 +238,9 @@ def csv_excel_text_to_docs(df, in_file, text_column, clean = "No", return_interm
         clean_tic = time.perf_counter()
         print("Starting data clean.")
         
-        #df = df.drop_duplicates(text_column)
-        
         df_list = list(df[text_column])
         df_list = initial_clean(df_list)
 
-        # Get rid of old data and keep only the new
-        #df = df.drop(text_column, axis = 1)
-
-        
-
         # Save to file if you have cleaned the data. Text column has now been renamed with '_cleaned' at the send
         out_file_name, text_column, df = save_prepared_bm25_data(data_file_name, df_list, df, text_column)
 
@@ -272,13 +255,6 @@ def csv_excel_text_to_docs(df, in_file, text_column, clean = "No", return_interm
 
     df["metadata"] = combine_metadata_columns(df, cols)
 
-    #df = df.rename(columns={text_column:"page_content"})
-
-    #print(df[["page_content", "metadata"]].to_dict(orient='records'))
-
-    #doc_sections = df[["page_content", "metadata"]].to_dict(orient='records')
-    #doc_sections = [Document(**row) for row in df[["page_content", "metadata"]].to_dict(orient='records')]
-
     progress(0.3, desc = "Converting data to document format")
 
     # Create a list of Document objects
@@ -295,29 +271,17 @@ def csv_excel_text_to_docs(df, in_file, text_column, clean = "No", return_interm
         progress(0.5, desc = "Saving prepared documents")
         data_file_out_name_no_ext = get_file_path_end(data_file_name)
         file_name = data_file_out_name_no_ext
-        #print(doc_sections)
-        #page_content_series_string = pd.Series(doc_sections).astype(str)
-        #page_content_series_string = page_content_series_string.str.replace(" type='Document'", "").str.replace("' metadata=", "', 'metadata':").str.replace("page_content=", "{'page_content':")
-        #page_content_series_string = page_content_series_string + "}"
-        #print(page_content_series_string[0])
-        #metadata_series_string = pd.Series(doc_sections[1]).astype(str)
-
 
         if clean == "No":
-            #pd.DataFrame(data = {"Documents":page_content_series_string}).to_parquet(file_name + "_prepared_docs.parquet")
             out_doc_file_name = output_folder + file_name + "_prepared_docs.pkl.gz"
             with gzip.open(out_doc_file_name, 'wb') as file:
                 pickle.dump(doc_sections, file)
 
-            #pd.Series(doc_sections).to_pickle(file_name + "_prepared_docs.pkl")
         elif clean == "Yes":
-            #pd.DataFrame(data = {"Documents":page_content_series_string}).to_parquet(file_name + "_prepared_docs_clean.parquet")
-
             out_doc_file_name = output_folder + file_name + "_cleaned_prepared_docs.pkl.gz"
             with gzip.open(out_doc_file_name, 'wb') as file:
                 pickle.dump(doc_sections, file)
 
-            #pd.Series(doc_sections).to_pickle(file_name + "_prepared_docs_clean.pkl")
         output_list.append(out_doc_file_name)
         print("Documents saved to file.")
 

search_funcs/spacy_search_funcs.py

@@ -7,7 +7,7 @@ import gradio as gr
 import pandas as pd
 from typing import List, Type
 from datetime import datetime
-from search_funcs.helper_functions import create_highlighted_excel_wb
+from search_funcs.helper_functions import create_highlighted_excel_wb, output_folder
 
 PandasDataFrame = Type[pd.DataFrame]
 
@@ -106,7 +106,7 @@ def spacy_fuzzy_search(string_query:str, df_list: List[str], original_data: Pand
 
     # Out file
     query_str_file = ("_").join(tokenised_query)
-    results_df_name = "keyword_search_result_" + today_rev + "_" +  query_str_file + ".xlsx"
+    results_df_name = output_folder + "fuzzy_keyword_search_result_" + today_rev + "_" +  query_str_file + ".xlsx"
 
     print("Saving search file output")
     progress(0.7, desc = "Saving search output to file")