Now checks for output folder before saving. Minor code cleaning

Dockerfile

@@ -14,7 +14,7 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Gradio needs to be installed after due to conflict with spacy in requirements
-RUN pip install --no-cache-dir gradio==4.31.0 
+RUN pip install --no-cache-dir gradio==4.31.4 
 
 # Download the BGE embedding model during the build process
 RUN curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash

README.md

@@ -4,7 +4,7 @@ emoji: 🔍
 colorFrom: purple
 colorTo: green
 sdk: gradio
-sdk_version: 4.31.0
+sdk_version: 4.31.4
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -129,7 +129,7 @@ depends on factors such as the type of documents or queries. Information taken f
         with gr.Accordion(label="Data load / save options", open = True):
             with gr.Row():
                 in_clean_data = gr.Dropdown(label = "Clean text during load (remove html tags). For large files this may take some time!", value="No", choices=["Yes", "No"])
-                return_intermediate_files = gr.Dropdown(label = "Return intermediate processing files from file preparation. Files can be loaded in to save processing time in future.", value="False", choices=["Yes", "No"])
+                return_intermediate_files = gr.Dropdown(label = "Return intermediate processing files from file preparation. Files can be loaded in to save processing time in future.", value="No", choices=["Yes", "No"])
                 embedding_super_compress = gr.Dropdown(label = "Round embeddings to three dp for smaller files with less accuracy.", value="Yes", choices=["Yes", "No"])
             #save_clean_data_button = gr.Button(value = "Save loaded data to file", scale = 1)
         with gr.Accordion(label="Keyword search options", open = False):
@@ -156,6 +156,7 @@ depends on factors such as the type of documents or queries. Information taken f
             in_join_message = gr.Textbox(label="Join file load progress")
             in_join_column = gr.Dropdown(label="Column to join in new data frame")
             search_df_join_column = gr.Dropdown(label="Column to join in search data frame")
+
         with gr.Accordion(label = "AWS data access", open = False):
             aws_password_box = gr.Textbox(label="Password for AWS data access (ask the Data team if you don't have this)")
             with gr.Row():

how_to_create_exe_dist.txt

@@ -14,7 +14,7 @@ NOTE: for ensuring that spaCy models are loaded into the program correctly in re
 
 9.Run the following, assuming you want to make one single .exe file (This helped me: https://github.com/pyinstaller/pyinstaller/issues/8108):
 
-a) In command line: pyi-makespec --additional-hooks-dir="build_deps\\" --collect-data=gradio_client --collect-data=gradio --hidden-import pyarrow.vendored.version --onefile --name DataSearchApp_0.3 app.py
+a) In command line: pyi-makespec --additional-hooks-dir="build_deps\\" --collect-data=gradio_client --collect-data=gradio --hidden-import pyarrow.vendored.version --onefile --name DataSearchApp_0.4 app.py
 
 b) Open the created spec file in Notepad. Add the following to the end of the Analysis section then save:
 
@@ -25,7 +25,7 @@ a = Analysis(
     }
 )
 
-c) Back in command line, run this: pyinstaller --clean --noconfirm DataSearchApp_0.3.spec
+c) Back in command line, run this: pyinstaller --clean --noconfirm DataSearchApp_0.4.spec
 
 
 9. A 'dist' folder will be created with the executable inside along with all dependencies('dist\data_text_search'). 

search_funcs/aws_functions.py

@@ -7,8 +7,8 @@ import os
 PandasDataFrame = Type[pd.DataFrame]
 
 try:
+    session = boto3.Session()
     bucket_name = os.environ['DATA_TEXT_SEARCH_BUCKET']
-    session = boto3.Session(profile_name="default")
 except Exception as e:
     bucket_name = ''
     print(e)

search_funcs/bm25_functions.py

@@ -14,7 +14,7 @@ from datetime import datetime
 today_rev = datetime.now().strftime("%Y%m%d")
 
 from search_funcs.clean_funcs import initial_clean # get_lemma_tokens, stem_sentence
-from search_funcs.helper_functions import get_file_path_end_with_ext, get_file_path_end, create_highlighted_excel_wb
+from search_funcs.helper_functions import get_file_path_end_with_ext, get_file_path_end, create_highlighted_excel_wb, ensure_output_folder_exists
 
 # Load the SpaCy model
 from spacy.cli.download import download
@@ -232,6 +232,7 @@ class BM25:
 
 def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, clean="No",  return_intermediate_files = "No", progress=gr.Progress(track_tqdm=True)):
 	#print(in_file)
+	ensure_output_folder_exists()
 
 	if not in_file:
 		print("No input file found. Please load in at least one file.")
@@ -324,6 +325,7 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 		message = "Data loaded. Warning: dataset may be too short to get consistent search results."
 
 	if return_intermediate_files == "Yes":
+
 		if clean == "Yes":
 			tokenised_data_file_name = "output/" + data_file_out_name_no_ext + "_cleaned_tokenised.parquet"
 		else:
@@ -337,6 +339,8 @@ def prepare_bm25_input_data(in_file, text_column, data_state, tokenised_state, c
 
 def save_prepared_bm25_data(in_file_name, prepared_text_list, in_df, in_bm25_column, progress=gr.Progress(track_tqdm=True)):
 
+	ensure_output_folder_exists()
+
 	# Check if the list and the dataframe have the same length
 	if len(prepared_text_list) != len(in_df):
 		raise ValueError("The length of 'prepared_text_list' and 'in_df' must match.")
@@ -543,6 +547,8 @@ def bm25_search(free_text_query, in_no_search_results, original_data, searched_d
 	results_df_out = results_df_out.sort_values(['search_score_abs', "search_text"], ascending=False)	
 
 	# Out file
+	ensure_output_folder_exists()
+
 	query_str_file = ("_").join(token_query)
 	results_df_name = "output/keyword_search_result_" + today_rev + "_" +  query_str_file + ".xlsx"
 

search_funcs/helper_functions.py

@@ -58,6 +58,18 @@ def get_file_path_end_with_ext(file_path):
 
     return filename_end
 
+def ensure_output_folder_exists():
+    """Checks if the 'output/' folder exists, creates it if not."""
+
+    folder_name = "output/"
+
+    if not os.path.exists(folder_name):
+        # Create the folder if it doesn't exist
+        os.makedirs(folder_name)
+        print(f"Created the 'output/' folder.")
+    else:
+        print(f"The 'output/' folder already exists.")
+
 def detect_file_type(filename):
     """Detect the file type based on its extension."""
     if (filename.endswith('.csv')) | (filename.endswith('.csv.gz')) | (filename.endswith('.zip')):

search_funcs/semantic_functions.py

@@ -25,7 +25,7 @@ else:
 
 print("Device used is: ", torch_device)
 
-from search_funcs.helper_functions import create_highlighted_excel_wb
+from search_funcs.helper_functions import create_highlighted_excel_wb, ensure_output_folder_exists
 
 PandasDataFrame = Type[pd.DataFrame]
 
@@ -67,8 +67,11 @@ else:
     
 def docs_to_bge_embed_np_array(docs_out, in_file, embeddings_state, output_file_state, clean, return_intermediate_files = "No", embeddings_super_compress = "No", embeddings_model = embeddings_model, progress=gr.Progress(track_tqdm=True)):
     '''
-    Takes a Langchain document class and saves it into a Chroma sqlite file.
+    Takes a Langchain document class and saves it into a Numpy array.
     '''
+
+    ensure_output_folder_exists()
+
     if not in_file:
         out_message = "No input file found. Please load in at least one file."
         print(out_message)
@@ -229,6 +232,8 @@ def bge_simple_retrieval(query_str:str, vectorstore, docs, orig_df_col:str, k_va
     # print("vectorstore loaded: ", vectorstore)
     progress(0, desc = "Conducting semantic search")
 
+    ensure_output_folder_exists()
+
     print("Searching")
 
     # Convert it to a PyTorch tensor and transfer to GPU

search_funcs/semantic_ingest_functions.py

@@ -31,7 +31,7 @@ chunk_size = 512
 chunk_overlap = 0
 start_index = True
 
-from search_funcs.helper_functions import get_file_path_end_with_ext, detect_file_type, get_file_path_end
+from search_funcs.helper_functions import get_file_path_end_with_ext, detect_file_type, get_file_path_end, ensure_output_folder_exists
 from search_funcs.bm25_functions import save_prepared_bm25_data
 from search_funcs.clean_funcs import initial_clean
 
@@ -198,6 +198,7 @@ def parse_metadata(row):
 def csv_excel_text_to_docs(df, in_file, text_column, clean = "No", return_intermediate_files = "No", chunk_size=None, progress=gr.Progress(track_tqdm=True)) -> List[Document]:
     """Converts a DataFrame's content to a list of dictionaries in the 'Document' format, containing page_content and associated metadata."""
 
+    ensure_output_folder_exists()
     output_list = []
 
     if not in_file: