Some code rearranged. Fixed API call paths for Linux systems

app.py

@@ -1,27 +1,20 @@
-# Load in packages, variables for fuzzy matching
 import os
 from datetime import datetime
 from pathlib import Path
-import time
-import copy
 import gradio as gr
-import re
-#import polars as pl
+import pandas as pd
 
-from tools.constants import *
-from tools.matcher_funcs import load_matcher_data, run_match_batch, combine_two_matches, create_match_summary
+from tools.matcher_funcs import run_matcher
 from tools.gradio import initial_data_load
 from tools.aws_functions import load_data_from_aws
-from tools.preparation import prepare_search_address_string, prepare_search_address, prepare_ref_address, remove_non_postal, check_no_number_addresses
-from tools.standardise import standardise_wrapper_func
 
 import warnings
+# Remove warnings from print statements
 warnings.filterwarnings("ignore", 'This pattern is interpreted as a regular expression')
 warnings.filterwarnings("ignore", 'Downcasting behavior')
 warnings.filterwarnings("ignore", 'A value is trying to be set on a copy of a slice from a DataFrame')
 warnings.filterwarnings("ignore")
 
-
 today = datetime.now().strftime("%d%m%Y")
 today_rev = datetime.now().strftime("%Y%m%d")
 
@@ -32,335 +25,7 @@ output_folder = base_folder/"Output/"
 diagnostics_folder = base_folder/"Diagnostics/"
 prep_folder = base_folder/"Helper functions/"
 
-def create_simple_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:int, ref_batch_size:int):
-    #print("Search df batch size: ", batch_size)
-    #print("ref_df df batch size: ", ref_batch_size)
-
-    total_rows = df.shape[0]
-    ref_total_rows = ref_df.shape[0]
-
-    # Creating bottom and top limits for search data
-    search_ranges = []
-    for start in range(0, total_rows, batch_size):
-        end = min(start + batch_size - 1, total_rows - 1)  # Adjusted to get the top limit
-        search_ranges.append((start, end))
-
-    # Creating bottom and top limits for reference data
-    ref_ranges = []
-    for start in range(0, ref_total_rows, ref_batch_size):
-        end = min(start + ref_batch_size - 1, ref_total_rows - 1)  # Adjusted to get the top limit
-        ref_ranges.append((start, end))
-
-    # Create DataFrame with combinations of search_range and ref_range
-    result_data = []
-    for search_range in search_ranges:
-        for ref_range in ref_ranges:
-            result_data.append((search_range, ref_range))
-
-    range_df = pd.DataFrame(result_data, columns=['search_range', 'ref_range'])
-
-    return range_df
-
-
-def create_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:int, ref_batch_size:int, search_postcode_col:str, ref_postcode_col:str):
-    '''
-    Create batches of address indexes for search and reference dataframes based on shortened postcodes.
-    '''
-
-    # If df sizes are smaller than the batch size limits, no need to run through everything
-    if len(df) < batch_size and len(ref_df) < ref_batch_size:
-        print("Dataframe sizes are smaller than maximum batch sizes, no need to split data.")
-        lengths_df = pd.DataFrame(data={'search_range':[df.index.tolist()], 'ref_range':[ref_df.index.tolist()], 'batch_length':len(df), 'ref_length':len(ref_df)})
-        return lengths_df
-    
-    #df.index = df[search_postcode_col]
-
-    df['index'] = df.index
-    ref_df['index'] = ref_df.index    
-
-    # Remove the last character of postcode
-    df['postcode_minus_last_character'] = df[search_postcode_col].str.lower().str.strip().str.replace("\s+", "", regex=True).str[:-1]
-    ref_df['postcode_minus_last_character'] = ref_df[ref_postcode_col].str.lower().str.strip().str.replace("\s+", "", regex=True).str[:-1]
-
-    unique_postcodes = df['postcode_minus_last_character'][df['postcode_minus_last_character'].str.len()>=4].unique().tolist()
-
-    df = df.set_index('postcode_minus_last_character')
-    ref_df = ref_df.set_index('postcode_minus_last_character')
-
-    df = df.sort_index()
-    ref_df = ref_df.sort_index()
-
-    #df.to_csv("batch_search_df.csv")
-
-    # Overall batch variables
-    batch_indexes = []
-    ref_indexes = []
-    batch_lengths = []
-    ref_lengths = []
-
-    # Current batch variables for loop
-    current_batch = []
-    current_ref_batch = []
-    current_batch_length = []
-    current_ref_length = []
-
-    unique_postcodes_iterator = unique_postcodes.copy()
-
-    while unique_postcodes_iterator:
-        
-        unique_postcodes_loop = unique_postcodes_iterator.copy()
-
-        #print("Current loop postcodes: ", unique_postcodes_loop)
-
-        for current_postcode in unique_postcodes_loop:
-            
-
-
-            if len(current_batch) >= batch_size or len(current_ref_batch) >= ref_batch_size:
-                print("Batch length reached - breaking")
-                break
-            
-            try:
-                current_postcode_search_data_add = df.loc[[current_postcode]]#[df['postcode_minus_last_character'].isin(current_postcode)]
-                current_postcode_ref_data_add = ref_df.loc[[current_postcode]]#[ref_df['postcode_minus_last_character'].isin(current_postcode)]
-
-                #print(current_postcode_search_data_add)
-
-                if not current_postcode_search_data_add.empty:
-                    current_batch.extend(current_postcode_search_data_add['index'])
-
-                if not current_postcode_ref_data_add.empty:
-                    current_ref_batch.extend(current_postcode_ref_data_add['index'])  
-
-            except:
-                #print("postcode not found: ", current_postcode)
-                pass
-
-            unique_postcodes_iterator.remove(current_postcode)
-
-        # Append the batch data to the master lists and reset lists
-        batch_indexes.append(current_batch)
-        ref_indexes.append(current_ref_batch)
-
-        current_batch_length = len(current_batch)
-        current_ref_length = len(current_ref_batch)
-
-        batch_lengths.append(current_batch_length)
-        ref_lengths.append(current_ref_length)
-
-        current_batch = []
-        current_ref_batch = []
-        current_batch_length = []
-        current_ref_length = []
-        
-    # Create df to store lengths
-    lengths_df = pd.DataFrame(data={'search_range':batch_indexes, 'ref_range':ref_indexes, 'batch_length':batch_lengths, 'ref_length':ref_lengths})
-    
-    return lengths_df
-
-
-def run_matcher(in_text:str, in_file:str, in_ref:str, data_state:PandasDataFrame, results_data_state:PandasDataFrame, ref_data_state:PandasDataFrame, in_colnames:List[str], in_refcol:List[str], in_joincol:List[str], in_existing:List[str], in_api:str, in_api_key:str, InitMatch:MatcherClass = InitMatch, progress=gr.Progress()):  
-    '''
-    Split search and reference data into batches. Loop and run through the match script for each batch of data.
-    '''
-
-    overall_tic = time.perf_counter()
-
-    # Load in initial data. This will filter to relevant addresses in the search and reference datasets that can potentially be matched, and will pull in API data if asked for.
-    InitMatch = load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state, ref_data_state, in_colnames, in_refcol, in_joincol, in_existing, InitMatch, in_api, in_api_key)
-
-    if InitMatch.search_df.empty or InitMatch.ref_df.empty:
-        out_message = "Nothing to match!"
-        print(out_message)
-        return out_message, [InitMatch.results_orig_df_name, InitMatch.match_outputs_name]
-    
-    # Run initial address preparation and standardisation processes   
-    # Prepare address format
-
-    # Polars implementation not yet finalised
-    #InitMatch.search_df = pl.from_pandas(InitMatch.search_df)
-    #InitMatch.ref_df = pl.from_pandas(InitMatch.ref_df)
-
-    
-    # Prepare all search addresses
-    if type(InitMatch.search_df) == str:
-        InitMatch.search_df_cleaned, InitMatch.search_df_key_field, InitMatch.search_address_cols = prepare_search_address_string(InitMatch.search_df)
-    else: 
-        InitMatch.search_df_cleaned = prepare_search_address(InitMatch.search_df, InitMatch.search_address_cols, InitMatch.search_postcode_col, InitMatch.search_df_key_field)
-
-        # Remove addresses that are not postal addresses
-    InitMatch.search_df_cleaned = remove_non_postal(InitMatch.search_df_cleaned, "full_address")
-
-    # Remove addresses that have no numbers in from consideration
-    InitMatch.search_df_cleaned = check_no_number_addresses(InitMatch.search_df_cleaned, "full_address")
-
-    # Initial preparation of reference addresses
-    InitMatch.ref_df_cleaned = prepare_ref_address(InitMatch.ref_df, InitMatch.ref_address_cols, InitMatch.new_join_col)
-    
-
-    # Sort dataframes by postcode - will allow for more efficient matching process if using multiple batches
-    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.sort_values(by="postcode")
-    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.sort_values(by="Postcode")
-
-    # Polars implementation - not finalised
-    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.to_pandas()
-    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.to_pandas()
-
-    # Standardise addresses    
-    # Standardise - minimal
-
-
-    tic = time.perf_counter()
-    InitMatch.search_df_after_stand, InitMatch.ref_df_after_stand = standardise_wrapper_func(
-        InitMatch.search_df_cleaned.copy(),
-        InitMatch.ref_df_cleaned.copy(),
-        standardise = False,
-        filter_to_lambeth_pcodes=filter_to_lambeth_pcodes,
-        match_task="fuzzy") # InitMatch.search_df_after_stand_series, InitMatch.ref_df_after_stand_series
-
-    toc = time.perf_counter()
-    print(f"Performed the minimal standardisation step in {toc - tic:0.1f} seconds")
-
-    # Standardise - full
-    tic = time.perf_counter()
-    InitMatch.search_df_after_full_stand, InitMatch.ref_df_after_full_stand = standardise_wrapper_func(
-        InitMatch.search_df_cleaned.copy(),
-        InitMatch.ref_df_cleaned.copy(),
-        standardise = True,
-        filter_to_lambeth_pcodes=filter_to_lambeth_pcodes,
-        match_task="fuzzy") # , InitMatch.search_df_after_stand_series_full_stand, InitMatch.ref_df_after_stand_series_full_stand
-
-    toc = time.perf_counter()
-    print(f"Performed the full standardisation step in {toc - tic:0.1f} seconds")
-
-    # Determine length of search df to create batches to send through the functions.
-    #try:
-    range_df = create_batch_ranges(InitMatch.search_df_cleaned.copy(), InitMatch.ref_df_cleaned.copy(), batch_size, ref_batch_size, "postcode", "Postcode")
-    #except:
-    #    range_df = create_simple_batch_ranges(InitMatch.search_df_cleaned, InitMatch.ref_df_cleaned, batch_size, #ref_batch_size)
-
-    print("Batches to run in this session: ", range_df)
-
-    OutputMatch = copy.copy(InitMatch)
-
-    n = 0
-    number_of_batches = range_df.shape[0]
-
-    for row in progress.tqdm(range(0,len(range_df)), desc= "Running through batches", unit="batches", total=number_of_batches):
-        print("Running batch ", str(n+1))
-
-        search_range = range_df.iloc[row]['search_range']
-        ref_range = range_df.iloc[row]['ref_range']
-
-        #print("search_range: ", search_range)
-        #pd.DataFrame(search_range).to_csv("search_range.csv")
-        #print("ref_range: ", ref_range)
-        
-        BatchMatch = copy.copy(InitMatch)
-
-        # Subset the search and reference dfs based on current batch ranges
-        # BatchMatch.search_df = BatchMatch.search_df.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
-        # BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df = BatchMatch.ref_df.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-
-
-        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.iloc[search_range[0]:search_range[1] + 1]
-        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.iloc[ref_range[0]:ref_range[1] + 1]
-        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.iloc[search_range[0]:search_range[1] + 1]
-        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.iloc[ref_range[0]:ref_range[1] + 1]
-
-        # BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
-        # BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
-
-        BatchMatch.search_df = BatchMatch.search_df[BatchMatch.search_df.index.isin(search_range)].reset_index(drop=True)
-        BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
-        BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned[BatchMatch.search_df_cleaned.index.isin(search_range)].reset_index(drop=True)
-
-        BatchMatch.ref_df = BatchMatch.ref_df[BatchMatch.ref_df.index.isin(ref_range)].reset_index(drop=True)
-        BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned[BatchMatch.ref_df_cleaned.index.isin(ref_range)].reset_index(drop=True)
-
-        # Dataframes after standardisation process
-        BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand[BatchMatch.search_df_after_stand.index.isin(search_range)].reset_index(drop=True)
-        BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand[BatchMatch.search_df_after_full_stand.index.isin(search_range)].reset_index(drop=True)
-
-        ### Create lookup lists for fuzzy matches
-        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand.copy().set_index('postcode_search')['search_address_stand']
-        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_full_stand.copy().set_index('postcode_search')['search_address_stand']
-        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.sort_index()
-        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.sort_index()
-
-        #BatchMatch.search_df_after_stand.reset_index(inplace=True, drop = True)
-        #BatchMatch.search_df_after_full_stand.reset_index(inplace=True, drop = True)
-
-        BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand[BatchMatch.ref_df_after_stand.index.isin(ref_range)].reset_index(drop=True)
-        BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand[BatchMatch.ref_df_after_full_stand.index.isin(ref_range)].reset_index(drop=True)
-
-        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand.copy().set_index('postcode_search')['ref_address_stand']
-        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_full_stand.copy().set_index('postcode_search')['ref_address_stand']
-        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.sort_index()
-        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.sort_index()
-
-        # BatchMatch.ref_df_after_stand.reset_index(inplace=True, drop=True)
-        # BatchMatch.ref_df_after_full_stand.reset_index(inplace=True, drop=True)
-
-        # Match the data, unless the search or reference dataframes are empty
-        if BatchMatch.search_df.empty or BatchMatch.ref_df.empty:
-            out_message = "Nothing to match for batch: " + str(n)
-            print(out_message)
-            BatchMatch_out = BatchMatch
-            BatchMatch_out.results_on_orig_df = pd.DataFrame(data={"index":BatchMatch.search_df.index,
-                                                                   "Excluded from search":False,
-                                                                    "Matched with reference address":False})
-        else:
-            summary_of_summaries, BatchMatch_out = run_match_batch(BatchMatch, n, number_of_batches)
-        
-        print("BatchMatch_out match shape: ", BatchMatch_out.results_on_orig_df.shape)
-
-        OutputMatch = combine_two_matches(OutputMatch, BatchMatch_out, "All up to and including batch " + str(n+1))
-
-        print("Output results match shape: ", OutputMatch.results_on_orig_df.shape)
-
-        n += 1
-
-    if in_api==True:
-        OutputMatch.results_on_orig_df['Matched with reference address'] = OutputMatch.results_on_orig_df['Matched with reference address'].replace({1:True, 0:False})
-        OutputMatch.results_on_orig_df['Excluded from search'] = OutputMatch.results_on_orig_df['Excluded from search'].replace('nan', False).fillna(False)
-
-    # Remove any duplicates from reference df, prioritise successful matches
-    OutputMatch.results_on_orig_df = OutputMatch.results_on_orig_df.sort_values(by=["index", "Matched with reference address"], ascending=[True,False]).drop_duplicates(subset="index")
-
-
-    overall_toc = time.perf_counter()
-    time_out = f"The overall match (all batches) took {overall_toc - overall_tic:0.1f} seconds"
-
-    print(OutputMatch.output_summary)
-
-    if OutputMatch.output_summary == "":
-        OutputMatch.output_summary = "No matches were found."
-
-    fuzzy_not_std_output = OutputMatch.match_results_output.copy()
-    fuzzy_not_std_output_mask = ~(fuzzy_not_std_output["match_method"].str.contains("Fuzzy match")) | (fuzzy_not_std_output["standardised_address"] == True)
-    fuzzy_not_std_output.loc[fuzzy_not_std_output_mask, "full_match"] = False
-    fuzzy_not_std_summary = create_match_summary(fuzzy_not_std_output, "Fuzzy not standardised")
-
-    fuzzy_std_output = OutputMatch.match_results_output.copy()
-    fuzzy_std_output_mask = fuzzy_std_output["match_method"].str.contains("Fuzzy match")
-    fuzzy_std_output.loc[fuzzy_std_output_mask == False, "full_match"] = False
-    fuzzy_std_summary = create_match_summary(fuzzy_std_output, "Fuzzy standardised")
-
-    nnet_std_output = OutputMatch.match_results_output.copy()
-    nnet_std_summary = create_match_summary(nnet_std_output, "Neural net standardised")
-
-    final_summary = fuzzy_not_std_summary + "\n" + fuzzy_std_summary + "\n" + nnet_std_summary + "\n" + time_out
-
-    return final_summary, [OutputMatch.results_orig_df_name, OutputMatch.match_outputs_name]
-
 # Create the gradio interface
-
 block = gr.Blocks(theme = gr.themes.Base())
 
 with block:
@@ -401,8 +66,8 @@ with block:
         in_ref = gr.File(label="Input reference addresses from file", file_count= "multiple")
 
         with gr.Accordion("Use Addressbase API instead of reference file", open = False):
-            in_api = gr.Dropdown(label="Choose API type", multiselect=False, value=None, choices=["Postcode", "UPRN"]) #choices=["Address", "Postcode", "UPRN"])
-            in_api_key = gr.Textbox(label="Addressbase API key")
+            in_api = gr.Dropdown(label="Choose API type", multiselect=False, value=None, choices=["Postcode"])#["Postcode", "UPRN"]) #choices=["Address", "Postcode", "UPRN"])
+            in_api_key = gr.Textbox(label="Addressbase API key", type='password')
         
         with gr.Accordion("Custom reference file format or join columns (i.e. not LLPG LPI format)", open = False):
             in_refcol = gr.Dropdown(choices=[], multiselect=True, label="Select columns that make up the reference address. Make sure postcode is at the end")
@@ -439,7 +104,7 @@ with block:
 #block.queue().launch(debug=True) # root_path="/address-match", debug=True, server_name="0.0.0.0",
 
 # Simple run for AWS server
-block.queue().launch(ssl_verify=False) # root_path="/address-match", debug=True, server_name="0.0.0.0", server_port=7861
+block.queue().launch(ssl_verify=False, inbrowser=True) # root_path="/address-match", debug=True, server_name="0.0.0.0", server_port=7861
 
 # Download OpenSSL from here: 
 # Running on local server with https: https://discuss.huggingface.co/t/how-to-run-gradio-with-0-0-0-0-and-https/38003 or https://dev.to/rajshirolkar/fastapi-over-https-for-development-on-windows-2p7d

tools/fuzzy_match.py

@@ -169,7 +169,11 @@ def string_match_by_post_code_multiple(match_address_series:PandasSeries, refere
 
     return out_frame
 
-def _create_fuzzy_match_results_output(results, search_df_after_stand, ref_df_cleaned, ref_df_after_stand, fuzzy_match_limit, search_df_cleaned, search_df_key_field, new_join_col, standardise, blocker_col):
+def _create_fuzzy_match_results_output(results:PandasDataFrame, search_df_after_stand:PandasDataFrame, ref_df_cleaned:PandasDataFrame, ref_df_after_stand:PandasDataFrame, fuzzy_match_limit:int, search_df_cleaned:PandasDataFrame, search_df_key_field:str, new_join_col:str, standardise:bool, blocker_col:str):
+
+        '''
+        Take fuzzy match outputs, create shortlist dataframes, rearrange, return diagnostics and shortlist dataframes for export
+        '''
 
         ## Diagnostics
 
@@ -227,7 +231,7 @@ def _create_fuzzy_match_results_output(results, search_df_after_stand, ref_df_cl
                 
         return match_results_output, diag_shortlist, diag_best_match
 
-def create_diag_shortlist(results_df, matched_col, fuzzy_match_limit, blocker_col, fuzzy_col="fuzzy_score", search_mod_address = "search_mod_address", resolve_tie_breaks=True, no_number_fuzzy_match_limit=no_number_fuzzy_match_limit):
+def create_diag_shortlist(results_df:PandasDataFrame, matched_col:str, fuzzy_match_limit:int, blocker_col:str, fuzzy_col:str="fuzzy_score", search_mod_address:str = "search_mod_address", resolve_tie_breaks:bool=True, no_number_fuzzy_match_limit:int=no_number_fuzzy_match_limit) -> PandasDataFrame:
     '''
     Create a shortlist of the best matches from a list of suggested matches
     '''

tools/matcher_funcs.py

@@ -24,8 +24,10 @@ run_fuzzy_match = True
 run_nnet_match = True
 run_standardise = True
 
-from tools.preparation import prepare_search_address_string, prepare_search_address,  extract_street_name 
+from tools.constants import *
+from tools.preparation import prepare_search_address_string, prepare_search_address,  extract_street_name, prepare_ref_address, remove_non_postal, check_no_number_addresses
 from tools.fuzzy_match import string_match_by_post_code_multiple, _create_fuzzy_match_results_output, join_to_orig_df
+from tools.standardise import standardise_wrapper_func
 
 # Neural network functions
 ### Predict function for imported model
@@ -64,17 +66,17 @@ def read_file(filename:str) -> PandasDataFrame:
     elif file_type == 'parquet':
         return pd.read_parquet(filename)
 
-def get_file_name(in_name:str) -> str:
-    '''
-    Get the name of a file from a string using the re package.
-    '''
+def get_file_name(in_name: str) -> str:
+    """Get the name of a file from a string, handling both Windows and Unix paths."""
 
-    # Corrected regex pattern
-    match = re.search(r'\\(?!.*\\)(.*)', in_name)
+    print("in_name: ", in_name)
+    match = re.search(rf'{os.sep}(?!.*{os.sep})(.*)', in_name) 
     if match:
         matched_result = match.group(1)
     else:
         matched_result = None
+
+    print("Matched result: ", matched_result)
     
     return matched_result
 
@@ -492,24 +494,22 @@ def check_match_data_filter(Matcher:MatcherClass, data_state:PandasDataFrame, re
     if not data_state.empty:
         
         Matcher.search_df = data_state        
-
         Matcher.search_df['index'] = Matcher.search_df.reset_index().index
 
     else:        
         Matcher.search_df = pd.DataFrame()       
 
-    # If someone has just entered open text, just load this instead
+    # If a single address entered into the text box, just load this instead
     if in_text:
         Matcher.search_df, Matcher.search_df_key_field, Matcher.search_address_cols, Matcher.search_postcode_col = prepare_search_address_string(in_text) 
 
-    # If two matcher files are loaded in, the algorithm will combine them together
+    # If no file loaded yet and a file has been selected
     if Matcher.search_df.empty and in_file:
         output_message, drop1, drop2, Matcher.search_df, results_data_state = initial_data_load(in_file)
 
         file_list = [string.name for string in in_file]
-        data_file_names = [string for string in file_list if "results_on_orig" not in string.lower()]
-        
-        #print("Data file names: ", data_file_names)
+        data_file_names = [string for string in file_list if "results_" not in string.lower()]
+
         Matcher.file_name = get_file_name(data_file_names[0])
         
         # search_df makes column to use as index
@@ -524,20 +524,11 @@ def check_match_data_filter(Matcher:MatcherClass, data_state:PandasDataFrame, re
         Matcher.search_df = Matcher.search_df.merge(results_data_state, on = "index", how = "left") 
 
     # If no join on column suggested, assume the user wants the UPRN
-    print("in_joincol: ", in_joincol)
-
     if not in_joincol:
         Matcher.new_join_col = ['UPRN']
-        #Matcher.new_join_col = Matcher.new_join_col#[0]
         
     else:  
         Matcher.new_join_col = in_joincol
-        #Matcher.new_join_col = Matcher.new_join_col
-
-    # Extract the column names from the input data
-    #print("In colnames: ", in_colnames)
-
-    print("Matcher.in_joincol: ", Matcher.new_join_col)
 
     if len(in_colnames) > 1:
         Matcher.search_postcode_col = [in_colnames[-1]]
@@ -566,7 +557,6 @@ def check_match_data_filter(Matcher:MatcherClass, data_state:PandasDataFrame, re
 
     length_more_than_0 = Matcher.search_df["address_cols_joined"].str.len() > 0
 
-
     ### Filter addresses to match to postcode areas present in both search_df and ref_df_cleaned only (postcode without the last three characters). Only run if API call is false. When the API is called, relevant addresses and postcodes should be brought in by the API.
     if not in_api:
         if Matcher.filter_to_lambeth_pcodes == True:
@@ -621,7 +611,6 @@ def check_match_data_filter(Matcher:MatcherClass, data_state:PandasDataFrame, re
 
     Matcher.search_df_not_matched = Matcher.search_df
 
-
     # If this is for an API call, we need to convert the search_df address columns to one column now. This is so the API call can be made and the reference dataframe created.
     if in_api:
 
@@ -629,10 +618,14 @@ def check_match_data_filter(Matcher:MatcherClass, data_state:PandasDataFrame, re
             output_message, drop1, drop2, df, results_data_state = initial_data_load(in_file)
 
             file_list = [string.name for string in in_file]
-            data_file_names = [string for string in file_list if "results_on_orig" not in string.lower()]
+            data_file_names = [string for string in file_list if "results_" not in string.lower()]
         
             Matcher.file_name = get_file_name(data_file_names[0])
 
+            print("File list in in_api bit: ", file_list)
+            print("data_file_names in in_api bit: ", data_file_names)
+            print("Matcher.file_name in in_api bit: ", Matcher.file_name)
+
         else:
             if in_text:
                 Matcher.file_name = in_text
@@ -654,8 +647,6 @@ def check_match_data_filter(Matcher:MatcherClass, data_state:PandasDataFrame, re
 
 
         Matcher.search_df['full_address_postcode'] = search_df_cleaned["full_address"]
-        #Matcher.search_df = Matcher.search_df.reset_index(drop=True)
-        #Matcher.search_df.index.name = 'index'
 
     return Matcher
 
@@ -677,15 +668,11 @@ def load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state,
         # If doing API calls, we need to know the search data before querying for specific addresses/postcodes
         Matcher = check_match_data_filter(Matcher, data_state, results_data_state, in_file, in_text, in_colnames, in_joincol, in_existing, in_api)
 
-
         # If an API call, ref_df data is loaded after
         if in_api:
+            
             Matcher = check_ref_data_exists(Matcher, ref_data_state, in_ref, in_refcol, in_api, in_api_key, query_type=in_api)
             
-            #print("Resetting index.")
-            # API-called data will often have duplicate indexes in it - drop them to avoid conflicts down the line
-            #Matcher.ref_df = Matcher.ref_df.reset_index(drop = True)
-
         print("Shape of ref_df after filtering is: ", Matcher.ref_df.shape)
         print("Shape of search_df after filtering is: ", Matcher.search_df.shape)
     
@@ -697,7 +684,328 @@ def load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state,
         
         return Matcher
 
+# Run whole matcher process
+def run_matcher(in_text:str, in_file:str, in_ref:str, data_state:PandasDataFrame, results_data_state:PandasDataFrame, ref_data_state:PandasDataFrame, in_colnames:List[str], in_refcol:List[str], in_joincol:List[str], in_existing:List[str], in_api:str, in_api_key:str, InitMatch:MatcherClass = InitMatch, progress=gr.Progress()):  
+    '''
+    Split search and reference data into batches. Loop and run through the match script for each batch of data.
+    '''
+
+    overall_tic = time.perf_counter()
+
+    # Load in initial data. This will filter to relevant addresses in the search and reference datasets that can potentially be matched, and will pull in API data if asked for.
+    InitMatch = load_matcher_data(in_text, in_file, in_ref, data_state, results_data_state, ref_data_state, in_colnames, in_refcol, in_joincol, in_existing, InitMatch, in_api, in_api_key)
+
+    if InitMatch.search_df.empty or InitMatch.ref_df.empty:
+        out_message = "Nothing to match!"
+        print(out_message)
+        return out_message, [InitMatch.results_orig_df_name, InitMatch.match_outputs_name]
+    
+    # Run initial address preparation and standardisation processes   
+    # Prepare address format
+
+    # Polars implementation not yet finalised
+    #InitMatch.search_df = pl.from_pandas(InitMatch.search_df)
+    #InitMatch.ref_df = pl.from_pandas(InitMatch.ref_df)
+
+    
+    # Prepare all search addresses
+    if type(InitMatch.search_df) == str:
+        InitMatch.search_df_cleaned, InitMatch.search_df_key_field, InitMatch.search_address_cols = prepare_search_address_string(InitMatch.search_df)
+    else: 
+        InitMatch.search_df_cleaned = prepare_search_address(InitMatch.search_df, InitMatch.search_address_cols, InitMatch.search_postcode_col, InitMatch.search_df_key_field)
+
+        # Remove addresses that are not postal addresses
+    InitMatch.search_df_cleaned = remove_non_postal(InitMatch.search_df_cleaned, "full_address")
+
+    # Remove addresses that have no numbers in from consideration
+    InitMatch.search_df_cleaned = check_no_number_addresses(InitMatch.search_df_cleaned, "full_address")
+
+    # Initial preparation of reference addresses
+    InitMatch.ref_df_cleaned = prepare_ref_address(InitMatch.ref_df, InitMatch.ref_address_cols, InitMatch.new_join_col)
+    
+
+    # Sort dataframes by postcode - will allow for more efficient matching process if using multiple batches
+    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.sort_values(by="postcode")
+    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.sort_values(by="Postcode")
+
+    # Polars implementation - not finalised
+    #InitMatch.search_df_cleaned = InitMatch.search_df_cleaned.to_pandas()
+    #InitMatch.ref_df_cleaned = InitMatch.ref_df_cleaned.to_pandas()
+
+    # Standardise addresses    
+    # Standardise - minimal
+
+
+    tic = time.perf_counter()
+    InitMatch.search_df_after_stand, InitMatch.ref_df_after_stand = standardise_wrapper_func(
+        InitMatch.search_df_cleaned.copy(),
+        InitMatch.ref_df_cleaned.copy(),
+        standardise = False,
+        filter_to_lambeth_pcodes=filter_to_lambeth_pcodes,
+        match_task="fuzzy") # InitMatch.search_df_after_stand_series, InitMatch.ref_df_after_stand_series
+
+    toc = time.perf_counter()
+    print(f"Performed the minimal standardisation step in {toc - tic:0.1f} seconds")
+
+    # Standardise - full
+    tic = time.perf_counter()
+    InitMatch.search_df_after_full_stand, InitMatch.ref_df_after_full_stand = standardise_wrapper_func(
+        InitMatch.search_df_cleaned.copy(),
+        InitMatch.ref_df_cleaned.copy(),
+        standardise = True,
+        filter_to_lambeth_pcodes=filter_to_lambeth_pcodes,
+        match_task="fuzzy") # , InitMatch.search_df_after_stand_series_full_stand, InitMatch.ref_df_after_stand_series_full_stand
+
+    toc = time.perf_counter()
+    print(f"Performed the full standardisation step in {toc - tic:0.1f} seconds")
+
+    # Determine length of search df to create batches to send through the functions.
+    #try:
+    range_df = create_batch_ranges(InitMatch.search_df_cleaned.copy(), InitMatch.ref_df_cleaned.copy(), batch_size, ref_batch_size, "postcode", "Postcode")
+    #except:
+    #    range_df = create_simple_batch_ranges(InitMatch.search_df_cleaned, InitMatch.ref_df_cleaned, batch_size, #ref_batch_size)
+
+    print("Batches to run in this session: ", range_df)
+
+    OutputMatch = copy.copy(InitMatch)
+
+    n = 0
+    number_of_batches = range_df.shape[0]
+
+    for row in progress.tqdm(range(0,len(range_df)), desc= "Running through batches", unit="batches", total=number_of_batches):
+        print("Running batch ", str(n+1))
+
+        search_range = range_df.iloc[row]['search_range']
+        ref_range = range_df.iloc[row]['ref_range']
+
+        #print("search_range: ", search_range)
+        #pd.DataFrame(search_range).to_csv("search_range.csv")
+        #print("ref_range: ", ref_range)
+        
+        BatchMatch = copy.copy(InitMatch)
+
+        # Subset the search and reference dfs based on current batch ranges
+        # BatchMatch.search_df = BatchMatch.search_df.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
+        # BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df = BatchMatch.ref_df.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+
+
+        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.iloc[search_range[0]:search_range[1] + 1]
+        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.iloc[ref_range[0]:ref_range[1] + 1]
+        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.iloc[search_range[0]:search_range[1] + 1]
+        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.iloc[ref_range[0]:ref_range[1] + 1]
+
+        # BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand.iloc[search_range[0]:search_range[1] + 1,:].reset_index(drop=True)
+        # BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand.iloc[ref_range[0]:ref_range[1] + 1,:].reset_index(drop=True)
+
+        BatchMatch.search_df = BatchMatch.search_df[BatchMatch.search_df.index.isin(search_range)].reset_index(drop=True)
+        BatchMatch.search_df_not_matched = BatchMatch.search_df.copy()
+        BatchMatch.search_df_cleaned = BatchMatch.search_df_cleaned[BatchMatch.search_df_cleaned.index.isin(search_range)].reset_index(drop=True)
+
+        BatchMatch.ref_df = BatchMatch.ref_df[BatchMatch.ref_df.index.isin(ref_range)].reset_index(drop=True)
+        BatchMatch.ref_df_cleaned = BatchMatch.ref_df_cleaned[BatchMatch.ref_df_cleaned.index.isin(ref_range)].reset_index(drop=True)
+
+        # Dataframes after standardisation process
+        BatchMatch.search_df_after_stand = BatchMatch.search_df_after_stand[BatchMatch.search_df_after_stand.index.isin(search_range)].reset_index(drop=True)
+        BatchMatch.search_df_after_full_stand = BatchMatch.search_df_after_full_stand[BatchMatch.search_df_after_full_stand.index.isin(search_range)].reset_index(drop=True)
+
+        ### Create lookup lists for fuzzy matches
+        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand.copy().set_index('postcode_search')['search_address_stand']
+        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_full_stand.copy().set_index('postcode_search')['search_address_stand']
+        # BatchMatch.search_df_after_stand_series = BatchMatch.search_df_after_stand_series.sort_index()
+        # BatchMatch.search_df_after_stand_series_full_stand = BatchMatch.search_df_after_stand_series_full_stand.sort_index()
+
+        #BatchMatch.search_df_after_stand.reset_index(inplace=True, drop = True)
+        #BatchMatch.search_df_after_full_stand.reset_index(inplace=True, drop = True)
+
+        BatchMatch.ref_df_after_stand = BatchMatch.ref_df_after_stand[BatchMatch.ref_df_after_stand.index.isin(ref_range)].reset_index(drop=True)
+        BatchMatch.ref_df_after_full_stand = BatchMatch.ref_df_after_full_stand[BatchMatch.ref_df_after_full_stand.index.isin(ref_range)].reset_index(drop=True)
+
+        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand.copy().set_index('postcode_search')['ref_address_stand']
+        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_full_stand.copy().set_index('postcode_search')['ref_address_stand']
+        # BatchMatch.ref_df_after_stand_series = BatchMatch.ref_df_after_stand_series.sort_index()
+        # BatchMatch.ref_df_after_stand_series_full_stand = BatchMatch.ref_df_after_stand_series_full_stand.sort_index()
+
+        # BatchMatch.ref_df_after_stand.reset_index(inplace=True, drop=True)
+        # BatchMatch.ref_df_after_full_stand.reset_index(inplace=True, drop=True)
+
+        # Match the data, unless the search or reference dataframes are empty
+        if BatchMatch.search_df.empty or BatchMatch.ref_df.empty:
+            out_message = "Nothing to match for batch: " + str(n)
+            print(out_message)
+            BatchMatch_out = BatchMatch
+            BatchMatch_out.results_on_orig_df = pd.DataFrame(data={"index":BatchMatch.search_df.index,
+                                                                   "Excluded from search":False,
+                                                                    "Matched with reference address":False})
+        else:
+            summary_of_summaries, BatchMatch_out = run_match_batch(BatchMatch, n, number_of_batches)
+
+        OutputMatch = combine_two_matches(OutputMatch, BatchMatch_out, "All up to and including batch " + str(n+1))
+
+        n += 1
+
+    if in_api==True:
+        OutputMatch.results_on_orig_df['Matched with reference address'] = OutputMatch.results_on_orig_df['Matched with reference address'].replace({1:True, 0:False})
+        OutputMatch.results_on_orig_df['Excluded from search'] = OutputMatch.results_on_orig_df['Excluded from search'].replace('nan', False).fillna(False)
+
+    # Remove any duplicates from reference df, prioritise successful matches
+    OutputMatch.results_on_orig_df = OutputMatch.results_on_orig_df.sort_values(by=["index", "Matched with reference address"], ascending=[True,False]).drop_duplicates(subset="index")
+
+    overall_toc = time.perf_counter()
+    time_out = f"The overall match (all batches) took {overall_toc - overall_tic:0.1f} seconds"
+
+    print(OutputMatch.output_summary)
+
+    if OutputMatch.output_summary == "":
+        OutputMatch.output_summary = "No matches were found."
+
+    fuzzy_not_std_output = OutputMatch.match_results_output.copy()
+    fuzzy_not_std_output_mask = ~(fuzzy_not_std_output["match_method"].str.contains("Fuzzy match")) | (fuzzy_not_std_output["standardised_address"] == True)
+    fuzzy_not_std_output.loc[fuzzy_not_std_output_mask, "full_match"] = False
+    fuzzy_not_std_summary = create_match_summary(fuzzy_not_std_output, "Fuzzy not standardised")
+
+    fuzzy_std_output = OutputMatch.match_results_output.copy()
+    fuzzy_std_output_mask = fuzzy_std_output["match_method"].str.contains("Fuzzy match")
+    fuzzy_std_output.loc[fuzzy_std_output_mask == False, "full_match"] = False
+    fuzzy_std_summary = create_match_summary(fuzzy_std_output, "Fuzzy standardised")
+
+    nnet_std_output = OutputMatch.match_results_output.copy()
+    nnet_std_summary = create_match_summary(nnet_std_output, "Neural net standardised")
+
+    final_summary = fuzzy_not_std_summary + "\n" + fuzzy_std_summary + "\n" + nnet_std_summary + "\n" + time_out
+
+    return final_summary, [OutputMatch.results_orig_df_name, OutputMatch.match_outputs_name]
+
 # Run a match run for a single batch
+def create_simple_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:int, ref_batch_size:int):
+    #print("Search df batch size: ", batch_size)
+    #print("ref_df df batch size: ", ref_batch_size)
+
+    total_rows = df.shape[0]
+    ref_total_rows = ref_df.shape[0]
+
+    # Creating bottom and top limits for search data
+    search_ranges = []
+    for start in range(0, total_rows, batch_size):
+        end = min(start + batch_size - 1, total_rows - 1)  # Adjusted to get the top limit
+        search_ranges.append((start, end))
+
+    # Creating bottom and top limits for reference data
+    ref_ranges = []
+    for start in range(0, ref_total_rows, ref_batch_size):
+        end = min(start + ref_batch_size - 1, ref_total_rows - 1)  # Adjusted to get the top limit
+        ref_ranges.append((start, end))
+
+    # Create DataFrame with combinations of search_range and ref_range
+    result_data = []
+    for search_range in search_ranges:
+        for ref_range in ref_ranges:
+            result_data.append((search_range, ref_range))
+
+    range_df = pd.DataFrame(result_data, columns=['search_range', 'ref_range'])
+
+    return range_df
+
+def create_batch_ranges(df:PandasDataFrame, ref_df:PandasDataFrame, batch_size:int, ref_batch_size:int, search_postcode_col:str, ref_postcode_col:str):
+    '''
+    Create batches of address indexes for search and reference dataframes based on shortened postcodes.
+    '''
+
+    # If df sizes are smaller than the batch size limits, no need to run through everything
+    if len(df) < batch_size and len(ref_df) < ref_batch_size:
+        print("Dataframe sizes are smaller than maximum batch sizes, no need to split data.")
+        lengths_df = pd.DataFrame(data={'search_range':[df.index.tolist()], 'ref_range':[ref_df.index.tolist()], 'batch_length':len(df), 'ref_length':len(ref_df)})
+        return lengths_df
+    
+    #df.index = df[search_postcode_col]
+
+    df['index'] = df.index
+    ref_df['index'] = ref_df.index    
+
+    # Remove the last character of postcode
+    df['postcode_minus_last_character'] = df[search_postcode_col].str.lower().str.strip().str.replace("\s+", "", regex=True).str[:-1]
+    ref_df['postcode_minus_last_character'] = ref_df[ref_postcode_col].str.lower().str.strip().str.replace("\s+", "", regex=True).str[:-1]
+
+    unique_postcodes = df['postcode_minus_last_character'][df['postcode_minus_last_character'].str.len()>=4].unique().tolist()
+
+    df = df.set_index('postcode_minus_last_character')
+    ref_df = ref_df.set_index('postcode_minus_last_character')
+
+    df = df.sort_index()
+    ref_df = ref_df.sort_index()
+
+    #df.to_csv("batch_search_df.csv")
+
+    # Overall batch variables
+    batch_indexes = []
+    ref_indexes = []
+    batch_lengths = []
+    ref_lengths = []
+
+    # Current batch variables for loop
+    current_batch = []
+    current_ref_batch = []
+    current_batch_length = []
+    current_ref_length = []
+
+    unique_postcodes_iterator = unique_postcodes.copy()
+
+    while unique_postcodes_iterator:
+        
+        unique_postcodes_loop = unique_postcodes_iterator.copy()
+
+        #print("Current loop postcodes: ", unique_postcodes_loop)
+
+        for current_postcode in unique_postcodes_loop:
+            
+
+
+            if len(current_batch) >= batch_size or len(current_ref_batch) >= ref_batch_size:
+                print("Batch length reached - breaking")
+                break
+            
+            try:
+                current_postcode_search_data_add = df.loc[[current_postcode]]#[df['postcode_minus_last_character'].isin(current_postcode)]
+                current_postcode_ref_data_add = ref_df.loc[[current_postcode]]#[ref_df['postcode_minus_last_character'].isin(current_postcode)]
+
+                #print(current_postcode_search_data_add)
+
+                if not current_postcode_search_data_add.empty:
+                    current_batch.extend(current_postcode_search_data_add['index'])
+
+                if not current_postcode_ref_data_add.empty:
+                    current_ref_batch.extend(current_postcode_ref_data_add['index'])  
+
+            except:
+                #print("postcode not found: ", current_postcode)
+                pass
+
+            unique_postcodes_iterator.remove(current_postcode)
+
+        # Append the batch data to the master lists and reset lists
+        batch_indexes.append(current_batch)
+        ref_indexes.append(current_ref_batch)
+
+        current_batch_length = len(current_batch)
+        current_ref_length = len(current_ref_batch)
+
+        batch_lengths.append(current_batch_length)
+        ref_lengths.append(current_ref_length)
+
+        current_batch = []
+        current_ref_batch = []
+        current_batch_length = []
+        current_ref_length = []
+        
+    # Create df to store lengths
+    lengths_df = pd.DataFrame(data={'search_range':batch_indexes, 'ref_range':ref_indexes, 'batch_length':batch_lengths, 'ref_length':ref_lengths})
+    
+    return lengths_df
+
 def run_match_batch(InitialMatch:MatcherClass, batch_n:int, total_batches:int, progress=gr.Progress()):
     '''
     Over-arching function for running a single batch of data through the full matching process. Calls fuzzy matching, then neural network match functions in order. It outputs a summary of the match, and a MatcherClass with the matched data included.
@@ -721,12 +1029,7 @@ def run_match_batch(InitialMatch:MatcherClass, batch_n:int, total_batches:int, p
             print(message)
             return message, InitialMatch
         
-        print("FuzzyNotStdMatch shape before combine two matches: ", FuzzyNotStdMatch.results_on_orig_df.shape)
-
         FuzzyNotStdMatch = combine_two_matches(InitialMatch, FuzzyNotStdMatch, df_name)
-
-        print("InitialMatch shape: ", InitialMatch.results_on_orig_df.shape)
-        print("FuzzyNotStdMatch shape: ", FuzzyNotStdMatch.results_on_orig_df.shape)
         
         if (len(FuzzyNotStdMatch.search_df_not_matched) == 0) | (sum(FuzzyNotStdMatch.match_results_output[FuzzyNotStdMatch.match_results_output['full_match']==False]['fuzzy_score'])==0): 
             overall_toc = time.perf_counter()