Fixes, Download button

app.py

@@ -1,9 +1,11 @@
+import math
 import streamlit as st
 import pandas as pd
 import numpy as np
 import torch
 from transformers import AlbertTokenizer, AlbertModel
 from sklearn.metrics.pairwise import cosine_similarity
+from io import BytesIO
 
 # base is smaller, vs large
 model_size='base'
@@ -20,7 +22,7 @@ def get_embedding(input_text):
 
     list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
 
-    print( "Tokens : " + ' '.join(list_of_tokens))
+    #print( "Tokens : " + ' '.join(list_of_tokens))
     with torch.no_grad():
 
         outputs = model(**encoded_input)
@@ -45,11 +47,16 @@ if uploaded_file is not None:
     # Data cleaning CAQH
     data_caqh['postalcode'] = data_caqh['postalcode'].astype(str).apply(lambda x: x[:5] + '-' + x[5:] if len(x) > 5 and not '-' in x else x)
     data_caqh['full-addr'] = data_caqh['address1'].astype(str) + ', ' \
-                             + np.where(data_caqh['address2'].isnull(),  '' , data_caqh['address2'].astype(str))  \
+                             + np.where(data_caqh['address2'].isnull(),  '' , data_caqh['address2'].astype(str)+ ', ')  \
                              + data_caqh['city'].astype(str) + ', '\
                              + data_caqh['state'].astype(str) + ', ' \
                              + data_caqh['postalcode'].astype(str)
 
+    st.write(f"CAQH before duplicate removal {len(data_caqh)}")
+    data_caqh.drop_duplicates(subset='full-addr',inplace=True)
+    data_caqh = data_caqh.reset_index(drop=True) # reset the index.
+    st.write(f"CAQH after duplicate removal {len(data_caqh)}")
+
     # Data cleaning NDB
     data_ndb['zip_pls_4_cd'] = data_ndb['zip_pls_4_cd'].astype(str).apply(lambda x: x if (x[-1] != '0' and x[-1] != '1') else '')
 
@@ -58,15 +65,75 @@ if uploaded_file is not None:
                                       + data_ndb['zip_pls_4_cd'].astype(str))
 
     data_ndb['full-addr'] = data_ndb['adr_ln_1_txt'].astype(str).str.strip() + ', ' \
-                            + data_ndb['cty_nm'].astype(str) + ', ' \
+                            + data_ndb['cty_nm'].astype(str).str.strip() + ', ' \
                             + data_ndb['st_cd'].astype(str) + ', ' + data_ndb['zip_cd_zip_pls_4_cd']
 
-    # App
-    data_caqh['embedding'] = data_caqh['full-addr'].apply(get_embedding)
-    data_ndb['embedding'] = data_ndb['full-addr'].apply(get_embedding)
+    # Calculate similarity For CAQH
+    num_items = len(data_caqh)
+    progress_bar = st.progress(0)
+    total_steps = 100
+    step_size = math.ceil(num_items / total_steps)
+
+    data_caqh['embedding'] = 0
+
+    embedding_col_index = data_caqh.columns.get_loc('embedding')
+    full_addr_col_index = data_caqh.columns.get_loc('full-addr')
+    for i in range(total_steps):
+        # Update progress bar
+        progress = (i + 1) / total_steps
+
+
+        # Process a batch of rows
+        start = i * step_size
+        end = start + step_size
+
+        stop_iter = False
+        if end >= num_items:
+            end = num_items
+            stop_iter = True
+
+        data_caqh.iloc[start:end, embedding_col_index]  = data_caqh.iloc[start:end, full_addr_col_index].apply(get_embedding)
+
+        progress_bar.progress(value=progress, text=f"CAQH embeddings: {(i + 1) * step_size} processed out of {num_items}")
+
+        if stop_iter:
+            break
+
+    st.write(f"Embeddings for CAQH calculated")
+    # Calculate similarity For NDB
+    num_items = len(data_ndb)
+    progress_bar = st.progress(0)
+    total_steps = 100
+    step_size = math.ceil(num_items / total_steps)
+
+    data_ndb['embedding'] = 0
 
-    data_caqh['matched-addr'] = ''
+    embedding_col_index = data_ndb.columns.get_loc('embedding')
+    full_addr_col_index = data_ndb.columns.get_loc('full-addr')
+    for i in range(total_steps):
+        # Update progress bar
+        progress = (i + 1) / total_steps
 
+        # Process a batch of rows
+        start = i * step_size
+        end = start + step_size
+
+        stop_iter = False
+        if end >= num_items:
+            end = num_items
+            stop_iter = True
+
+        data_ndb.iloc[start:end, embedding_col_index]  = data_ndb.iloc[start:end, full_addr_col_index].apply(get_embedding)
+
+        progress_bar.progress(value=progress, text=f"NDB embeddings: {(i + 1) * step_size} processed out of {num_items}")
+
+        if stop_iter:
+            break
+
+    st.write(f"Embeddings for NDB calculated... matching")
+
+    progress_bar = st.progress(0)
+    num_items = len(data_caqh)
     for i, row in data_caqh.iterrows():
         max_similarity = 0
         matched_row = None
@@ -81,22 +148,40 @@ if uploaded_file is not None:
         else:
             data_caqh.at[i, 'matched-addr'] = 'No Matches'
 
+        progress = i / num_items
+        if progress > 1.0:
+            progress = 1.0
+        progress_bar.progress(value=progress, text=f"matching similarities - {i} done out of {num_items}")
+
     # Drop columns not needed for display
     data_caqh.drop(columns=['embedding'], inplace=True)
     data_ndb.drop(columns=['embedding'], inplace=True)
 
     st.header('CAQH addresses and matches')
     st.dataframe(data_caqh, use_container_width=True)
+
+    # Create an in-memory binary stream
+    output = BytesIO()
+    # Save the DataFrame to the binary stream as an Excel file
+    with pd.ExcelWriter(output, engine='xlsxwriter') as writer:
+        data_caqh.to_excel(writer, sheet_name='Sheet1', index=False)
+        writer.save()
+
+    # Get the binary data from the stream
+    data = output.getvalue()
+
+    # Add a download button for the Excel file
+    st.download_button(
+        label='Download CAQH matches as Excel file',
+        data=data,
+        file_name='data.xlsx',
+        mime='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    )
+
     st.header('NDB data')
     st.dataframe(data_ndb, use_container_width=True)
 
-    # calculate the embedding of each item.
 
-    #st.dataframe(data_caqh)
-    # Do some matching
-    #data_caqh.loc[data_caqh['full-addr'] == '1000 Vale Terrace, Vista, CA, 92084', 'matched-addr'] = '456 Main St'
-    #time.sleep(10)
-    #st.dataframe(data_caqh)
 
 
 

requirements.txt

@@ -1,4 +1,5 @@
 streamlit
 pandas
 numpy
-openpyxl
+openpyxl
+xlsxwriter