Updaets

README.md

@@ -1,5 +1,5 @@
 ---
-title: Strealit Data Synthesis Example
+title: Address matching Example
 emoji: 💩
 colorFrom: blue
 colorTo: gray

app.py

@@ -1,8 +1,31 @@
 import streamlit as st
 import pandas as pd
 import numpy as np
+import torch
+#from transformers import AlbertTokenizer, AlbertModel
+#from sklearn.metrics.pairwise import cosine_similarity
 
-st.title('Excel File Uploader')
+#tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+#model = AlbertModel.from_pretrained("albert-base-v2")
+
+#def get_embedding(input_text):
+#    encoded_input = tokenizer(input_text, return_tensors='pt')
+#    input_ids = encoded_input.input_ids
+#    input_num_tokens = input_ids.shape[1]
+#
+#    #print( "Number of input tokens: " + str(input_num_tokens))
+#    #print("Length of input: " + str(len(input_text)))
+#
+#    list_of_tokens = tokenizer.convert_ids_to_tokens(input_ids.view(-1).tolist())
+#
+#    #print( "Tokens : " + ' '.join(list_of_tokens))
+#    with torch.no_grad():
+#        output = model(**encoded_input)
+#
+#    embedding = output.last_hidden_state[0][0]
+#    return embedding.tolist()
+
+st.title('Upload the Address Dataset')
 
 st.markdown('Upload an Excel file to view the data in a table.')
 
@@ -13,20 +36,44 @@ if uploaded_file is not None:
     data_caqh = pd.read_excel(uploaded_file, sheet_name='CAQH')
     data_ndb = pd.read_excel(uploaded_file, sheet_name='NDB')
 
+    # Data cleaning CAQH
+    data_caqh['postalcode'] = data_caqh['postalcode'].astype(str).apply(lambda x: x[:5] + '-' + x[5:] if len(x) > 5 and not '-' in x else x)
     data_caqh['full-addr'] = data_caqh['address1'].astype(str) + ', ' \
                              + np.where(data_caqh['address2'].isnull(),  '' , data_caqh['address2'].astype(str))  \
                              + data_caqh['city'].astype(str) + ', '\
                              + data_caqh['state'].astype(str) + ', ' \
                              + data_caqh['postalcode'].astype(str)
 
+    # Data cleaning NDB
     data_ndb['zip_pls_4_cd'] = data_ndb['zip_pls_4_cd'].astype(str).apply(lambda x: x if (x[-1] != '0' and x[-1] != '1') else '')
 
     data_ndb['zip_cd_zip_pls_4_cd'] = data_ndb['zip_cd'].astype(str) +\
-                                      np.where( data_ndb['zip_pls_4_cd'].isnull() , '', '-' \
+                                      np.where( data_ndb['zip_pls_4_cd'] == '', '', '-' \
                                       + data_ndb['zip_pls_4_cd'].astype(str))
 
-    data_ndb['full-addr'] = data_ndb['adr_ln_1_txt'].astype(str) + ', ' \
+    data_ndb['full-addr'] = data_ndb['adr_ln_1_txt'].astype(str).str.strip() + ', ' \
                             + data_ndb['st_cd'].astype(str) + ', ' + data_ndb['zip_cd_zip_pls_4_cd']
 
+    # Add a matched column
+    data_caqh['matched-addr'] = ''
+
+    # App
+    #data_caqh['embed'] = data_caqh['full-addr'].apply(get_embedding)
+
     st.dataframe(data_caqh)
-    st.dataframe(data_ndb)
+    st.dataframe(data_ndb)
+
+    # Do some matching
+    #data_caqh.loc[data_caqh['full-addr'] == '1000 Vale Terrace, Vista, CA, 92084', 'matched-addr'] = '456 Main St'
+    #time.sleep(10)
+    #st.dataframe(data_caqh)
+
+
+
+
+
+
+
+
+
+

data.py

@@ -1,8 +1,10 @@
 from transformers import AlbertTokenizer, AlbertModel
 from sklearn.metrics.pairwise import cosine_similarity
 
-tokenizer = AlbertTokenizer.from_pretrained('albert-large-v2')
-model = AlbertModel.from_pretrained("albert-large-v2")
+# base
+# large
+tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2')
+model = AlbertModel.from_pretrained("albert-base-v2")
 
 a1 = "65 Mountain Blvd Ext, Warren, NJ 07059"
 a2 = "112 Mountain Blvd Ext, Warren, NJ 07059"
@@ -32,11 +34,11 @@ e2 = get_embedding(a2)
 e4 = get_embedding(a4)
 e5 = get_embedding(a5)
 
-print("a1 to a2")
+print(f"a1 {a1} to {a2} a2")
 print(cosine_similarity([e1], [e2]))
-print("a1 to a4")
+print(f"a1 {a1} to {a4} a4")
 print(cosine_similarity([e1], [e4]))
-print("a1 to a5")
+print(f"a1 {a1} to {a5} a5")
 print(cosine_similarity([e1], [e5]))
 
 # with base

requirements.txt

@@ -1,3 +1,4 @@
 streamlit
 pandas
+numpy
 openpyxl