import streamlit as st
import streamlit as st
import random
import requests
import pandas as pd
import pickle
import joblib
import re
import pandas as pd
import numpy as np
import re
import string
from string import digits
from sklearn import metrics
import pickle
import time
from sentence_transformers import SentenceTransformer
# List of URLs of background images
background_image_urls = [
'https://www.canarahsbclife.com/content/dam/choice/blog-inner/images/what-is-insurance-meaning-and-benefits-of-insurance.jpg',
'https://www.avivaindia.com/sites/default/files/Types-of-Insurance.jpg',
'https://images.livemint.com/img/2022/09/01/1600x900/Health_Insurance_1662032759457_1662032759610_1662032759610.jpg',
]
# Randomly select a background image URL
selected_image_url = random.choice(background_image_urls)
# Fetch the selected image from the URL
response = requests.get(selected_image_url)
if response.status_code == 200:
# Set the background image using CSS
background_style = f"""
"""
# Display the random background image
st.markdown(background_style, unsafe_allow_html=True)
else:
st.warning("Failed to fetch the background image.")
# Create a Streamlit app
st.title("Gallagher : Text Classification and Excel Processing App")
# File upload for Excel file
uploaded_file = st.file_uploader("Upload an Excel file", type=["xlsx"])
import base64
from io import BytesIO
def get_binary_file_downloader_link(file_data, file_name, link_text):
# Write the DataFrame to an in-memory Excel file
excel_buffer = BytesIO()
file_data.to_excel(excel_buffer, index=False, engine='xlsxwriter')
# Create a base64-encoded string of the Excel file's contents
b64 = base64.b64encode(excel_buffer.getvalue()).decode()
# Generate the download link
href = f'{link_text}'
return href
def pre_processing(data_frame):
# Lowercase all characters
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: x.lower())
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"won\'t", "will not", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"can\'t", "can not", x))
# general
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"n\'t", " not", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'re", " are", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'s", " is", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'d", " would", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'ll", " will", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'t", " not", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'ve", " have", x))
data_frame['Claim Description'] = data_frame['Claim Description'].apply(lambda x: re.sub(r"\'m", " am", x))
# Remove quotes
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: re.sub("'", '', x))
exclude = set(string.punctuation) # Set of all special characters
# Remove all the special characters
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: ''.join(ch for ch in x if ch not in exclude))
# Remove all numbers from text
remove_digits = str.maketrans('', '', digits)
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: x.translate(remove_digits))
# remove extra
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: re.sub('[-_.:;\[\]\|,]', '', x))
# Remove extra spaces
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: x.strip())
data_frame['Claim Description']=data_frame['Claim Description'].apply(lambda x: re.sub(" +", " ", x))
return data_frame
step_1_model_path = "output/lr_step_1.pickle"
step_2_model_path = "output/lr_basemodel_step_2.pickle"
step_1_model = pickle.load(open(step_1_model_path, 'rb'))
step_2_model = pickle.load(open(step_2_model_path, 'rb'))
count_vector_step_1 = joblib.load("output/count_vector_step_1.pkl")
count_vector_step_2 = joblib.load("output/count_vector_step_2.pkl")
fewer_class_dict = joblib.load("output/fewer_class_dictionary.pkl")
acc_src_model = joblib.load("output/bert_acc_src.pickle")
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
def predict(model_1,model_2,final_dict,query):
# predict
test_1 = count_vector_step_1.transform([query])
y_pred = model_1.predict(test_1)
if y_pred == 'med':
test_2 = count_vector_step_2.transform([query])
y_pred = model_2.predict(test_2)
else:
y_pred = y_pred
if query in final_dict.keys():
y_pred = final_dict[query]
else:
y_pred = y_pred
return y_pred[0]
if uploaded_file is not None:
# Read the uploaded Excel file
excel_data = pd.read_excel(uploaded_file)
final_result= []
print('Preprocessing Started')
test_data = pre_processing(excel_data)
x_test = test_data['Claim Description']
print('Prediction Started')
for query in x_test:
result = predict(step_1_model,step_2_model,fewer_class_dict,query)
final_result.append(result)
excel_data['predicted_coverage_code'] = final_result
X_bert_enc = model.encode(x_test.values, show_progress_bar=True,)
accident_source_pred = acc_src_model.predict(X_bert_enc)
excel_data['predicted_accident_src'] = accident_source_pred
st.dataframe(excel_data) # Display the processed data
link = get_binary_file_downloader_link(excel_data, 'my_processed_file.xlsx', 'Download Processed Data')
st.markdown(link, unsafe_allow_html=True)
# Create a new Excel file with the processed data
output_filename = "processed_data.xlsx"
excel_data.to_excel(output_filename, index=False)
# Display a link to download the processed file
st.markdown(f"Download Processed Data: [Processed Data](data:{output_filename})")
# Add a placeholder for displaying "Done" after processing
if uploaded_file is not None:
st.write("Done")