| # set path | |
| import glob, os, sys | |
| sys.path.append('../utils') | |
| #import needed libraries | |
| import seaborn as sns | |
| import matplotlib.pyplot as plt | |
| import numpy as np | |
| import pandas as pd | |
| import streamlit as st | |
| from utils.adapmit_classifier import load_adapmitClassifier,adapmit_classification | |
| # from utils.keyword_extraction import textrank | |
| import logging | |
| logger = logging.getLogger(__name__) | |
| from utils.config import get_classifier_params | |
| from utils.preprocessing import paraLengthCheck | |
| from io import BytesIO | |
| import xlsxwriter | |
| import plotly.express as px | |
| # Declare all the necessary variables | |
| classifier_identifier = 'adapmit' | |
| params = get_classifier_params(classifier_identifier) | |
| def to_excel(df): | |
| len_df = len(df) | |
| output = BytesIO() | |
| writer = pd.ExcelWriter(output, engine='xlsxwriter') | |
| df.to_excel(writer, index=False, sheet_name='Sheet1') | |
| workbook = writer.book | |
| worksheet = writer.sheets['Sheet1'] | |
| worksheet.data_validation('E2:E{}'.format(len_df), | |
| {'validate': 'list', | |
| 'source': ['No', 'Yes', 'Discard']}) | |
| worksheet.data_validation('F2:F{}'.format(len_df), | |
| {'validate': 'list', | |
| 'source': ['No', 'Yes', 'Discard']}) | |
| worksheet.data_validation('G2:G{}'.format(len_df), | |
| {'validate': 'list', | |
| 'source': ['No', 'Yes', 'Discard']}) | |
| writer.save() | |
| processed_data = output.getvalue() | |
| return processed_data | |
| def app(): | |
| ### Main app code ### | |
| with st.container(): | |
| if 'key1' in st.session_state: | |
| df = st.session_state.key1 | |
| classifier = load_adapmitClassifier(classifier_name=params['model_name']) | |
| st.session_state['{}_classifier'.format(classifier_identifier)] = classifier | |
| if sum(df['Target Label'] == 'TARGET') > 100: | |
| warning_msg = ": This might take sometime, please sit back and relax." | |
| else: | |
| warning_msg = "" | |
| df = adapmit_classification(haystack_doc=df, | |
| threshold= params['threshold']) | |
| st.session_state.key1 = df | |
| # threshold= params['threshold'] | |
| # truth_df = df.drop(['text'],axis=1) | |
| # truth_df = truth_df.astype(float) >= threshold | |
| # truth_df = truth_df.astype(str) | |
| # categories = list(truth_df.columns) | |
| # placeholder = {} | |
| # for val in categories: | |
| # placeholder[val] = dict(truth_df[val].value_counts()) | |
| # count_df = pd.DataFrame.from_dict(placeholder) | |
| # count_df = count_df.T | |
| # count_df = count_df.reset_index() | |
| # # st.write(count_df) | |
| # placeholder = [] | |
| # for i in range(len(count_df)): | |
| # placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'Yes']) | |
| # placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'No']) | |
| # count_df = pd.DataFrame(placeholder, columns = ['category','count','truth_value']) | |
| # # st.write("Total Paragraphs: {}".format(len(df))) | |
| # fig = px.bar(count_df, y='category', x='count', | |
| # color='truth_value',orientation='h', height =200) | |
| # c1, c2 = st.columns([1,1]) | |
| # with c1: | |
| # st.plotly_chart(fig,use_container_width= True) | |
| # truth_df['labels'] = truth_df.apply(lambda x: {i if x[i]=='True' else None for i in categories}, axis=1) | |
| # truth_df['labels'] = truth_df.apply(lambda x: list(x['labels'] -{None}),axis=1) | |
| # # st.write(truth_df) | |
| # df = pd.concat([df,truth_df['labels']],axis=1) | |
| # st.markdown("###### Top few 'Mitigation' related paragraph/text ######") | |
| # df = df.sort_values(by = ['Mitigation'], ascending=False) | |
| # for i in range(3): | |
| # if df.iloc[i]['Mitigation'] >= 0.50: | |
| # st.write('**Result {}** (Relevancy Score: {:.2f})'.format(i+1,df.iloc[i]['Mitigation'])) | |
| # st.write("\t Text: \t{}".format(df.iloc[i]['text'].replace("\n", " "))) | |
| # st.markdown("###### Top few 'Adaptation' related paragraph/text ######") | |
| # df = df.sort_values(by = ['Adaptation'], ascending=False) | |
| # for i in range(3): | |
| # if df.iloc[i]['Adaptation'] > 0.5: | |
| # st.write('**Result {}** (Relevancy Score: {:.2f})'.format(i+1,df.iloc[i]['Adaptation'])) | |
| # st.write("\t Text: \t{}".format(df.iloc[i]['text'].replace("\n", " "))) | |
| # # st.write(df[['text','labels']]) | |
| # df['Validation'] = 'No' | |
| # df['Val-Mitigation'] = 'No' | |
| # df['Val-Adaptation'] = 'No' | |
| # df_xlsx = to_excel(df) | |
| # st.download_button(label='📥 Download Current Result', | |
| # data=df_xlsx , | |
| # file_name= 'file_adaptation-mitigation.xlsx') | |
| # # st.session_state.key4 = | |
| # # category =set(df.columns) | |
| # # removecols = {'Validation','Val-Adaptation','Val-Mitigation','text'} | |
| # # category = list(category - removecols) | |
| # else: | |
| # st.info("🤔 No document found, please try to upload it at the sidebar!") | |
| # logging.warning("Terminated as no document provided") | |
| # # Creating truth value dataframe | |
| # if 'key4' in st.session_state: | |
| # if st.session_state.key4 is not None: | |
| # df = st.session_state.key4 | |
| # st.markdown("###### Select the threshold for classifier ######") | |
| # c4, c5 = st.columns([1,1]) | |
| # with c4: | |
| # threshold = st.slider("Threshold", min_value=0.00, max_value=1.0, | |
| # step=0.01, value=0.5, | |
| # help = "Keep High Value if want refined result, low if dont want to miss anything" ) | |
| # category =set(df.columns) | |
| # removecols = {'Validation','Val-Adaptation','Val-Mitigation','text'} | |
| # category = list(category - removecols) | |
| # placeholder = {} | |
| # for val in category: | |
| # temp = df[val].astype(float) > threshold | |
| # temp = temp.astype(str) | |
| # placeholder[val] = dict(temp.value_counts()) | |
| # count_df = pd.DataFrame.from_dict(placeholder) | |
| # count_df = count_df.T | |
| # count_df = count_df.reset_index() | |
| # placeholder = [] | |
| # for i in range(len(count_df)): | |
| # placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'False']) | |
| # placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'True']) | |
| # count_df = pd.DataFrame(placeholder, columns = ['category','count','truth_value']) | |
| # fig = px.bar(count_df, x='category', y='count', | |
| # color='truth_value', | |
| # height=400) | |
| # st.write("") | |
| # st.plotly_chart(fig) | |
| # df['Validation'] = 'No' | |
| # df['Val-Mitigation'] = 'No' | |
| # df['Val-Adaptation'] = 'No' | |
| # df_xlsx = to_excel(df) | |
| # st.download_button(label='📥 Download Current Result', | |
| # data=df_xlsx , | |
| # file_name= 'file_adaptation-mitigation.xlsx') | |