# set path import glob, os, sys; sys.path.append('../utils') #import needed libraries import seaborn as sns import matplotlib.pyplot as plt import numpy as np import pandas as pd import streamlit as st from utils.vulnerability_classifier import load_vulnerabilityClassifier, vulnerability_classification import logging logger = logging.getLogger(__name__) from utils.config import get_classifier_params from utils.preprocessing import paraLengthCheck from io import BytesIO import xlsxwriter import plotly.express as px # Declare all the necessary variables classifier_identifier = 'vulnerability' params = get_classifier_params(classifier_identifier) @st.cache_data def to_excel(df,sectorlist): len_df = len(df) output = BytesIO() writer = pd.ExcelWriter(output, engine='xlsxwriter') df.to_excel(writer, index=False, sheet_name='Sheet1') workbook = writer.book worksheet = writer.sheets['Sheet1'] worksheet.data_validation('S2:S{}'.format(len_df), {'validate': 'list', 'source': ['No', 'Yes', 'Discard']}) worksheet.data_validation('X2:X{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('T2:T{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('U2:U{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('V2:V{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) worksheet.data_validation('W2:U{}'.format(len_df), {'validate': 'list', 'source': sectorlist + ['Blank']}) writer.save() processed_data = output.getvalue() return processed_data def app(): ### Main app code ### with st.container(): if 'key0' in st.session_state: df = st.session_state.key0 classifier = load_vulnerabilityClassifier(classifier_name=params['model_name']) st.session_state['{}_classifier'.format(classifier_identifier)] = classifier # if sum(df['Target Label'] == 'TARGET') > 100: # warning_msg = ": This might take sometime, please sit back and relax." # else: # warning_msg = "" df = vulnerability_classification(haystack_doc=df, threshold= params['threshold']) st.session_state.key0 = df # # st.write(df) # threshold= params['threshold'] # truth_df = df.drop(['text'],axis=1) # truth_df = truth_df.astype(float) >= threshold # truth_df = truth_df.astype(str) # categories = list(truth_df.columns) # placeholder = {} # for val in categories: # placeholder[val] = dict(truth_df[val].value_counts()) # count_df = pd.DataFrame.from_dict(placeholder) # count_df = count_df.T # count_df = count_df.reset_index() # # st.write(count_df) # placeholder = [] # for i in range(len(count_df)): # placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'Yes']) # placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'No']) # count_df = pd.DataFrame(placeholder, columns = ['category','count','truth_value']) # # st.write("Total Paragraphs: {}".format(len(df))) # fig = px.bar(count_df, x='category', y='count', # color='truth_value') # # c1, c2 = st.columns([1,1]) # # with c1: # st.plotly_chart(fig,use_container_width= True) # truth_df['labels'] = truth_df.apply(lambda x: {i if x[i]=='True' else None for i in categories}, axis=1) # truth_df['labels'] = truth_df.apply(lambda x: list(x['labels'] -{None}),axis=1) # # st.write(truth_df) # df = pd.concat([df,truth_df['labels']],axis=1) # df['Validation'] = 'No' # df['Sector1'] = 'Blank' # df['Sector2'] = 'Blank' # df['Sector3'] = 'Blank' # df['Sector4'] = 'Blank' # df['Sector5'] = 'Blank' # df_xlsx = to_excel(df,categories) # st.download_button(label='📥 Download Current Result', # data=df_xlsx , # # file_name= 'file_sector.xlsx') # else: # st.info("🤔 No document found, please try to upload it at the sidebar!") # logging.warning("Terminated as no document provided") # # Creating truth value dataframe # if 'key' in st.session_state: # if st.session_state.key is not None: # df = st.session_state.key # st.markdown("###### Select the threshold for classifier ######") # c4, c5 = st.columns([1,1]) # with c4: # threshold = st.slider("Threshold", min_value=0.00, max_value=1.0, # step=0.01, value=0.5, # help = "Keep High Value if want refined result, low if dont want to miss anything" ) # sectors =set(df.columns) # removecols = {'Validation','Sector1','Sector2','Sector3','Sector4', # 'Sector5','text'} # sectors = list(sectors - removecols) # placeholder = {} # for val in sectors: # temp = df[val].astype(float) > threshold # temp = temp.astype(str) # placeholder[val] = dict(temp.value_counts()) # count_df = pd.DataFrame.from_dict(placeholder) # count_df = count_df.T # count_df = count_df.reset_index() # placeholder = [] # for i in range(len(count_df)): # placeholder.append([count_df.iloc[i]['index'],count_df['False'][i],'False']) # placeholder.append([count_df.iloc[i]['index'],count_df['True'][i],'True']) # count_df = pd.DataFrame(placeholder, columns = ['sector','count','truth_value']) # fig = px.bar(count_df, x='sector', y='count', # color='truth_value', # height=400) # st.write("") # st.plotly_chart(fig) # df['Validation'] = 'No' # df['Sector1'] = 'Blank' # df['Sector2'] = 'Blank' # df['Sector3'] = 'Blank' # df['Sector4'] = 'Blank' # df['Sector5'] = 'Blank' # df_xlsx = to_excel(df,sectors) # st.download_button(label='📥 Download Current Result', # data=df_xlsx , # file_name= 'file_sector.xlsx')