File size: 7,349 Bytes
22b8e0b 550b85d 22b8e0b b26b139 2663a97 1b62a9f b26b139 fa191c0 2663a97 b26b139 fa191c0 2663a97 b26b139 fa191c0 b26b139 8c4c590 2663a97 1b62a9f 2663a97 1b62a9f 2663a97 22b8e0b b26b139 1b62a9f b26b139 2663a97 1b62a9f 2663a97 1b62a9f dacf87d 1b62a9f 2663a97 1b62a9f 2663a97 1b62a9f 183851c 49548a6 183851c 1b62a9f 2663a97 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 |
# set path
import glob, os, sys;
sys.path.append('../utils')
import streamlit as st
import ast
import logging
from utils.ndc_explorer import countrySpecificCCA, countrySpecificCCM
from utils.checkconfig import getconfig
from utils.semantic_search import runSemanticPreprocessingPipeline,process_semantic_output
from utils.semantic_search import semanticSearchPipeline, runSemanticPipeline
from st_aggrid import AgGrid
from st_aggrid.shared import ColumnsAutoSizeMode
# Reading data and Declaring necessary variables
with open('docStore/ndcs/countryList.txt') as dfile:
countryList = dfile.read()
countryList = ast.literal_eval(countryList)
countrynames = list(countryList.keys())
with open('docStore/ndcs/cca.txt', encoding='utf-8', errors='ignore') as dfile:
cca_sent = dfile.read()
cca_sent = ast.literal_eval(cca_sent)
with open('docStore/ndcs/ccm.txt', encoding='utf-8', errors='ignore') as dfile:
ccm_sent = dfile.read()
ccm_sent = ast.literal_eval(ccm_sent)
config = getconfig('paramconfig.cfg')
split_by = config.get('coherence','SPLIT_BY')
split_length = int(config.get('coherence','SPLIT_LENGTH'))
split_overlap = int(config.get('coherence','SPLIT_OVERLAP'))
split_respect_sentence_boundary = bool(int(config.get('coherence',
'RESPECT_SENTENCE_BOUNDARY')))
remove_punc = bool(int(config.get('coherence','REMOVE_PUNC')))
embedding_model = config.get('coherence','RETRIEVER')
embedding_model_format = config.get('coherence','RETRIEVER_FORMAT')
embedding_layer = int(config.get('coherence','RETRIEVER_EMB_LAYER'))
embedding_dim = int(config.get('coherence','EMBEDDING_DIM'))
max_seq_len = int(config.get('coherence','MAX_SEQ_LENGTH'))
retriever_top_k = int(config.get('coherence','RETRIEVER_TOP_K'))
def app():
#### APP INFO #####
with st.container():
st.markdown("<h1 style='text-align: center; \
color: black;'> Check NDC Coherence</h1>",
unsafe_allow_html=True)
st.write(' ')
st.write(' ')
with st.expander("ℹ️ - About this app", expanded=False):
st.write(
"""
The *Check NDC Coherence* application provides easy evaluation of
coherence between a given policy document and a country’s (Intended)\
Nationally Determined Contribution (INDCs/NDCs) using open-source \
data from the German Institute of Development and Sustainability’s \
(IDOS) [NDC Explorer]
(https://klimalog.idos-research.de/ndc/#NDCExplorer/worldMap?NewAndUpdatedNDC??income???catIncome).\
""")
st.write("")
st.write(""" User can select a country context via the drop-down menu \
on the left-hand side of the application. Subsequently, the user is \
given the opportunity to manually upload another policy document \
from the same national context or to select a pre-loaded example \
document. Thereafter, the user can choose between two categories \
to compare coherence between the documents: climate change adaptation \
and climate change mitigation. Based on the selected information, \
the application identifies relevant paragraphs in the uploaded \
document and assigns them to the respective indicator from the NDC \
Explorer. Currently, the NDC Explorer has 20 indicators under \
climate change mitigation (e.g., fossil fuel production, REDD+) and \
22 indicators under climate change adaptation (e.g., sea level rise,\
investment needs). The assignment of the paragraph to a corresponding\
indicator is based on vector similarities in which only paragraphs \
with similarity above 0.55 to the indicators are considered. """)
with st.sidebar:
option = st.selectbox('Select Country', (countrynames))
countryCode = countryList[option]
st.markdown("---")
genre = st.radio( "Select Category",('Climate Change Adaptation',
'Climate Change Mitigation'))
st.markdown("---")
with st.container():
if st.button("Check Coherence"):
sent_cca = countrySpecificCCA(cca_sent,1,countryCode)
sent_ccm = countrySpecificCCM(ccm_sent,1,countryCode)
if 'filepath' in st.session_state:
allDocuments = runSemanticPreprocessingPipeline(
file_path= st.session_state['filepath'],
file_name = st.session_state['filename'],
split_by=split_by,
split_length= split_length,
split_overlap=split_overlap,
remove_punc= remove_punc,
split_respect_sentence_boundary=split_respect_sentence_boundary)
# genre = st.radio( "Select Category",('Climate Change Adaptation', 'Climate Change Mitigation'))
if genre == 'Climate Change Adaptation':
sent_dict = sent_cca
else:
sent_dict = sent_ccm
sent_labels = []
for key,sent in sent_dict.items():
sent_labels.append(sent)
if len(allDocuments['documents']) > 100:
warning_msg = ": This might take sometime, please sit back and relax."
else:
warning_msg = ""
logging.info("starting Coherence analysis, \
country selected {}".format(option))
with st.spinner("Performing Coherence Analysis for {} \
under {} category{}".format(option,genre,warning_msg)):
semanticsearch_pipeline, doc_store = semanticSearchPipeline(documents = allDocuments['documents'],
embedding_model= embedding_model,
embedding_layer= embedding_layer,
embedding_model_format= embedding_model_format,
retriever_top_k= retriever_top_k,
embedding_dim=embedding_dim,
max_seq_len=max_seq_len, useQueryCheck=False)
raw_output = runSemanticPipeline(pipeline=semanticsearch_pipeline,queries=sent_labels)
results_df = process_semantic_output(raw_output)
results_df = results_df.drop(['answer','answer_offset',
'context_offset','context','reader_score','id'],
axis = 1)
for i,key in enumerate(list(sent_dict.keys())):
st.write("Relevant paragraphs for topic:{}".format(key))
df = results_df[results_df['query']==sent_dict[key]].reset_index(drop=True)
for j in range(3):
st.write('Result {}.'.format(j+1))
st.write(df.loc[j]['content']+'\n')
else:
st.info("🤔 No document found, please try to upload it at the sidebar!")
logging.warning("Terminated as no document provided") |