import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))) import pandas as pd import streamlit as st import time from src.rag.pipeline import RAGPipeline from src.utils.data_v2 import ( build_filter, get_meta, load_json, load_css, ) from src.utils.writer import typewriter st.set_page_config(layout="wide") EMBEDDING_MODEL = "sentence-transformers/distiluse-base-multilingual-cased-v1" PROMPT_TEMPLATE = os.path.join("src", "rag", "prompt_template.yaml") @st.cache_data def load_css_style(path: str) -> None: load_css(path) @st.cache_data def get_meta_data() -> pd.DataFrame: return pd.read_csv( os.path.join("database", "meta_data.csv"), dtype=({"retriever_id": str}) ) @st.cache_data def get_df() -> pd.DataFrame: return pd.read_csv( os.path.join("data", "inc_df.csv"), dtype=({"retriever_id": str}) )[["retriever_id", "draft_labs", "author", "href", "round"]] @st.cache_data def get_authors_taxonomy() -> list[str]: taxonomy = load_json(os.path.join("data", "authors_taxonomy.json")) countries = [] members = taxonomy["Members"] for key, value in members.items(): if key == "Countries" or key == "International and Regional State Associations": countries.extend(value) return countries @st.cache_data def get_draft_cat_taxonomy() -> dict[str, list[str]]: taxonomy = load_json(os.path.join("data", "draftcat_taxonomy_filter.json")) draft_labels = [] for _, subpart in taxonomy.items(): for label in subpart: draft_labels.append(label) return draft_labels @st.cache_data def get_example_prompts() -> list[str]: return [ example["question"] for example in load_json(os.path.join("data", "example_prompts.json")) ] @st.cache_data def set_trigger_state_values() -> tuple[bool, bool]: trigger_filter = st.session_state.setdefault("trigger", False) trigger_ask = st.session_state.setdefault("trigger", False) return trigger_filter, trigger_ask @st.cache_resource def load_pipeline() -> RAGPipeline: return RAGPipeline( embedding_model=EMBEDDING_MODEL, prompt_template=PROMPT_TEMPLATE, ) @st.cache_data def load_app_init() -> None: # Define the title of the app st.title("INC Plastic Pollution Country Profile Analysis") # add warning emoji and style st.markdown( """
⚠️ The app is a beta version that serves as a basis for further development. We are aware that the performance is not yet sufficient and that the data basis is not yet complete. We are grateful for any feedback that contributes to the further development and improvement of the app!
The app is tailored to enhance the efficiency of finding and accessing information on the UN Plastics Treaty Negotiations. It hosts a comprehensive database of relevant documents submitted by the members available here, which users can explore through an intuitive chatbot interface as well as simple filtering options. The app excels in querying specific information about countries and their positions in the negotiations, providing targeted and precise answers. However, it can process only up to 8 relevant documents at a time, which may limit responses to more complex inquiries. Filter options by authors and sections of the negotiation draft ensure the accuracy of the answers. Each document found via these filters is also directly accessible via a link, ensuring complete and easy access to the desired information.
""", unsafe_allow_html=True, ) load_css_style("style/style.css") # Load the data df = get_df() df_transformed = get_meta_data() countries = get_authors_taxonomy() draft_labels = get_draft_cat_taxonomy() example_prompts = get_example_prompts() trigger_filter, trigger_ask = set_trigger_state_values() # Load pipeline pipeline = load_pipeline() # Load app init load_app_init() application_col = st.columns(1) with application_col[0]: st.markdown("""1️⃣ Select countries""", unsafe_allow_html=True) st.markdown( """
Please select the countries of interest. Your selection will refine the database to include documents submitted by these countries or recognized groupings such as Small Developing States, the African States Group, etc.
""", unsafe_allow_html=True, ) selected_authors = st.multiselect( label="country", options=countries, label_visibility="collapsed", placeholder="Select country/countries", ) st.write("\n") st.write("\n") st.markdown( """2️⃣ Select parts of the negotiation draft""", unsafe_allow_html=True, ) st.markdown( """
Please select the parts of the negotiation draft of interest. The negotiation draft can be accessed here.
""", unsafe_allow_html=True, ) selected_draft_cats = st.multiselect( label="Subpart", options=draft_labels, label_visibility="collapsed", placeholder="Select draft category/draft categories", ) st.write("\n") st.write("\n") st.markdown( """3️⃣ Ask a question or show documents based on selected filters""", unsafe_allow_html=True, ) asking, filtering = st.tabs(["Ask a question", "Filter documents"]) with filtering: application_col_filter, output_col_filter = st.columns([1, 1.5]) # make the buttons text smaller with application_col_filter: st.markdown( """
This filter function allows you to see all documents that match the selected filters. The documents can be accessed via a link. \n """, unsafe_allow_html=True, ) if st.button("Filter documents"): filters, status = build_filter( meta_data=df_transformed, authors_filter=selected_authors, draft_cats_filter=selected_draft_cats, ) if status == "no filters selected": st.info("No filters selected. All documents will be shown.") df_filtered = df[ ["author", "href", "draft_labs", "round"] ].sort_values(by="author") trigger_filter = True if status == "no results found": st.info( "No documents found for the combination of filters you've chosen. All countries are represented at least once in the data. Remove the draft categories to see all documents for the countries selected or try other draft categories." ) if status == "success": df_filtered = df[df["retriever_id"].isin(filters["retriever_id"])][ ["author", "href", "draft_labs", "round"] ].sort_values(by="author") trigger_filter = True with asking: application_col_ask, output_col_ask = st.columns([1, 1.5]) with application_col_ask: st.markdown( """
Ask a question, noting that the database has been restricted by filters and that your question should pertain to the selected data. \n """, unsafe_allow_html=True, ) if "prompt" not in st.session_state: prompt = st.text_area("Enter a question") if ( "prompt" in st.session_state and st.session_state.prompt in example_prompts # noqa: E501 ): # noqa: E501 prompt = st.text_area( "Enter a question", value=st.session_state.prompt ) # noqa: E501 if ( "prompt" in st.session_state and st.session_state.prompt not in example_prompts # noqa: E501 ): # noqa: E501 del st.session_state["prompt"] prompt = st.text_area("Enter a question") trigger_ask = st.session_state.setdefault("trigger", False) if st.button("Ask"): if prompt == "": st.error( "Please enter a question. Reloading the app in few seconds" ) time.sleep(3) st.rerun() with st.spinner("Filtering data...") as status: filter_selection_transformed, status = build_filter( meta_data=df_transformed, authors_filter=selected_authors, draft_cats_filter=selected_draft_cats, ) if status == "no filters selected": st.info( "No filters selcted.This will increase the prcessing time significantly. Please select at least one filter." ) # st.error( # "Selecting a filter is mandatory. We especially recommend to select countries you are interested in. Selecting at least one filter is mandatory, because otherwise the model would have to analyze all available documents which results in inaccurate answers and long processing times. Please select at least one filter." # ) # st.stop() documents = pipeline.document_store.get_all_documents( filters=filter_selection_transformed ) st.success("Filtering data completed.") with st.spinner("Answering question...") as status: if filter_selection_transformed == {}: st.warning( "The combination of filters you've chosen does not match any documents. Giving answer based on all documents. Please note that the answer might not be accurate. We highly recommend to use a combination of filters that match the data. All countries are represented at least once in the data. Thus, for example, you could remove the draft categories to match the documents. Or you could check with the Filter documents function which documents are available for the selected countries by removing the draft categories and filter the documents." ) result = pipeline.run( prompt=prompt, filters=filter_selection_transformed ) trigger_ask = True st.success("Answering question completed.") st.markdown("### Examples") for i, prompt in enumerate(example_prompts): # with col[i % 4]: if st.button(prompt): if "key" not in st.session_state: st.session_state["prompt"] = prompt st.markdown( """