tabbed page and highlighted context

README.md

@@ -5,6 +5,6 @@ colorFrom: green
 colorTo: yellow
 sdk: streamlit
 sdk_version: 1.2.0
-app_file: app.py
+app_file: 🏡_Home.py
 pinned: false
 ---

pages/⚙️_Settings.py

@@ -0,0 +1,3 @@
+import streamlit as st
+
+st.write("This is the settings page")

pages/💁‍♀️_Info.py

@@ -0,0 +1,12 @@
+import streamlit as st
+
+st.markdown("""
+# Better Image Retrieval With Reinforced CLIP 🧠
+
+
+CLIP is a neural network trained on image-text pairs that can predict how semantically close images are with some text
+But, although CLIP understands what it sees, it doesn't know its properties. While other models can understand text that contains such information, like Wikipedia.
+
+In this demo application, we see if we can 'help' CLIP by reinforcing it with another model.""")
+
+st.image("diagram.png")

requirements.txt

@@ -1,2 +1,4 @@
 farm-haystack[faiss]==1.11.1
 streamlit==1.12.0
+markdown
+st-annotated-text

utils/frontend.py

@@ -5,6 +5,4 @@ def set_state_if_absent(key, value):
         st.session_state[key] = value
 
 def reset_results(*args):
-    st.write("Called reset")
-    st.session_state.answer = None
     st.session_state.results = None

utils/haystack.py

@@ -73,7 +73,7 @@ pipe = start_haystack()
 
 @st.cache(allow_output_mutation=True)
 def query(statement: str, text_retriever_top_k: int = 5, image_retriever_top_k = 1):
-    """Run query and verify statement"""
+    """Run query"""
     params = {"image_retriever": {"top_k": image_retriever_top_k},"text_retriever": {"top_k": text_retriever_top_k} }
     results = pipe.run(statement, params=params)
     return results

app.py → 🏡_Home.py

@@ -3,18 +3,40 @@ import time
 import logging
 from json import JSONDecodeError
 from PIL import Image
-
-
+from markdown import markdown
+from annotated_text import annotation
 from utils.haystack import query
 from utils.frontend import reset_results, set_state_if_absent
 
+def create_answer_objects(predictions):
+    results = []
+    for answer in predictions:
+        answer = answer.to_dict()
+        if answer["answer"]:
+            results.append(
+                {
+                    "context": "..." + answer["context"] + "...",
+                    "answer": answer["answer"],
+                    "relevance": round(answer["score"] * 100, 2),
+                    "offset_start_in_doc": answer["offsets_in_document"][0]["start"],
+                }
+            )
+        else:
+            results.append(
+                {
+                    "context": None,
+                    "answer": None,
+                    "relevance": round(answer["score"] * 100, 2),
+                }
+            )
+    return results
+
 def main():
 
     set_state_if_absent("statement", "What is the fastest animal?")
     set_state_if_absent("results", None)
 
     st.write("# Look for images with MultiModalRetrieval 🐅")
-    st.write()
     st.markdown(
         """
     ##### Enter a question about animals
@@ -41,15 +63,13 @@ def main():
         time_start = time.time()
         reset_results()
         st.session_state.statement = statement
-        with st.spinner("🧠    Performing neural search on documents..."):
+        with st.spinner("🔎 🐼🐷🦊    Looking for the right animal"):
             try:
                 docs = query(statement)
-                st.write(docs["documents"])
                 for doc in docs["documents"]:
                     image = Image.open(doc.content)
                     st.image(image)
-                for answer in docs["answers"]:
-                    st.write(answer)
+                st.session_state.results = create_answer_objects(docs["answers"])
                 print(f"S: {statement}")
                 time_end = time.time()
                 print(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime()))
@@ -62,19 +82,24 @@ def main():
             except Exception as e:
                 logging.exception(e)
                 st.error("🐞    An error occurred during the request.")
-            return
+                return
     
-    # if st.session_state.results:
-    #     st.write("Got some results")
-    #     print("GOT RESTULTS")
-        # st.write("Received Results")
-        # results = st.session_state.results
-        # print(results)
-        # docs = results["documents"]
-        # st.write(results)
-        # # show different messages depending on entailment results
-        # for doc in docs:
-        #     image = Image(filename=doc.content)
-        #     st.image(image)
+    if st.session_state.results:
+        st.write('## Why this image?')
+        answers = st.session_state.results
+        for count, answer in enumerate(answers):
+            if answer["answer"]:
+                text, context = answer["answer"], answer["context"]
+                start_idx = context.find(text)
+                end_idx = start_idx + len(text)
+                st.write(
+                    markdown(context[:start_idx] + str(annotation(body=text, label="ANSWER", background="#964448", color='#ffffff')) + context[end_idx:]),
+                    unsafe_allow_html=True,
+                )
+                st.markdown(f"**Relevance:** {answer['relevance']}")
+            else:
+                st.info(
+                    "🤔    Haystack is unsure whether any of the documents contain an answer to your question. Try to reformulate it!"
+                )
 
 main()