First app version

app.py

@@ -1,5 +1,43 @@
 import streamlit as st
 
-st.title("🧠 Neural Search 🔎")
+st.set_page_config(
+    page_title="Neural Search",
+    page_icon="🔎",
+    layout="wide",
+    initial_sidebar_state="expanded",
+)
 
-st.write("Coming soon...")
+from streamlit_option_menu import option_menu
+from interface.config import session_state_variables, pages
+from interface.components import component_select_pipeline
+
+# Initialization of session state
+for key, value in session_state_variables.items():
+    if key not in st.session_state:
+        st.session_state[key] = value
+
+
+def run_demo():
+
+    main_page = st.container()
+
+    st.sidebar.title("🧠 Neural Search 🔎")
+
+    navigation = st.sidebar.container()
+
+    with navigation:
+
+        selected_page = option_menu(
+            "Navigation",
+            list(pages.keys()),
+            icons=[f[1] for f in pages.values()],
+            menu_icon="cast",
+            default_index=0,
+        )
+        component_select_pipeline(navigation)
+
+    # Draw the correct page
+    pages[selected_page][0](main_page)
+
+
+run_demo()

core/pipelines.py

@@ -0,0 +1,45 @@
+"""
+Haystack Pipelines
+"""
+
+import tokenizers
+from haystack import Pipeline
+from haystack.document_stores import InMemoryDocumentStore
+from haystack.nodes.retriever import DensePassageRetriever
+from haystack.nodes.preprocessor import PreProcessor
+import streamlit as st
+
+@st.cache(hash_funcs={tokenizers.Tokenizer: lambda _: None, tokenizers.AddedToken: lambda _: None}, allow_output_mutation=True)
+def dense_passage_retrieval(
+    index='documents',
+    query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
+    passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
+):
+    document_store = InMemoryDocumentStore(index=index)
+    dpr_retriever = DensePassageRetriever(
+        document_store=document_store,
+        query_embedding_model=query_embedding_model,
+        passage_embedding_model=passage_embedding_model,
+    )
+    processor = PreProcessor(
+        clean_empty_lines=True,
+        clean_whitespace=True,
+        clean_header_footer=True,
+        split_by="word",
+        split_length=100,
+        split_respect_sentence_boundary=True,
+        split_overlap=0,
+    )
+    # SEARCH PIPELINE
+    search_pipeline = Pipeline()
+    search_pipeline.add_node(dpr_retriever, name="DPRRetriever", inputs=["Query"])
+
+    # INDEXING PIPELINE
+    index_pipeline = Pipeline()
+    index_pipeline.add_node(processor, name="Preprocessor", inputs=["File"])
+    index_pipeline.add_node(dpr_retriever, name="DPRRetriever", inputs=["Preprocessor"])
+    index_pipeline.add_node(
+        document_store, name="DocumentStore", inputs=["DPRRetriever"]
+    )
+
+    return search_pipeline, index_pipeline

core/search_index.py

@@ -0,0 +1,42 @@
+from haystack.schema import Document
+import uuid
+
+
+def format_docs(documents):
+    """Given a list of documents, format the documents and return the documents and doc ids."""
+    db_docs: list = []
+    for doc in documents:
+        doc_id = doc['id'] if doc['id'] is not None else str(uuid.uuid4())
+        db_doc = {
+            "content": doc['text'],
+            "content_type": "text",
+            "id": str(uuid.uuid4()),
+            "meta": {"id": doc_id},
+        }
+        db_docs.append(Document(**db_doc))
+    return db_docs, [doc.meta["id"] for doc in db_docs]
+
+def index(documents, pipeline):
+    documents, doc_ids = format_docs(documents)
+    pipeline.run(documents=documents)
+    return doc_ids
+
+def search(queries, pipeline):
+    results = []
+    matches_queries = pipeline.run_batch(queries=queries)
+    for matches in matches_queries["documents"]:
+        query_results = []
+        for res in matches:
+            metadata = res.meta
+            query_results.append(
+                {
+                    "text": res.content,
+                    "score": res.score,
+                    "id": res.meta["id"],
+                    "fragment_id": res.id
+                }
+            )
+        results.append(
+            sorted(query_results, key=lambda x: x["score"], reverse=True)
+        )
+    return results

interface/components.py

@@ -0,0 +1,49 @@
+import streamlit as st
+import core.pipelines as pipelines_functions
+from inspect import getmembers, isfunction
+
+def component_select_pipeline(container):
+    pipeline_names, pipeline_funcs = list(zip(*getmembers(pipelines_functions, isfunction)))
+    pipeline_names = [' '.join([n.capitalize() for n in name.split('_')]) for name in pipeline_names]
+    with container:
+        selected_pipeline = st.selectbox(
+            'Select pipeline',
+            pipeline_names
+        )
+        st.session_state['search_pipeline'], \
+            st.session_state['index_pipeline'] = \
+                pipeline_funcs[pipeline_names.index(selected_pipeline)]()
+
+def component_show_pipeline(container, pipeline):
+    """Draw the pipeline"""
+    with container:
+        pass
+    
+def component_show_search_result(container, results):
+    with container:
+        for idx, document in enumerate(results):
+            st.markdown(f"### Match {idx+1}")
+            st.markdown(f"**Text**: {document['text']}")
+            st.markdown(f"**Document**: {document['id']}")
+            st.markdown(f"**Score**: {document['score']:.3f}")
+            st.markdown("---")
+
+def component_text_input(container):
+    """Draw the Text Input widget"""
+    with container:
+        texts = []
+        doc_id = 1
+        with st.expander("Enter documents"):
+            while True:
+                text = st.text_input(f"Document {doc_id}", key=doc_id)
+                if text != "":
+                    texts.append({"text": text})
+                    doc_id += 1
+                    st.markdown("---")
+                else:
+                    break
+        corpus = [
+            {"text": doc["text"], "id": doc_id}
+            for doc_id, doc in enumerate(texts)
+        ]
+        return corpus

interface/config.py

@@ -0,0 +1,11 @@
+from interface.pages import page_landing_page, page_search, page_index
+
+# Define default Session Variables over the whole session.
+session_state_variables = {}
+
+# Define Pages for the demo
+pages = {
+    "Introduction": (page_landing_page, "house-fill"),
+    "Search": (page_search, "search"),
+    "Index": (page_index, "files"),
+}

interface/pages.py

@@ -0,0 +1,58 @@
+import streamlit as st
+from streamlit_option_menu import option_menu
+from core.search_index import index, search
+from interface.components import component_show_search_result, component_text_input
+
+def page_landing_page(container):
+    with container:
+        st.header("🧠 Neural Search 🔎")
+
+        st.markdown(
+            "This is a tool to allow indexing & search content using neural capabilities"
+        )
+        
+def page_search(container):
+    with container:
+        st.title("Query me!")
+        
+        ## SEARCH ##
+        query = st.text_input("Query")
+        
+        if st.button("Search"):
+            st.session_state['search_results'] = search(
+                queries=[query],
+                pipeline=st.session_state['search_pipeline'],
+            )
+        if 'search_results' in st.session_state:
+            component_show_search_result(
+                container=container,
+                results=st.session_state['search_results'][0]
+            )
+        
+def page_index(container):
+    with container:
+        st.title("Index time!")
+        
+        input_funcs = {
+            "Raw Text": (component_text_input, "card-text"),
+        }
+        selected_input = option_menu(
+            "Input Text",
+            list(input_funcs.keys()),
+            icons=[f[1] for f in input_funcs.values()],
+            menu_icon="list",
+            default_index=0,
+            orientation="horizontal",
+        )
+        
+        corpus = input_funcs[selected_input][0](container)
+        
+        if len(corpus) > 0:
+            index_results = None
+            if st.button("Index"):
+                index_results = index(
+                    corpus,
+                    st.session_state['index_pipeline'],
+                )
+            if index_results:
+                st.write(index_results)

requirements.txt

@@ -1 +1,3 @@
-streamlit
+streamlit
+streamlit_option_menu
+farm-haystack