Upload 23 files

.gitattributes

@@ -32,3 +32,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.doc filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.dvd filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.pos filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.tim filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5.fdt filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5.nvd filter=lfs diff=lfs merge=lfs -text
+lucene-index.miracl-v1.0-ar.20221004.2b2856/_5.tvd filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,10 +1,10 @@
 ---
-title: Miracl Arabic
-emoji: 🐢
-colorFrom: green
-colorTo: red
+title: Miracl Search
+emoji: 🌍🙌🌏
+colorFrom: red
+colorTo: pink
 sdk: streamlit
-sdk_version: 1.17.0
+sdk_version: 1.10.0
 app_file: app.py
 pinned: false
 license: apache-2.0

app.py

@@ -0,0 +1,214 @@
+import http.client as http_client
+import json
+import logging
+import os
+import pprint
+import re
+import time
+import string
+
+import streamlit as st
+
+import streamlit.components.v1 as components
+from typing import Callable, Optional, Tuple, Union
+from pyserini import util
+from pyserini.search import LuceneSearcher, FaissSearcher, AutoQueryEncoder
+
+
+VERSION = '1.0'
+st.set_page_config(page_title="Miracl Search", layout="wide")
+
+os.makedirs(os.path.join(os.getcwd(),".streamlit"), exist_ok = True)
+with open(os.path.join(os.getcwd(),".streamlit/config.toml"), "w") as file:
+    file.write(
+        '[theme]\nbase="light"'
+    )
+
+Searcher = Union[FaissSearcher, LuceneSearcher]
+LANG_MAPPING = {'Arabic':'ar'}
+
+def download_indices(language):
+    if not os.path.exists('indices'):
+        os.mkdir("indices")
+
+    if not os.path.exists(f'indices/lucene-index.miracl-v1.0-{language}.20221004.2b2856'):
+        util.download_and_unpack_index(f"https://rgw.cs.uwaterloo.ca/JIMMYLIN-bucket0/pyserini-indexes/lucene-index.miracl-v1.0-{language}.20221004.2b2856.tar.gz", "indices")
+
+
+
+st.sidebar.markdown(
+"""
+<style>
+.aligncenter {
+    text-align: center;
+    font-weight: bold;
+    font-size: 30px;
+}
+</style>
+<p class="aligncenter">MIRACL Arabic Demo</p>
+<p class="aligncenter">🌍🙌🌏</p>
+<p style="text-align: center;"> MIRACL is a multilingual dataset for ad hoc retrieval that consists of 18 different languages, collectively encompassing over three billion native speakers around the world.</p>
+""",
+unsafe_allow_html=True,
+)
+
+st.sidebar.markdown(
+"""
+<style>
+.aligncenter {
+    text-align: center;
+}
+</style>
+<p style='text-align: center'>
+<a href="https://github.com/project-miracl" >GitHub</a> | <a href="https://arxiv.org/abs/2210.09984" >Paper</a>
+</p>
+""",
+unsafe_allow_html=True,
+)
+
+query = st.sidebar.text_input(label='Search query', value='')
+language = st.sidebar.selectbox(
+    'Language',
+    LANG_MAPPING.keys(),
+    index=3)
+max_results = st.sidebar.slider(
+    "Maximum Number of Results",
+    min_value=1,
+    max_value=1000,
+    step=1,
+    value=10,
+    help="Maximum Number of Documents to return",
+)
+
+
+def _load_sparse_searcher(language: str, k1: Optional[float]=None, b: Optional[float]=None) -> (Searcher):
+    searcher = LuceneSearcher(f'indices/lucene-index.miracl-v1.0-{language}.20221004.2b2856')
+    searcher.set_language(language)
+    if k1 is not None and b is not None:
+        searcher.set_bm25(k1, b)
+        retriever_name = f'BM25 (k1={k1}, b={b})'
+    else:
+        retriever_name = 'BM25'
+
+    return searcher
+
+def search(query, language, num_results=10):
+    searcher = _load_sparse_searcher(language=LANG_MAPPING[language])
+
+    t_0 = time.time()
+    search_results = searcher.search(query, k=num_results)
+    search_time = time.time() - t_0
+
+    results_dict ={"docs": [], "doc_ids": [], "score":[], "lang": language}
+    for i, result in enumerate(search_results):
+        result = json.loads(result.raw)
+        results_dict["docs"].append(result["text"])
+        results_dict["doc_ids"].append(result["docid"])
+        results_dict["score"].append(search_results[i].score)
+
+    return results_dict, search_time
+
+
+
+def highlight_string(paragraph: str, highlight_terms: list) -> str:
+    for term in highlight_terms:
+        paragraph = re.sub(f"\\b{term}\\b", f"<b>{term}</b>", paragraph, flags=re.I)
+    return paragraph
+
+def process_results(hits: dict, highlight_terms: list) -> str:
+    hit_list = []
+    for i in range(len(hits['doc_ids'])):
+        res_head = f"""
+                    <div class='searchresult'>
+                        <h2>{i+1}. Document ID: {hits['doc_ids'][i]}</h2>
+                        <p>Language: <string>{hits['lang']}</string>, Score: {round(hits['score'][i], 2)}</p>              
+                        <p>{highlight_string(hits['docs'][i], highlight_terms)}</p>
+                    </div>
+                    <hr>
+                    """
+        hit_list.append(res_head)
+    return " ".join(hit_list)
+
+
+
+if st.sidebar.button("Search"):
+    download_indices(LANG_MAPPING[language])
+    hits, search_time = search(query, language, max_results)
+    html_results = process_results(hits, [])
+    rendered_results = f"""
+            <div id="searchresultsarea">
+                <br>
+                <p id="searchresultsnumber">About {max_results} results</p>
+                {html_results}
+            </div>
+                        """
+    st.markdown("""
+                <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" 
+                    integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+                """,
+                unsafe_allow_html=True)
+    st.markdown(
+        """
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0/css/all.min.css">                                  
+        """,
+        unsafe_allow_html=True)
+    st.markdown(
+        f"""
+        <div class="row no-gutters mt-3 align-items-center">
+            <h2> Search Results </h2>
+        </div> 
+        """,
+        unsafe_allow_html=True)
+    components.html(
+        """
+        <style>
+        #searchresultsarea {
+            font-family: 'Arial';
+        }
+        
+        #searchresultsnumber {
+            font-size: 0.8rem;
+            color: gray;
+        }
+        
+        .searchresult h2 {
+            font-size: 19px;
+            line-height: 18px;
+            font-weight: normal;
+            color: rgb(7, 111, 222);
+            margin-bottom: 0px;
+            margin-top: 25px;
+        }
+        
+        .searchresult a {
+            font-size: 12px;
+            line-height: 12px;
+            color: green;
+            margin-bottom: 0px;
+        }
+    
+        .dark-mode {
+          color: white;
+        }
+        </style>
+        <script>
+        function load_image(id){
+            console.log(id)
+            var x = document.getElementById(id);
+            console.log(x)
+            if (x.style.display === "none") {
+                x.style.display = "block";
+            } else {
+                x.style.display = "none";
+            }
+        };
+        function myFunction() {
+           var element = document.body;
+           element.classList.toggle("dark-mode");
+        }
+        </script>
+        <button onclick="myFunction()">Toggle dark mode</button>
+        """ + rendered_results, height=800, scrolling=True
+        )
+
+

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5.fdt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a3711a2b0977bc2e49928e5a113e70b64ef32f3b776875f2beea98720fccf5f
+size 701613923

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5.nvd

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e780d2789d039a662615c248a04050f9d592f5667abf5cd77417dcf4a0f0cc75
+size 2061473

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5.tvd

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e7f73815d662265e24a0992b47a1f6d8229a4631d038b336672621db9c50a6e
+size 470885378

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.doc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbec3544ee98c9cb4cb71857ade75f882c2ed3aafbc579888b246d18519ce4dc
+size 123262995

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.dvd

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cdc9fff54b5c643f716746f105539031a84cd2b3352064451f74d526cd8511b
+size 21357338

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.pos

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30cd9ff3eae496fdc67878a4f48cf7ca47f82bbf8f9185ae1fd81bc36b45102d
+size 90321986

lucene-index.miracl-v1.0-ar.20221004.2b2856/_5_Lucene90_0.tim

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94783d87515ecacbb3c85495229cbfcc502aaa71d7ea058e9330bd0682f40fc8
+size 26275474

packages.txt

@@ -0,0 +1 @@
+openjdk-11-jdk

requirements.txt

@@ -0,0 +1,3 @@
+pyserini
+faiss-cpu
+torch