Upload 3 files

app.py

@@ -6,13 +6,13 @@ import torch
 from sentence_transformers import SentenceTransformer, util
 
 # Set Streamlit page configuration
-st.set_page_config(page_title="home", layout="wide")
+st.set_page_config(page_title="App", layout="wide")
 
-st.title("Streamlit App around GPT")
+st.title("HAL UNIV-COTEDAZUR Collection Semantic Search")
 
 model_options = st.sidebar.selectbox("Choose the free embeddings model to use ?", ('all-MiniLM-L6-v2', 'multi-qa-mpnet-base-dot-v1'))
 
-if ok_openai := st.checkbox('You want to use the OpenAI Embeddings'):
+if ok_openai := st.sidebar.checkbox('You want to use the OpenAI Embeddings too'):
     if openai_api_key := st.text_input(
         ":blue[Put Your OPENAI API-KEY :]",
         placeholder="Paste your OpenAI API key here ",

pages/documentation.py

@@ -0,0 +1,68 @@
+import streamlit as st
+
+# Set Streamlit page configuration
+st.set_page_config(page_title="Documentation", layout="wide")
+
+# Set up the Streamlit app layout
+st.title("Documentation")
+
+st.header("Dataset creation")
+
+st.subheader(":blue[HAL API harvest]")
+
+st.write("All the API documentation is available [here](https://api.archives-ouvertes.fr/docs/search)")
+st.write("All records of article type publications reported in the UNIV-COTEDAZUR collection of HAL are obtained with this recursive function that populates a pandas Dataframe as output ")
+st.code("""
+global_list = []
+def recursive_hal_harvest(cursor="*"):
+  url = f"https://api.archives-ouvertes.fr/search/UNIV-COTEDAZUR/?q=docType_s:ART&rows=1000&cursorMark={cursor}&fl=uri_s,title_s,subTitle_s,authFullName_s,producedDate_s,domain_t,journalTitle_s,journalPublisher_s,anrProjectCallTitle_s,abstract_s&sort=docid asc"
+  print(url)
+  response = requests.request("GET", url).text
+  data = json.loads(response)
+  for doc in data["response"]["docs"]:
+    global_list.append(doc)
+  if len(data["response"]["docs"]) != 0:
+    return recursive_hal_harvest(cursor=data["nextCursorMark"])
+  else:
+    return global_list
+df = pd.DataFrame(recursive_hal_harvest())
+
+""", language='python')
+
+st.write("The dataframe's colmumns of metadata are then concatenated into a single combined text in a new column. It is therefore on this new column that the different embeddings models will be applied to encode this combined text and output a single vector embedding.")
+st.code("""
+df = df.astype(str)
+df["combined"] = (
+    "Title: " + df.title_s + ";Subtitle:" + df.subTitle_s + ";Author:" + df.authFullName_s + ";Date:" + df.producedDate_s + ";Journal Title:" + df.journalTitle_s + ";Publisher:" + df.journalPublisher_s + ";ANR Project:" + df.anrProjectCallTitle_s + "; Abstract: " + df.abstract_s
+)
+
+""", language='python')
+
+st.subheader(":blue[OpenAI Embeddings]")
+
+st.code("""
+import openai
+import tiktoken
+from openai.embeddings_utils import get_embedding
+        
+openai.api_key = os.getenv("OPENAI_API_KEY")
+
+# embedding model parameters
+embedding_model = "text-embedding-ada-002"
+embedding_encoding = "cl100k_base"  # this the encoding for text-embedding-ada-002
+max_tokens = 8000  # the maximum for text-embedding-ada-002 is 8191
+
+# filtering dataset on text under the max tokens limit      
+encoding = tiktoken.get_encoding(embedding_encoding)
+df["n_tokens"] = df.combined.apply(lambda x: len(encoding.encode(x)))
+df = df[df.n_tokens <= max_tokens]
+        
+# générate embeddings
+df["openai_embedding"] = df.combined.apply(lambda x: get_embedding(x, engine=embedding_model) )
+df["openai_embedding"] = df.embedding.astype(str).apply(eval).apply(np.array)
+
+""", language='python')
+
+st.subheader(":blue[Huggingface free models for Embeddings]")
+
+st.write("The open source Huggingface platform hosts a large number of pre-trained models that can then be reused for many tasks (text or image classification, summarization, document QA etc...). We can then use the popular sentence-transformers library applied on free available text embedding models for creating embeddings ")

requirements.txt

@@ -39,6 +39,7 @@ openai==0.27.8
 packaging==23.1
 pandas==2.0.3
 Pillow==9.5.0
+plotly==5.15.0
 protobuf==4.23.4
 pyarrow==12.0.1
 pydeck==0.8.1b0