Upload folder using huggingface_hub

.gitattributes

@@ -37,3 +37,4 @@ PDF/Anticiper-les-effets-de-l-adaptation-dun-rechauffement-climatique-de-plus-4-
 PDF/deu-2023.pdf filter=lfs diff=lfs merge=lfs -text
 PDF/memo_risques_physiques_focus_batiment_2022.pdf filter=lfs diff=lfs merge=lfs -text
 vectors/index.annoy filter=lfs diff=lfs merge=lfs -text
+sources/Anticiper-les-effets-de-l-adaptation-dun-rechauffement-climatique-de-plus-4-degres-quels-couts-de-l-adaptation.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -9,6 +9,9 @@ setAPIKEY.sh
 .AppleDouble
 .LSOverride
 
+# Historique conversasion with chatbot
+*.json
+
 # Icon must end with two \r
 Icon
 

README.md

@@ -1,5 +1,5 @@
 ---
-title: Clara
+title: clara
 app_file: app.py
 sdk: gradio
 sdk_version: 4.19.1

app.py

@@ -1,8 +1,7 @@
+
+
 # , get_pinecone_vectorstore, find_similar_vectors
-from climateqa.engine.vectorstore import build_vectores_stores
-from climateqa.engine.rag import make_rag_papers_chain
-from climateqa.engine.keywords import make_keywords_chain
-from climateqa.sample_questions import QUESTIONS
+from climateqa.engine.vectorstore import build_vectores_stores, get_PDF_Names_from_GCP
 from climateqa.engine.text_retriever import ClimateQARetriever
 from climateqa.engine.rag import make_rag_chain
 from climateqa.engine.llm import get_llm
@@ -11,11 +10,9 @@ from datetime import datetime
 import json
 import re
 import gradio as gr
-from climateqa.papers.openalex import OpenAlex
 from sentence_transformers import CrossEncoder
 
 reranker = CrossEncoder("mixedbread-ai/mxbai-rerank-xsmall-v1")
-oa = OpenAlex()
 
 # Load environment variables in local mode
 try:
@@ -25,9 +22,9 @@ except Exception as e:
     pass
 
 # Set up Gradio Theme
-theme = gr.themes.Base(
-    primary_hue="blue",
-    secondary_hue="red",
+theme = gr.themes.Soft(
+    primary_hue="yellow",
+    secondary_hue="orange",
     font=[gr.themes.GoogleFont("Poppins"), "ui-sans-serif",
           "system-ui", "sans-serif"],
 )
@@ -163,7 +160,7 @@ async def chat(query, history):
         "answer": history[-1][1],
         "time": timestamp,
     }
-    log_locally(log_file, logs)
+    #log_locally(log_file, logs)
 
     yield history, docs_html, output_query, output_language, gallery, output_query, output_keywords
 
@@ -181,7 +178,7 @@ def make_html_source(source, i):
         <div class="card-content">
             <div>
                 <div style="float:right;width 10%;position:relative;top:0px">
-                    <a href='{meta['ax_url']}'><img style="width:20px" src='/file/assets/download.png' /></a>
+                    <a href='{meta['ax_url']}' target='_blank'><img style="width:20px" src='/file/assets/download.png' /></a>
                 </div>
                 <div>
                     <h2>Extrait {i}</h2>
@@ -191,9 +188,9 @@ def make_html_source(source, i):
             <p>{text_content}</p>
 
         </div>
-        <div class="card-footer">
+        <!-- <div class="card-footer">
             <span>{name}</span>
-        </div>
+        </div> -->
     </div>
     """
 
@@ -209,79 +206,6 @@ def log_locally(file, logs):
         f.write(logs_json)
 
 
-def generate_keywords(query):
-    chain = make_keywords_chain(llm)
-    keywords = chain.invoke(query)
-    keywords = " AND ".join(keywords["keywords"])
-    return keywords
-
-
-papers_cols_widths = {
-    "doc": 50,
-    "id": 100,
-    "title": 300,
-    "doi": 100,
-    "publication_year": 100,
-    "abstract": 500,
-    "rerank_score": 100,
-    "is_oa": 50,
-}
-
-papers_cols = list(papers_cols_widths.keys())
-papers_cols_widths = list(papers_cols_widths.values())
-
-
-async def find_papers(query, keywords, after):
-
-    summary = ""
-
-    df_works = oa.search(keywords, after=after)
-    df_works = df_works.dropna(subset=["abstract"])
-    df_works = oa.rerank(query, df_works, reranker)
-    df_works = df_works.sort_values("rerank_score", ascending=False)
-    G = oa.make_network(df_works)
-
-    height = "750px"
-    network = oa.show_network(
-        G, color_by="rerank_score", notebook=False, height=height)
-    network_html = network.generate_html()
-
-    network_html = network_html.replace("'", "\"")
-    css_to_inject = "<style>#mynetwork { border: none !important; } .card { border: none !important; }</style>"
-    network_html = network_html + css_to_inject
-
-    network_html = f"""<iframe style="width: 100%; height: {height};margin:0 auto" name="result" allow="midi; geolocation; microphone; camera;
-    display-capture; encrypted-media;" sandbox="allow-modals allow-forms
-    allow-scripts allow-same-origin allow-popups
-    allow-top-navigation-by-user-activation allow-downloads" allowfullscreen=""
-    allowpaymentrequest="" frameborder="0" srcdoc='{network_html}'></iframe>"""
-
-    docs = df_works["content"].head(15).tolist()
-
-    df_works = df_works.reset_index(
-        drop=True).reset_index().rename(columns={"index": "doc"})
-    df_works["doc"] = df_works["doc"] + 1
-    df_works = df_works[papers_cols]
-
-    yield df_works, network_html, summary
-
-    chain = make_rag_papers_chain(llm)
-    result = chain.astream_log(
-        {"question": query, "docs": docs, "language": "English"})
-    path_answer = "/logs/StrOutputParser/streamed_output/-"
-
-    async for op in result:
-
-        op = op.ops[0]
-
-        if op['path'] == path_answer:  # reforulated question
-            new_token = op['value']  # str
-            summary += new_token
-        else:
-            continue
-        yield df_works, network_html, summary
-
-
 # --------------------------------------------------------------------
 # Gradio
 # --------------------------------------------------------------------
@@ -302,8 +226,13 @@ What would you like to know today?
 """
 
 
-with gr.Blocks(title="CLARA", css="style.css", theme=theme, elem_id="main-component") as demo:
+with gr.Blocks(title="CLARA", css="style.css", theme=theme, elem_id="main-component", elem_classes="ax_background") as demo:
 
+    gr.HTML("""
+        <img style="width:100px" src="file/assets/axionable.svg"/>
+    """, elem_classes="logo-axio ")
+
+    # TAB Clara
     with gr.Tab("CLARA"):
 
         with gr.Row(elem_id="chatbot-row"):
@@ -315,57 +244,44 @@ with gr.Blocks(title="CLARA", css="style.css", theme=theme, elem_id="main-compon
 
                 with gr.Row(elem_id="input-message"):
                     textbox = gr.Textbox(placeholder="Posez votre question", show_label=False,
-                                         scale=7, lines=1, interactive=True, elem_id="input-textbox")
+                                        scale=7, lines=1, interactive=True, elem_id="input-textbox")
 
-            with gr.Column(scale=1, variant="panel", elem_id="right-panel"):
 
-                with gr.Tabs() as tabs:
+            with gr.Column(scale=1, variant="panel", elem_id="right-panel"):
 
-                    with gr.Tab("Sources", elem_id="tab-citations", id=1):
-                        sources_textbox = gr.HTML(
-                            show_label=False, elem_id="sources-textbox")
-                        docs_textbox = gr.State("")
+                with gr.Column(scale=1, elem_id="tab-citations"):
+                
+                    gr.HTML("<p>Sources</p>")
 
-# ---------------------------------------------------------------------------------------
-# OTHER TABS
-# ---------------------------------------------------------------------------------------
+                    sources_textbox = gr.HTML(
+                        show_label=False, elem_id="sources-textbox")
+                    docs_textbox = gr.State("")
 
-    with gr.Tab("Figures", elem_id="tab-images", elem_classes="max-height other-tabs"):
-        gallery_component = gr.Gallery()
+                # l'object tabs est necessaire actuellement
+                # J'ai l'impression qu'il est utiliser pour freezre les contenu des tabs
+                # pendant que l'ia gènère une reponse ..
+                with gr.Tabs() as tabs:
+                    None
 
-    with gr.Tab("Papers (beta)", elem_id="tab-papers", elem_classes="max-height other-tabs"):
 
+    # TAB A propos
+    with gr.Tab("À propos", elem_classes="max-height other-tabs"):
         with gr.Row():
             with gr.Column(scale=1):
-                query_papers = gr.Textbox(
-                    placeholder="Question", show_label=False, lines=1, interactive=True, elem_id="query-papers")
-                keywords_papers = gr.Textbox(
-                    placeholder="Keywords", show_label=False, lines=1, interactive=True, elem_id="keywords-papers")
-                after = gr.Slider(minimum=1950, maximum=2023, step=1, value=1960,
-                                  label="Publication date", show_label=True, interactive=True, elem_id="date-papers")
-                search_papers = gr.Button(
-                    "Search", elem_id="search-papers", interactive=True)
-
-            with gr.Column(scale=7):
-
-                with gr.Tab("Summary", elem_id="papers-summary-tab"):
-                    papers_summary = gr.Markdown(
-                        visible=True, elem_id="papers-summary")
+                gr.Markdown(
+                    ("CLARA (Climate LLM for Adaptation & Risks Answers) by [Axionable](https://www.axionable.com/)"
+                    "– Fork de [ClimateQ&A](https://huggingface.co/spaces/Ekimetrics/climate-question-answering/tree/main)"), elem_classes="a-propos")
 
-                with gr.Tab("Relevant papers", elem_id="papers-results-tab"):
-                    papers_dataframe = gr.Dataframe(
-                        visible=True, elem_id="papers-table", headers=papers_cols)
 
-                with gr.Tab("Citations network", elem_id="papers-network-tab"):
-                    citations_network = gr.HTML(
-                        visible=True, elem_id="papers-citations-network")
+#    # TAB Configuration
+#    with gr.Tab("Configuration"):
+#
+#        with gr.Row(elem_id="config-row"):
+#            with gr.Column(scale=1):
+#
+#                for pdfName in get_PDF_Names_from_GCP():
+#                        gr.Markdown( pdfName, elem_classes="a-propos")
 
-    with gr.Tab("À propos", elem_classes="max-height other-tabs"):
-        with gr.Row():
-            with gr.Column(scale=1):
-                gr.Markdown(
-                    "CLARA (Climate LLM for Adaptation & Risks Answers) by [Axionable](https://www.axionable.com/)"
-                    "– Fork de [ClimateQ&A](https://huggingface.co/spaces/Ekimetrics/climate-question-answering/tree/main)")
 
     def start_chat(query, history):
         history = history + [(query, None)]
@@ -382,21 +298,8 @@ with gr.Blocks(title="CLARA", css="style.css", theme=theme, elem_id="main-compon
      )
 
 
-
-    def change_sample_questions(key):
-        index = list(QUESTIONS.keys()).index(key)
-        visible_bools = [False] * len(samples)
-        visible_bools[index] = True
-        return [gr.update(visible=visible_bools[i]) for i in range(len(samples))]
-
- #   dropdown_samples.change(change_sample_questions,dropdown_samples,samples)
-
-    query_papers.submit(generate_keywords, [query_papers], [keywords_papers])
-    search_papers.click(find_papers, [query_papers, keywords_papers, after], [
-                        papers_dataframe, citations_network, papers_summary])
-
     demo.queue()
 
 demo.launch(allowed_paths=["assets/download.png",
-            "assets/logo4.png"], 
-            favicon_path="assets/logo4.png")
+            "assets/logo4.png",
+            "assets/axionable.svg"],favicon_path="assets/logo4.png")

climateqa/engine/embeddings.py

@@ -8,8 +8,12 @@ def get_embeddings_function(version = "v1.2"):
 
         # https://huggingface.co/BAAI/bge-base-en-v1.5
         # Best embedding model at a reasonable size at the moment (2023-11-22)
-    
-        model_name = "BAAI/bge-base-en-v1.5"
+        # model_name = "BAAI/bge-base-en-v1.5"
+
+        # https://huggingface.co/BAAI/bge-m3
+        # A better one from 2024-04
+        model_name = "BAAI/bge-m3"
+
         encode_kwargs = {'normalize_embeddings': True} # set True to compute cosine similarity
         print("Loading embeddings model: ", model_name)
         embeddings_function = HuggingFaceBgeEmbeddings(

climateqa/engine/vectorstore.py

@@ -1,98 +1,94 @@
-# Pinecone
-# More info at https://docs.pinecone.io/docs/langchain
-# And https://python.langchain.com/docs/integrations/vectorstores/pinecone
-#import os
-#from pinecone import Pinecone
-#from langchain_community.vectorstores import Pinecone as PineconeVectorstore
 
-# LOAD ENVIRONMENT VARIABLES
-#try:
-#    from dotenv import load_dotenv
-#    load_dotenv()
-#except:
-#    pass
+from google.cloud import storage
+#storage_client = storage.Client()
+storage_client = storage.Client.create_anonymous_client()
+bucket_name = "docs-axio-clara"
 
+from langchain_pinecone import PineconeVectorStore
 
-#def get_pinecone_vectorstore(embeddings,text_key = "content"):
+from langchain_community.document_loaders import TextLoader
+from langchain_text_splitters import CharacterTextSplitter
+from climateqa.engine.embeddings import get_embeddings_function
+embeddings_function = get_embeddings_function()
 
-    # # initialize pinecone
-    # pinecone.init(
-    #     api_key=os.getenv("PINECONE_API_KEY"),  # find at app.pinecone.io
-    #     environment=os.getenv("PINECONE_API_ENVIRONMENT"),  # next to api key in console
-    # )
 
-    # index_name = os.getenv("PINECONE_API_INDEX")
-    # vectorstore = Pinecone.from_existing_index(index_name, embeddings,text_key = text_key)
 
-    # return vectorstore
+index_name = "my-index"
+namespace = "my-namespace"
 
-#    pc = Pinecone(api_key=os.getenv("PINECONE_API_KEY"))
-#    index = pc.Index(os.getenv("PINECONE_API_INDEX"))
 
-#    vectorstore = PineconeVectorstore(
-#        index, embeddings, text_key,
-#    )
-#    return vectorstore
+import os
+import pdfplumber
 
+def get_PDF_Names_from_GCP():
 
+    listName = []
+    # Récupération des fichier depuis GCP storage
+    blobs = storage_client.list_blobs(bucket_name, prefix='sources/')
+    for blob in blobs:
+        listName.append(blob.name)
+    return listName
 
-# def get_pinecone_retriever(vectorstore,k = 10,namespace = "vectors",sources = ["IPBES","IPCC"]):
+def get_PDF_from_GCP(folder_path, pdf_folder="./PDF"):
 
-#     assert isinstance(sources,list)
+    # Récupération des fichier depuis GCP storage
+    blobs = storage_client.list_blobs(bucket_name, prefix='sources/')
+    for blob in blobs:
 
-#     # Check if all elements in the list are either IPCC or IPBES
-#     filter = {
-#         "source": { "$in":sources},
-#     }
+        print( "\n"+blob.name+":")
+        print( " <- Téléchargement Depuis GCP")
+        blob.download_to_filename(pdf_folder+"/"+blob.name)
 
-#     retriever = vectorstore.as_retriever(search_kwargs={
-#         "k": k,
-#         "namespace":"vectors",
-#         "filter":filter
-#     })
+        # Extraction des textes dpuis les fichiers PDF
+        print(" >>> Extraction PDF")
+        for pdf_file in os.listdir(pdf_folder):
+            if pdf_file.startswith("."):
+                continue
+            print(" >  "+pdf_folder+"/"+pdf_file)
+            pdf_total_pages = 0
+            with pdfplumber.open(pdf_folder+"/"+pdf_file) as pdf:
+                pdf_total_pages = len(pdf.pages)
+            
+            # Fuite mémoire pour les gros fichiers
+            # Reouvrir le fichier à chaque N page semble rélgler le problème
+            N_page = 300
+            page_number = 0
+            while page_number < pdf_total_pages:
 
-#     return retriever
+                print(" -- ouverture du fichier pour "+str(N_page)+ " pages --" )
+                with pdfplumber.open(pdf_folder+"/"+pdf_file) as pdf:
 
-from langchain_community.vectorstores import Annoy
-from langchain_community.document_loaders import TextLoader
-from langchain_text_splitters import CharacterTextSplitter
-from climateqa.engine.embeddings import get_embeddings_function
-embeddings_function = get_embeddings_function()
+                    npage = 0
+                    while (npage < N_page and page_number < pdf_total_pages) :
 
-#def build_vectores_stores(content_path):
-#    loader = TextLoader(content_path)
-#    documents = loader.load()
-#    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
-#    docs = text_splitter.split_documents(documents)
+                        print(" >>> "+str(page_number+1))
+                        f = open(folder_path+"/"+pdf_file+"..:page:.."+str(page_number+1), "w")
+                        for char_pdf in pdf.pages[page_number].chars:
+                            f.write(char_pdf["text"])
+                        f.close()
 
-#    vector_store_from_docs = Annoy.from_documents(docs, embeddings_function)
-#    return vector_store_from_docs
+                        npage = npage + 1
+                        page_number = page_number + 1
 
 
-import os
-import pdfplumber
+        print(" X removing: " + blob.name )
+        os.remove(pdf_folder+"/"+blob.name)
+
 
 def build_vectores_stores(folder_path, pdf_folder="./PDF", vectors_path = "./vectors"):
-  
-    if os.path.isfile(vectors_path+"/index.annoy"):
-        return Annoy.load_local(vectors_path, embeddings_function,allow_dangerous_deserialization=True)
-
-    # Extract text from PDF files
-    print("Extraction PDF ...")
-    for pdf_file in os.listdir(pdf_folder):
-        if pdf_file.startswith("."):
-            continue
-        print(" >  "+pdf_folder+"/"+pdf_file)
-        with pdfplumber.open(pdf_folder+"/"+pdf_file) as pdf:
-            for pdf_page in pdf.pages:
-                f = open(folder_path+"/"+pdf_file+" page "+str(pdf_page.page_number), "w")
-                # f.write(pdf_file+" page "+str(pdf_page.page_number))
-                for char_pdf in pdf_page.chars:
-                    f.write(char_pdf["text"])
-                f.close()
+
+    vectorstore = PineconeVectorStore(
+    index_name=index_name,
+    embedding=embeddings_function,
+    #namespace=namespace
+    )
+
+    return vectorstore
+
+    print(" Vectorisation ...")
 
     docs = []
-    vector_store_from_docs = ()  # Créer un nouvel objet Annoy ou utiliser celui déjà initialisé selon votre code existant
+    #vector_store_from_docs = ()  # Créer un nouvel objet Annoy ou utiliser celui déjà initialisé selon votre code existant
     for filename in os.listdir(folder_path):
         if filename.startswith("."):
             continue
@@ -103,12 +99,17 @@ def build_vectores_stores(folder_path, pdf_folder="./PDF", vectors_path = "./vec
 
             for doc in documents:
                 if (doc.metadata):
-                    doc.metadata["ax_page"] = doc.metadata['source'].split(" ")[-1]
-                    doc.metadata["ax_name"] = doc.metadata['source'].split(" ")[0].split("/")[-1]
+                    doc.metadata["ax_page"] = doc.metadata['source'].split("..:page:..")[-1]
+                    doc.metadata["ax_name"] = doc.metadata['source'].split("..:page:..")[0].split("/")[-1]
                     doc.metadata["ax_url"] = "https://storage.googleapis.com/docs-axio-clara/sources/"+doc.metadata["ax_name"]
 
             text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
             docs += text_splitter.split_documents(documents)
-    vector_store_from_docs = Annoy.from_documents(docs, embeddings_function)
-    vector_store_from_docs.save_local(vectors_path)
-    return vector_store_from_docs
+    #vector_store_from_docs = Annoy.from_documents(docs, embeddings_function)
+    vectorstore = PineconeVectorStore.from_documents(docs, embeddings_function, index_name=index_name)
+    #vector_store_from_docs.save_local(vectors_path)
+    return vectorstore
+
+
+    print("MISSING VECTORS")
+    exit(0)

requirements.txt

@@ -1,5 +1,5 @@
+google-cloud-storage==2.16.0
 gradio==4.19.1
-gunicorn==22.0.0
 python-dotenv==1.0.0
 langchain==0.1.10
 langchain_openai==0.0.6
@@ -10,5 +10,4 @@ msal
 pyalex==0.13
 networkx==3.2.1
 pyvis==0.3.2
-annoy==1.17.3
-pdfplumber
+annoy==1.17.3

sources/Anticiper-les-effets-de-l-adaptation-dun-rechauffement-climatique-de-plus-4-degres-quels-couts-de-l-adaptation.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be9d2d29a6545fc1949b10eb8428e6fac632aa84020fa61f4f76600817a21cd5
+size 2079496

style.css

@@ -3,6 +3,78 @@
     --user-image: url('https://ih1.redbubble.net/image.4776899543.6215/st,small,507x507-pad,600x600,f8f8f8.jpg');
   } */
 
+.logo-axio {
+    float: right;
+    position: absolute;
+    right: 0px;
+}
+
+
+/* couleur text */
+p {
+    color: black !important;
+}
+li {
+    color: black !important;
+}
+
+button.selected {
+    border-radius: 20px !important;
+}
+button:hover {
+    color: #ffc000 !important;
+}
+
+
+/* fond panels/blocks */
+.panel {
+    background-color: #eeeeee !important;
+    border: 0px;
+}
+.block {
+    background-color: #eeeeee !important;
+}
+
+/* fond bot */
+.bot {
+    background-color: #eeeeee !important;
+}
+
+/* avatar en debut de reponse */
+.avatar-container {
+    align-self: baseline !important;
+    margin-top: 35px;
+}
+
+
+
+/* fond user */
+.user {
+    background-color: #d2d2d2 !important;
+}
+textarea {
+    background-color: #d2d2d2 !important;
+    color: black !important;
+}
+
+
+/* fond app */
+gradio-app {
+    background-color: #ffffff !important;
+}
+.gradio-container {
+    background-color: #ffffff !important;
+    max-width: 100% !important;
+    width: 100% !important;
+}
+
+
+.a-propos {
+    margin: 20px !important;
+}
+
+
+
 .telecharger {
     border: 1px solid;
     padding: 5px;
@@ -43,7 +115,7 @@ body.dark .warning-box * {
 
 
 body.dark .tip-box * {
-    color:black !important;
+    color:rgb(216, 216, 216) !important;
 }
 
 

test

@@ -19,8 +19,7 @@ ENV HOME=/home/user \
 	GRADIO_NUM_PORTS=1 \
 	GRADIO_SERVER_NAME=0.0.0.0 \
 	GRADIO_THEME=huggingface \
-	SYSTEM=spaces  \
-	PORT=7860
+	SYSTEM=spaces
  
 # Set the working directory to the user's home directory
 WORKDIR $HOME/app
@@ -28,8 +27,6 @@ WORKDIR $HOME/app
 # Copy the current directory contents into the container at $HOME/app setting the owner to the user
 COPY --chown=user . $HOME/app
 
-#CMD ["python","setup.py"]
+CMD ["python","setup.py"]
 
-#CMD ["python", "app.py"]
-
-CMD gunicorn -b 0.0.0.0:$PORT app:demo
+CMD ["python", "app.py"]