Update app.py

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 import os
-import tempfile
+
 from langchain_community.document_loaders import PyPDFLoader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import Chroma
@@ -10,39 +10,25 @@ from langchain_community.llms import HuggingFacePipeline
 from langchain.chains import ConversationChain
 from langchain.memory import ConversationBufferMemory
 from langchain_community.llms import HuggingFaceEndpoint
-from langchain_google_vertexai import VertexAI
-import vertexai
+
 from pathlib import Path
 import chromadb
 from unidecode import unidecode
+
 from transformers import AutoTokenizer
 import transformers
 import torch
 import tqdm 
 import accelerate
 import re
-from langchain_openai import AzureChatOpenAI
 
-def get_credentials():
-    creds_json_str = os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON")
-    if creds_json_str is None:
-        raise ValueError("GOOGLE_APPLICATION_CREDENTIALS_JSON not found in environment")
+from langchain_openai import AzureChatOpenAI
 
-    # create a temporary file
-    with tempfile.NamedTemporaryFile(mode="w+", delete=False, suffix=".json") as temp:
-        temp.write(creds_json_str) # write in json format
-        temp_filename = temp.name 
 
-    return temp_filename
-    
-# os.environ["GOOGLE_APPLICATION_CREDENTIALS"]= get_credentials()
 os.environ["AZURE_OPENAI_API_KEY"] = os.getenv("AZURE_OPENAI_API_KEY")
-
-
-
-
-
+# default_persist_directory = './chroma_HF/'
 list_llm = ["Azure-OpenAI"]
+
 list_llm_simple = [os.path.basename(llm) for llm in list_llm]
 
 # Load PDF document and create doc splits
@@ -88,17 +74,40 @@ def load_db():
 # Initialize langchain LLM chain
 def initialize_llmchain(llm_model, temperature, max_tokens, top_k, vector_db, progress=gr.Progress()):
     progress(0.1, desc="Initializing HF tokenizer...")
-
+    # HuggingFacePipeline uses local model
+    # Note: it will download model locally...
+    # tokenizer=AutoTokenizer.from_pretrained(llm_model)
+    # progress(0.5, desc="Initializing HF pipeline...")
+    # pipeline=transformers.pipeline(
+    #     "text-generation",
+    #     model=llm_model,
+    #     tokenizer=tokenizer,
+    #     torch_dtype=torch.bfloat16,
+    #     trust_remote_code=True,
+    #     device_map="auto",
+    #     # max_length=1024,
+    #     max_new_tokens=max_tokens,
+    #     do_sample=True,
+    #     top_k=top_k,
+    #     num_return_sequences=1,
+    #     eos_token_id=tokenizer.eos_token_id
+    #     )
+    # llm = HuggingFacePipeline(pipeline=pipeline, model_kwargs={'temperature': temperature})
     
     # HuggingFaceHub uses HF inference endpoints
     progress(0.5, desc="Initializing HF Hub...")
-    vertexai.init(project="imgcp-ff81e7053b072ce5", location="us-central1")
+    # Use of trust_remote_code as model_kwargs
+    # Warning: langchain issue
+    # URL: https://github.com/langchain-ai/langchain/issues/6080
     llm = AzureChatOpenAI(
-        azure_endpoint = "https://cloudcafe42.openai.azure.com/",
-        azure_deployment = "gpt-4",
-        openai_api_version = "2024-02-15-preview"
-    )
-
+            azure_endpoint = "https://cloudcafe42.openai.azure.com/", 
+            azure_deployment = "gpt-4",
+            openai_api_version = "2024-02-15-preview",
+            temperature = temperature,
+            max_new_tokens = max_tokens,
+            top_k = top_k
+        )
+    
     
     progress(0.75, desc="Defining buffer memory...")
     memory = ConversationBufferMemory(
@@ -228,13 +237,13 @@ def demo():
         collection_name = gr.State()
         
         gr.Markdown(
-        """<center><h2>Vertex AI PDF-based chatbot</center></h2>
+        """<center><h2>PDF-based chatbot</center></h2>
         <h3>Ask any questions about your PDF documents</h3>""")
         gr.Markdown(
-        """<b>Note:</b> This AI assistant, using Langchain and Vertex AI, performs retrieval-augmented generation (RAG) from your PDF documents. \
+        """<b>Note:</b> This AI assistant, using Langchain and open-source LLMs, performs retrieval-augmented generation (RAG) from your PDF documents. \
         The user interface explicitely shows multiple steps to help understand the RAG workflow. 
         This chatbot takes past questions into account when generating answers (via conversational memory), and includes document references for clarity purposes.<br>
-        <br><b>Warning:</b> This space uses the free CPU Basic hardware from Hugging Face. Some steps can take some time to generate a reply.
+        <br><b>Warning:</b> This space uses the free CPU Basic hardware from Hugging Face. Some steps and LLM models used below (free inference endpoints) can take some time to generate a reply.
         """)
         
         with gr.Tab("Step 1 - Upload PDF"):
@@ -314,8 +323,9 @@ def demo():
             inputs=None, \
             outputs=[chatbot, doc_source1, source1_page, doc_source2, source2_page, doc_source3, source3_page], \
             queue=False)
-    demo.queue().launch(share=True)
+    demo.queue().launch(debug=True)
 
 
 if __name__ == "__main__":
     demo()
+