Upload 106 files

.env

@@ -1 +1 @@
-OPENAI_API_KEY="sk-proj-2FjtAue570qdHBcvCiTGT3BlbkFJNV8SYnSFHm6j8wjI0ARe"
+OPENAI_API_KEY=""

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "web-services-samples"]
-	path = web-services-samples
-	url = https://github.com/onetcenter/web-services-samples.git
+[submodule "web-services-samples"]
+	path = web-services-samples
+	url = https://github.com/onetcenter/web-services-samples.git

bot/bot-langchain-chat.py

@@ -1,96 +1,96 @@
-import streamlit as st
-from langchain.agents import AgentExecutor, create_openai_functions_agent,Tool
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_openai import ChatOpenAI
-from langchain.memory import ConversationBufferMemory
-from langchain.prompts import MessagesPlaceholder
-from langchain.schema.messages import SystemMessage
-import asyncio
-from dotenv import load_dotenv
-load_dotenv()
-
-st.set_page_config(page_title="Chatbot")
-st.header('Basic Chatbot')
-st.write('Allows users to interact with the Career Roadmap Genrator')
-agent_type = st.selectbox(
-    'How would you like to be contacted?',
-    ('Code Assistant', 'General Assistant', 'Roadmap Generator'))
-st.write('You selected:', agent_type)
-#st.write('[![view source code ](https://img.shields.io/badge/view_source_code-gray?logo=github)](https://github.com/shashankdeshpande/langchain-chatbot/blob/master/pages/1_%F0%9F%92%AC_basic_chatbot.py)')
-
-def create_agent(model_name,agent_type):
-    print('model:',model_name)
-    print('Assistant Type:',agent_type)
-    # agent_kwargs = {
-    #     "extra_prompt_messages": [MessagesPlaceholder(variable_name="memory")],
-    # }
-    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
-
-    increment = Tool(
-            name="Increment",
-            func=lambda x:x+'1',
-            description="useful to add one to input.",
-    )
-    tools = [increment]
-    llm = ChatOpenAI(model=model_name, temperature=0)
-    # agent_exe =  initialize_agent(
-    #     tools=[increment],
-    #     llm=llm, 
-    #     agent=AgentType.OPENAI_FUNCTIONS,
-    #     agent_kwargs=agent_kwargs,
-    #     memory = memory,
-    #     verbose=True,
-    # )
-    system_message = ""
-    if agent_type == 'Code Assistant':
-        system_message = """You are a code assistant. 
-        Answer questions in code with minimal to no explanation.
-        Put brief one line comments on the code for explanation.
-        """
-    elif agent_type == 'General Assistant':
-        system_message = """You are a general AI assistant. 
-        Answer questions with minimal and to the point explanation.
-        Don't put safety and cultural warnings. Only warn about security. 
-        """
-    else:
-        system_message = """You are a Career Roadmap Generator.
-        Answer questions with the help of given job description and create breif step by step solutions for every job description user provides to get that role in that company.
-        Put step by step process to get the job for the specific job description. List as many most relevant skills as possble for that role at that company.
-        If possible provide few projects to work on before applying for that role which will increace the chance of getting selected.
-        Add the resources to learn, watch, practice if possible for each step. Don't give me generic roadmap. Provide in-depth roadmap.
-        Link all the realatd skills and give what skill to learn first followed by another in the roadmap. 
-        """
-
-    prompt = ChatPromptTemplate.from_messages(
-        [
-            ("system", system_message),
-            MessagesPlaceholder("chat_history", optional=True),
-            ("human", "{input}"),
-            MessagesPlaceholder("agent_scratchpad"),
-        ]
-    )
-    agent = create_openai_functions_agent(llm, tools, prompt)
-    agent_exe = AgentExecutor(agent=agent, tools=tools,memory=memory)
-    return agent_exe
-
-async def main():
-    #chain = self.setup_chain()
-    #agent_exe = create_agent("gpt-3.5-turbo","General Assistant")
-    agent_exe = create_agent("gpt-4-turbo-preview", agent_type)
-    user_query = st.chat_input(placeholder="Ask your Question!")
-    if "messages" not in st.session_state:
-        st.session_state["messages"] = [{"role": "assistant", "content": "Hello! I'm here to help with your career progression needs."}]
-    if user_query:
-        st.session_state["messages"].append({"role": 'user', "content": user_query})
-        #with st.chat_message("assistant"):
-            #st_cb = StreamHandler(st.empty())
-            #response = chain.run(user_query, callbacks=[st_cb])
-        response = await agent_exe.ainvoke(input={"input":user_query})
-        st.session_state["messages"].append({"role": "assistant", "content": response['output']})
-
-    for msg in st.session_state["messages"]:
-            st.chat_message(msg["role"]).write(msg["content"])
-    print(st.session_state["messages"])
-
-if __name__ == "__main__":
+import streamlit as st
+from langchain.agents import AgentExecutor, create_openai_functions_agent,Tool
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_openai import ChatOpenAI
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import MessagesPlaceholder
+from langchain.schema.messages import SystemMessage
+import asyncio
+from dotenv import load_dotenv
+load_dotenv()
+
+st.set_page_config(page_title="Chatbot")
+st.header('Basic Chatbot')
+st.write('Allows users to interact with the Career Roadmap Genrator')
+agent_type = st.selectbox(
+    'How would you like to be contacted?',
+    ('Code Assistant', 'General Assistant', 'Roadmap Generator'))
+st.write('You selected:', agent_type)
+#st.write('[![view source code ](https://img.shields.io/badge/view_source_code-gray?logo=github)](https://github.com/shashankdeshpande/langchain-chatbot/blob/master/pages/1_%F0%9F%92%AC_basic_chatbot.py)')
+
+def create_agent(model_name,agent_type):
+    print('model:',model_name)
+    print('Assistant Type:',agent_type)
+    # agent_kwargs = {
+    #     "extra_prompt_messages": [MessagesPlaceholder(variable_name="memory")],
+    # }
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
+
+    increment = Tool(
+            name="Increment",
+            func=lambda x:x+'1',
+            description="useful to add one to input.",
+    )
+    tools = [increment]
+    llm = ChatOpenAI(model=model_name, temperature=0)
+    # agent_exe =  initialize_agent(
+    #     tools=[increment],
+    #     llm=llm, 
+    #     agent=AgentType.OPENAI_FUNCTIONS,
+    #     agent_kwargs=agent_kwargs,
+    #     memory = memory,
+    #     verbose=True,
+    # )
+    system_message = ""
+    if agent_type == 'Code Assistant':
+        system_message = """You are a code assistant. 
+        Answer questions in code with minimal to no explanation.
+        Put brief one line comments on the code for explanation.
+        """
+    elif agent_type == 'General Assistant':
+        system_message = """You are a general AI assistant. 
+        Answer questions with minimal and to the point explanation.
+        Don't put safety and cultural warnings. Only warn about security. 
+        """
+    else:
+        system_message = """You are a Career Roadmap Generator.
+        Answer questions with the help of given job description and create breif step by step solutions for every job description user provides to get that role in that company.
+        Put step by step process to get the job for the specific job description. List as many most relevant skills as possble for that role at that company.
+        If possible provide few projects to work on before applying for that role which will increace the chance of getting selected.
+        Add the resources to learn, watch, practice if possible for each step. Don't give me generic roadmap. Provide in-depth roadmap.
+        Link all the realatd skills and give what skill to learn first followed by another in the roadmap. 
+        """
+
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", system_message),
+            MessagesPlaceholder("chat_history", optional=True),
+            ("human", "{input}"),
+            MessagesPlaceholder("agent_scratchpad"),
+        ]
+    )
+    agent = create_openai_functions_agent(llm, tools, prompt)
+    agent_exe = AgentExecutor(agent=agent, tools=tools,memory=memory)
+    return agent_exe
+
+async def main():
+    #chain = self.setup_chain()
+    #agent_exe = create_agent("gpt-3.5-turbo","General Assistant")
+    agent_exe = create_agent("gpt-4-turbo-preview", agent_type)
+    user_query = st.chat_input(placeholder="Ask your Question!")
+    if "messages" not in st.session_state:
+        st.session_state["messages"] = [{"role": "assistant", "content": "Hello! I'm here to help with your career progression needs."}]
+    if user_query:
+        st.session_state["messages"].append({"role": 'user', "content": user_query})
+        #with st.chat_message("assistant"):
+            #st_cb = StreamHandler(st.empty())
+            #response = chain.run(user_query, callbacks=[st_cb])
+        response = await agent_exe.ainvoke(input={"input":user_query})
+        st.session_state["messages"].append({"role": "assistant", "content": response['output']})
+
+    for msg in st.session_state["messages"]:
+            st.chat_message(msg["role"]).write(msg["content"])
+    print(st.session_state["messages"])
+
+if __name__ == "__main__":
    asyncio.run(main())

bot/langchain-bot/bot-langchain-chat.py

@@ -1,50 +1,54 @@
-import streamlit as st
-import asyncio
-from engine.langchain_agent import create_agent, run_agent
-from dotenv import load_dotenv
-#from openai import OpenAI
-
-load_dotenv()
-import os
-# print(os.environ["OPENAI_API_KEY"])
-# Client = OpenAI()
-
-st.title("Langchain Agent")
-if "memory" not in st.session_state:
-    st.session_state["memory"] = [{"role":"system","content":""}]
-
-if "agent" not in st.session_state:
-    #model_name = "gpt-3.5-turbo"
-    model_name = "gpt-4-turbo-preview"
-    st.session_state['agent'] = create_agent(model_name)
-
-# updading the chat page with messages
-for message in st.session_state["memory"]:
-    if message["role"] == "assistant":
-        with st.chat_message(message["role"]):
-            msg = message["content"]["output"]
-            if "/bot/images/dall-e" in msg:
-                address = msg.split("(")[1][:-1]
-                print("address:",address)
-                st.image(address)
-            else:
-                st.markdown(msg)
-    elif message["role"] == "system":
-        pass
-    else:
-        with st.chat_message(message["role"]):
-            st.markdown(message["content"])
-
-# entering new message event handle
-if prompt := st.chat_input("Your message ..."):
-    st.session_state['memory'].append({"role":"user","content":prompt})
-    with st.chat_message("user"):
-        st.markdown(prompt)
-    response = asyncio.run(run_agent(st.session_state["agent"],prompt))
-    st.session_state['memory'].append({"role":"assistant","content":response})
-    with st.chat_message("assistant"):
-        if "/bot/images/dall-e" in response["output"]:
-            address = response["output"].split("(")[1][:-1]
-            st.image(address)
-        else:
+import streamlit as st
+import asyncio
+from engine.langchain_agent import create_agent, run_agent
+from dotenv import load_dotenv
+#from openai import OpenAI
+
+load_dotenv()
+import os
+# print(os.environ["OPENAI_API_KEY"])
+# Client = OpenAI()
+
+st.set_page_config(page_title="Career Roadmap Generator")
+st.header("Paste a job description and get a roadmap")
+st.write('Allows users to interact with the LLM')
+
+if "memory" not in st.session_state:
+    st.session_state["memory"] = [{"role":"system","content":""}]
+
+if "agent" not in st.session_state:
+    #model_name = "gpt-3.5-turbo"
+    #model_name = "gpt-4-turbo-preview"
+    model_name = "gpt-4o"
+    st.session_state['agent'] = create_agent(model_name)
+
+# updading the chat page with messages
+for message in st.session_state["memory"]:
+    if message["role"] == "assistant":
+        with st.chat_message(message["role"]):
+            msg = message["content"]["output"]
+            if "/bot/images/dall-e" in msg:
+                address = msg.split("(")[1][:-1]
+                print("address:",address)
+                st.image(address)
+            else:
+                st.markdown(msg)
+    elif message["role"] == "system":
+        pass
+    else:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+# entering new message event handle
+if prompt := st.chat_input("Your message ..."):
+    st.session_state['memory'].append({"role":"user","content":prompt})
+    with st.chat_message("user"):
+        st.markdown(prompt)
+    response = asyncio.run(run_agent(st.session_state["agent"],prompt))
+    st.session_state['memory'].append({"role":"assistant","content":response})
+    with st.chat_message("assistant"):
+        if "/bot/images/dall-e" in response["output"]:
+            address = response["output"].split("(")[1][:-1]
+            st.image(address)
+        else:
             st.markdown(response["output"])

bot/langchain-bot/engine/langchain_agent.py

@@ -1,48 +1,52 @@
-from langchain.agents import AgentExecutor, create_openai_tools_agent,Tool
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-from langchain_community.vectorstores import Chroma
-from langchain import hub
-from langchain.memory import ConversationBufferMemory
-from langchain.prompts import MessagesPlaceholder
-from engine.tools import GPT35TCodeGen, GPT4TAssistant, GPT4TCodeGen, DalleImageGen,RAGTool, CombinedTool, CareerRoadmapGenerator
-
-def create_agent(model_name):
-
-    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
-    rag_llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
-    #rag_llm = ChatOpenAI(model_name="gpt-4-turbo-preview", temperature=0)
-    rag_prompt = hub.pull("rlm/rag-prompt")
-    rag_db = Chroma(persist_directory="../../chroma_db", 
-                       embedding_function=OpenAIEmbeddings())
-    rag_retriever = rag_db.as_retriever()
-    
-    #tools = [GPT35TCodeGen(),GPT4TAssistant(),GPT4TCodeGen(), DalleImageGen(), RAGTool(rag_retriever,rag_llm,rag_prompt), CombinedTool(rag_retriever,rag_llm,rag_prompt), CareerRoadmapGenerator(rag_retriever,rag_llm,rag_prompt)]
-    tools = [GPT4TAssistant(),GPT4TCodeGen(), DalleImageGen(), RAGTool(rag_retriever,rag_llm,rag_prompt)]
-
-    llm = ChatOpenAI(model=model_name, temperature=0)
-
-    system_message = "You are a general AI assistant.\n" + \
-    "Don't answer the question if you are not getting the answer from a tool.\n" + \
-    "Don't change the answers you receive from a tool. Just pass them to the user."
-    "Don't put safety and cultural warnings. Only warn about security."
-
-    prompt = ChatPromptTemplate.from_messages(
-        [
-            ("system", system_message),
-            MessagesPlaceholder("chat_history", optional=True),
-            ("human", "{input}"),
-            MessagesPlaceholder("agent_scratchpad"),
-        ]
-    )
-    agent = create_openai_tools_agent(llm, tools, prompt)
-    agent_exe = AgentExecutor(agent=agent, tools=tools,memory=memory,verbose=True)
-    return agent_exe
-
-async def run_agent(agent,user_query):
-    #print(agent.memory.chat_memory.messages[-2:] if len(agent.memory.chat_memory.messages) > 1 else "")
-    #set_verbose(True)
-    print(agent.memory.chat_memory)
-    print('********************')
-    print()
+from langchain.agents import AgentExecutor, create_openai_tools_agent,Tool
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain_community.vectorstores import Chroma
+from langchain import hub
+from langchain.memory import ConversationBufferMemory
+from langchain.prompts import MessagesPlaceholder
+from engine.tools import GPT35TCodeGen, GPT4TAssistant, GPT4TCodeGen, DalleImageGen,RAGTool, CombinedTool, CareerRoadmapGenerator
+
+def create_agent(model_name):
+
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
+    #rag_llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0)
+    rag_llm = ChatOpenAI(model_name="gpt-4o", temperature=0.1)
+    #rag_llm = ChatOpenAI(model_name="gpt-4-turbo-preview", temperature=0)
+    rag_prompt = hub.pull("rlm/rag-prompt")
+    rag_db = Chroma(persist_directory="../../chroma_db", 
+                       embedding_function=OpenAIEmbeddings())
+    rag_retriever = rag_db.as_retriever()
+    
+    #tools = [GPT35TCodeGen(),GPT4TAssistant(),GPT4TCodeGen(), DalleImageGen(), RAGTool(rag_retriever,rag_llm,rag_prompt), CombinedTool(rag_retriever,rag_llm,rag_prompt), CareerRoadmapGenerator(rag_retriever,rag_llm,rag_prompt)]
+    tools = [RAGTool(rag_retriever,rag_llm,rag_prompt)]
+
+    llm = ChatOpenAI(model=model_name, temperature=0)
+
+    system_message = "You are a Career Roadmap Generator.\n" + \
+    "Answer questions with the help of given job description and create breif step by step solutions for every job description user provides to get that role in that company.\n" + \
+    "Put step by step process to get the job for the specific job description. List as many most relevant skills as possble for that role at that company.\n" + \
+    "If possible provide few projects to work on before applying for that role which will increace the chance of getting selected.\n" + \
+    "Add the resources to learn, watch, practice if possible for each step. Don't give me generic roadmap. Provide in-depth roadmap.\n" + \
+    "Link all the realatd skills and give what skill to learn first followed by another in the roadmap."
+
+
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", system_message),
+            MessagesPlaceholder("chat_history", optional=True),
+            ("human", "{input}"),
+            MessagesPlaceholder("agent_scratchpad"),
+        ]
+    )
+    agent = create_openai_tools_agent(llm, tools, prompt)
+    agent_exe = AgentExecutor(agent=agent, tools=tools,memory=memory,verbose=True)
+    return agent_exe
+
+async def run_agent(agent,user_query):
+    #print(agent.memory.chat_memory.messages[-2:] if len(agent.memory.chat_memory.messages) > 1 else "")
+    #set_verbose(True)
+    print(agent.memory.chat_memory)
+    print('********************')
+    print()
     return await agent.ainvoke(input={"input":user_query},verbose=True)

bot/langchain-bot/engine/tools.py

@@ -178,8 +178,6 @@ class RAGTool(BaseTool):
     "[question]\n" + \
     "Don't change the input question from the user and don't change answer from this tool\n."  +\
     "Just pass it through to the user." 
-    
-
     retriever: Type[VectorStoreRetriever] = None
     llm: Type[ChatOpenAI] = None
     prompt: Type[ChatPromptTemplate] = None

bot/openai-bot/bot-openai-chat.py

@@ -1,68 +1,68 @@
-import streamlit as st
-from dotenv import load_dotenv
-from openai_engine import OpenAIEngine
-
-load_dotenv()
-engine = OpenAIEngine()
-
-st.title("OpenAI API")
-
-# Define options for the dropdown lists
-chat_model_list = ["gpt-3.5-turbo","gpt-4-turbo-preview","gpt-4-vision-preview"]
-image_model_list = ["dall-e-3","dall-e-2"]
-chat_prompt_dictionary = {
-"Code Assistant":
-"""You are a code assistant. 
-Answer questions in code with minimal to no explanation.
-Put brief one line comments on the code for explanation\
-    """,
-"General Assistant": 
-"""You are a general AI assistant. 
-Answer questions with minimal and to the point explanation.
-Don't put safety and cultural warnings. Only warn about security."""
-    }
-
-# Create the first dropdown in the sidebar and update session state: generation type
-st.session_state["app_type_option"] = st.sidebar.selectbox("Generation Type:",["Chatting","Image Generation"])
-st.sidebar.write(f'You are in {st.session_state.app_type_option} mode.')
-
-# list of models is changed  based on the type of generation
-model_list  = chat_model_list if st.session_state.app_type_option == "Chatting" else image_model_list
-# second dropdown: list of models dropdown
-st.session_state["selected_option_1"] = st.sidebar.selectbox('Models:', model_list )
-
-# third dropdown in the sidebar and update session state: assistant type
-if st.session_state.app_type_option == "Chatting":
-    st.session_state.selected_option_2 = st.sidebar.selectbox('Prompts:', chat_prompt_dictionary.keys()) 
-    # Display the selected options
-    st.sidebar.write(f'You are using "{st.session_state.selected_option_1}\
-            " together with "{st.session_state.selected_option_2}" prompt.')
-else:
-    st.sidebar.write(f'You are using "{st.session_state.selected_option_1}".')
-
-# defining openai engine
-engine.change(st.session_state.app_type_option,
-              st.session_state.selected_option_1,
-              chat_prompt_dictionary[st.session_state.selected_option_2])
-
-# updading the chat page with messages
-for message in st.session_state["memory"]:
-    if message["role"] == "image assistant":
-        with st.chat_message("assistant"):
-            st.image(message["content"])
-    elif message["role"] == "system":
-        pass
-    else:
-        with st.chat_message(message["role"]):
-            st.markdown(message["content"])
-
-# entering new message event handle
-if prompt := st.chat_input("Start chat ..."):
-    with st.chat_message("user"):
-        st.markdown(prompt)
-
-    with st.chat_message("assistant"):
-        if st.session_state.app_type_option == "Chatting":
-            engine.generate_answer(prompt)
-        else:
+import streamlit as st
+from dotenv import load_dotenv
+from openai_engine import OpenAIEngine
+
+load_dotenv()
+engine = OpenAIEngine()
+
+st.title("OpenAI API")
+
+# Define options for the dropdown lists
+chat_model_list = ["gpt-3.5-turbo","gpt-4-turbo-preview","gpt-4-vision-preview"]
+image_model_list = ["dall-e-3","dall-e-2"]
+chat_prompt_dictionary = {
+"Code Assistant":
+"""You are a code assistant. 
+Answer questions in code with minimal to no explanation.
+Put brief one line comments on the code for explanation\
+    """,
+"General Assistant": 
+"""You are a general AI assistant. 
+Answer questions with minimal and to the point explanation.
+Don't put safety and cultural warnings. Only warn about security."""
+    }
+
+# Create the first dropdown in the sidebar and update session state: generation type
+st.session_state["app_type_option"] = st.sidebar.selectbox("Generation Type:",["Chatting","Image Generation"])
+st.sidebar.write(f'You are in {st.session_state.app_type_option} mode.')
+
+# list of models is changed  based on the type of generation
+model_list  = chat_model_list if st.session_state.app_type_option == "Chatting" else image_model_list
+# second dropdown: list of models dropdown
+st.session_state["selected_option_1"] = st.sidebar.selectbox('Models:', model_list )
+
+# third dropdown in the sidebar and update session state: assistant type
+if st.session_state.app_type_option == "Chatting":
+    st.session_state.selected_option_2 = st.sidebar.selectbox('Prompts:', chat_prompt_dictionary.keys()) 
+    # Display the selected options
+    st.sidebar.write(f'You are using "{st.session_state.selected_option_1}\
+            " together with "{st.session_state.selected_option_2}" prompt.')
+else:
+    st.sidebar.write(f'You are using "{st.session_state.selected_option_1}".')
+
+# defining openai engine
+engine.change(st.session_state.app_type_option,
+              st.session_state.selected_option_1,
+              chat_prompt_dictionary[st.session_state.selected_option_2])
+
+# updading the chat page with messages
+for message in st.session_state["memory"]:
+    if message["role"] == "image assistant":
+        with st.chat_message("assistant"):
+            st.image(message["content"])
+    elif message["role"] == "system":
+        pass
+    else:
+        with st.chat_message(message["role"]):
+            st.markdown(message["content"])
+
+# entering new message event handle
+if prompt := st.chat_input("Start chat ..."):
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    with st.chat_message("assistant"):
+        if st.session_state.app_type_option == "Chatting":
+            engine.generate_answer(prompt)
+        else:
             engine.generate_image(prompt)

bot/openai-bot/openai_engine.py

@@ -1,61 +1,61 @@
-from openai import OpenAI
-import streamlit as st
-import requests
-import os
-
-class OpenAIEngine():
-
-    def __init__(self,mode="",model="",prompt=""):
-        self.client = OpenAI()
-        if "memory" not in st.session_state:
-            st.session_state["memory"] = [{}]
-        if "image_number" not in st.session_state:
-            st.session_state['image_number'] = 1
-        if "image_folder" not in st.session_state:
-            st.session_state["image_folder"] = os.path.join("..","images")
-        self.change(mode,model,prompt)
-
-    def change(self,mode,model,prompt):
-        self.mode = mode
-        self.model = model
-        st.session_state["memory"][0] = {"role":"system","content":prompt}
-
-    def generate_answer(self,prompt):
-        st.session_state["memory"].append({"role":"user","content":prompt})
-        memory = []
-        for mem in st.session_state['memory']:
-            if mem['role'] != "image assistant":
-                memory.append(mem)
-            else:
-                memory = memory[:-1]
-        stream = self.client.chat.completions.create(
-            model=self.model,
-            messages=memory,
-            stream=True,
-            temperature=0,
-        )
-        response = st.write_stream(stream)
-        response = {"role": "assistant", "content": response}
-        st.session_state["memory"].append(response)
-        print(st.session_state["memory"])
-        print()
-
-    def generate_image(self,prompt):
-        image_data = self.client.images.generate(
-                model=self.model,
-                prompt=prompt,
-                size="1024x1024",
-                quality="standard",
-                n=1,
-        )
-        image = requests.get(image_data.data[0].url,stream=True)
-        if image.status_code == 200:
-            image_path = os.path.join(st.session_state["image_folder"],
-                                      f"{st.session_state['image_number']}.png")
-            st.session_state["image_number"] += 1
-            with open(image_path, 'wb') as f:
-                for chunk in image:
-                    f.write(chunk)
-            st.session_state["memory"].append({"role":"user","content":prompt})
-            st.session_state["memory"].append({"role": "image assistant", "content": image_path})
+from openai import OpenAI
+import streamlit as st
+import requests
+import os
+
+class OpenAIEngine():
+
+    def __init__(self,mode="",model="",prompt=""):
+        self.client = OpenAI()
+        if "memory" not in st.session_state:
+            st.session_state["memory"] = [{}]
+        if "image_number" not in st.session_state:
+            st.session_state['image_number'] = 1
+        if "image_folder" not in st.session_state:
+            st.session_state["image_folder"] = os.path.join("..","images")
+        self.change(mode,model,prompt)
+
+    def change(self,mode,model,prompt):
+        self.mode = mode
+        self.model = model
+        st.session_state["memory"][0] = {"role":"system","content":prompt}
+
+    def generate_answer(self,prompt):
+        st.session_state["memory"].append({"role":"user","content":prompt})
+        memory = []
+        for mem in st.session_state['memory']:
+            if mem['role'] != "image assistant":
+                memory.append(mem)
+            else:
+                memory = memory[:-1]
+        stream = self.client.chat.completions.create(
+            model=self.model,
+            messages=memory,
+            stream=True,
+            temperature=0,
+        )
+        response = st.write_stream(stream)
+        response = {"role": "assistant", "content": response}
+        st.session_state["memory"].append(response)
+        print(st.session_state["memory"])
+        print()
+
+    def generate_image(self,prompt):
+        image_data = self.client.images.generate(
+                model=self.model,
+                prompt=prompt,
+                size="1024x1024",
+                quality="standard",
+                n=1,
+        )
+        image = requests.get(image_data.data[0].url,stream=True)
+        if image.status_code == 200:
+            image_path = os.path.join(st.session_state["image_folder"],
+                                      f"{st.session_state['image_number']}.png")
+            st.session_state["image_number"] += 1
+            with open(image_path, 'wb') as f:
+                for chunk in image:
+                    f.write(chunk)
+            st.session_state["memory"].append({"role":"user","content":prompt})
+            st.session_state["memory"].append({"role": "image assistant", "content": image_path})
             st.image(image_path)

bot/rag_indexing/indexing.py

@@ -1,169 +1,169 @@
-import bs4
-import pandas as pd
-from langchain import hub
-from langchain_community.document_loaders import WebBaseLoader
-from langchain_community.vectorstores import Chroma
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.runnables import RunnablePassthrough
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-#from langchain.document_loaders import PyPDFLoader, CSVLoader, ExcelLoader
-from langchain_community.document_loaders import PyPDFLoader, CSVLoader, TextLoader
-from langchain_community.document_loaders import UnstructuredExcelLoader
-#from langchain.text_splitter import CharacterTextSplitter
-from langchain_community.embeddings import HuggingFaceEmbeddings
-from dotenv import load_dotenv
-import sys
-import shutil
-import os
-import uuid
-import csv
-
-def from_web(url):
-    loader = WebBaseLoader(web_paths=(url,),
-                           bs_kwargs=dict(parse_only=bs4.SoupStrainer(
-                               class_=("post-content", "post-title", "post-header")
-                           )),)
-    docs = loader.load()
-    return docs
-
-def from_excel(file_address):
-    if file_address.endswith(".xlsx"):
-        loader = UnstructuredExcelLoader(file_path=file_address)
-        docs = loader.load()
-        return docs
-    else:
-        docs = []
-        for file_name in os.listdir(file_address):
-            file_path = os.path.join(file_address, file_name)
-            if os.path.isfile(file_path) and file_name.endswith(".xlsx"):
-                # Load the Excel file
-                loader = UnstructuredExcelLoader(file_path=file_address)
-                docs.extend(loader.load())
-    return docs
-
-def from_csv(file_address):
-    docs = []
-    #Load the CSV file
-    if file_address.endswith(".csv"):
-        loader = CSVLoader(file_path=file_address, encoding='utf-8')
-        docs = loader.load()
-    return docs
-
-def from_pdf(file_address):
-    loader = PyPDFLoader(file_path=file_address)
-    docs = loader.load()
-    return docs
-
-def from_text_files(file_address):
-    docs = []
-    for file_name in os.listdir(file_address):
-        file_path = os.path.join(file_address, file_name)
-        if os.path.isfile(file_path) and file_name.endswith(".txt"):
-            loader = TextLoader(file_path)
-            docs.extend(loader.load())
-    return docs
-
-def retriever_from_docs(docs):
-    if not docs:
-        print("No documents to process.")
-        return
-    #print("Documents:", docs)
-
-    # Split the documents into smaller chunks
-    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
-    splits = text_splitter.split_documents(docs)
-    print(f"Number of document chunks: {len(splits)}")
-
-    # Create embeddings for the document chunks
-    #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")  # 384 dimensionality embeddings
-    embeddings = OpenAIEmbeddings()                                                          # 1536 dimensionality
-    #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") # 768 embedded dimension
-    #embeddings = HuggingFaceEmbeddings(model_name="bert-large-uncased")                      # 1024 dim
-
-    embeddings_list = embeddings.embed_documents([t.page_content for t in splits])
-    
-    # Generate unique IDs for each document chunk
-    doc_ids = [str(uuid.uuid4()) for _ in range(len(splits))]
-    print(f"Number of IDs generated: {len(doc_ids)}")
-
-    # Create or load the Chroma vector store
-    persist_directory="../../chroma_db"
-
-    # Check if the directory exists
-    if os.path.exists(persist_directory):
-        # Remove the directory and its contents
-        #shutil.rmtree(persist_directory)
-        #print(f"Deleted {persist_directory}")
-
-        # Load the existing vector store
-        #chroma_store = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
-
-        #print()
-
-        # Create a new vector store
-        chroma_store = Chroma.from_documents(documents=splits, embedding=embeddings, 
-                                             persist_directory=persist_directory)
-        
-        # Load the existing vector store
-        chroma_store = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
-    
-        chroma_store.add_texts([t.page_content for t in splits], embeddings=embeddings_list, ids=doc_ids)
-
-
-    else:
-        print(f"{persist_directory} does not exist")
-        # Create a new vector store
-        chroma_store = Chroma.from_documents(documents=splits, embedding=embeddings, 
-                                             persist_directory=persist_directory)
-    
-    #Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings(),persist_directory="../../chroma_db")
-    #chroma_store = Chroma.from_documents(documents=splits, embedding=embeddings, persist_directory="../../chroma_db")
-    
-    # Is used to add new documents and their corresponding embeddings to an existing Chroma vector store.
-    #chroma_store.add_texts([t.page_content for t in splits], embeddings=embeddings_list, ids=doc_ids)
-    
-    print("Embeddings are added to vector store.")
-
-
-def main():
-    print(sys.argv)
-    load_dotenv()
-
-    #file_address = "../../../db_28_2_text/db_28_2_text/"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/"
-    file_address = "../../../International Job Dataset/allJobs.xlsx"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Technology Skills.xlsx" 
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Tools Used.xlsx"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Alternate Titles.xlsx"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Emerging Tasks.xlsx" Job Zone Reference
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Job Zone Reference.xlsx"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Job Zones.xlsx"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Occupation Data.xlsx"
-    #file_address = "../../../db_28_2_excel/db_28_2_excel/Related Occupations.xlsx"
-
-    # Check if the file_address exists
-    if not os.path.exists(file_address):
-        print("File address does not exist.")
-        return
-
-    # Determine the input type and load the documents accordingly
-    if 'http' in sys.argv[1].lower():
-        retriever_from_docs(from_web(sys.argv[1]))
-    elif '.xls' in sys.argv[1].lower():
-        retriever_from_docs(from_excel(sys.argv[1]))
-    elif '.csv' in sys.argv[1].lower():
-        retriever_from_docs(from_csv(sys.argv[1]))
-    elif '.pdf' in sys.argv[1].lower():
-        retriever_from_docs(from_pdf(sys.argv[1]))
-    elif '.txt' in sys.argv[1].lower():
-        retriever_from_docs(from_text_files(sys.argv[1]))
-    elif 'excel' in sys.argv[1].lower():
-        retriever_from_docs(from_excel(sys.argv[1]))
-    elif 'text' in sys.argv[1].lower():
-        retriever_from_docs(from_text_files(sys.argv[1]))
-    else:
-        print(f"Unsupported file format for file.")
-    
-if __name__ == "__main__":
+import bs4
+import pandas as pd
+from langchain import hub
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.vectorstores import Chroma
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+#from langchain.document_loaders import PyPDFLoader, CSVLoader, ExcelLoader
+from langchain_community.document_loaders import PyPDFLoader, CSVLoader, TextLoader
+from langchain_community.document_loaders import UnstructuredExcelLoader
+#from langchain.text_splitter import CharacterTextSplitter
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from dotenv import load_dotenv
+import sys
+import shutil
+import os
+import uuid
+import csv
+
+def from_web(url):
+    loader = WebBaseLoader(web_paths=(url,),
+                           bs_kwargs=dict(parse_only=bs4.SoupStrainer(
+                               class_=("post-content", "post-title", "post-header")
+                           )),)
+    docs = loader.load()
+    return docs
+
+def from_excel(file_address):
+    if file_address.endswith(".xlsx"):
+        loader = UnstructuredExcelLoader(file_path=file_address)
+        docs = loader.load()
+        return docs
+    else:
+        docs = []
+        for file_name in os.listdir(file_address):
+            file_path = os.path.join(file_address, file_name)
+            if os.path.isfile(file_path) and file_name.endswith(".xlsx"):
+                # Load the Excel file
+                loader = UnstructuredExcelLoader(file_path=file_address)
+                docs.extend(loader.load())
+    return docs
+
+def from_csv(file_address):
+    docs = []
+    #Load the CSV file
+    if file_address.endswith(".csv"):
+        loader = CSVLoader(file_path=file_address, encoding='utf-8')
+        docs = loader.load()
+    return docs
+
+def from_pdf(file_address):
+    loader = PyPDFLoader(file_path=file_address)
+    docs = loader.load()
+    return docs
+
+def from_text_files(file_address):
+    docs = []
+    for file_name in os.listdir(file_address):
+        file_path = os.path.join(file_address, file_name)
+        if os.path.isfile(file_path) and file_name.endswith(".txt"):
+            loader = TextLoader(file_path)
+            docs.extend(loader.load())
+    return docs
+
+def retriever_from_docs(docs):
+    if not docs:
+        print("No documents to process.")
+        return
+    #print("Documents:", docs)
+
+    # Split the documents into smaller chunks
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    splits = text_splitter.split_documents(docs)
+    print(f"Number of document chunks: {len(splits)}")
+
+    # Create embeddings for the document chunks
+    #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")  # 384 dimensionality embeddings
+    embeddings = OpenAIEmbeddings()                                                          # 1536 dimensionality
+    #embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") # 768 embedded dimension
+    #embeddings = HuggingFaceEmbeddings(model_name="bert-large-uncased")                      # 1024 dim
+
+    embeddings_list = embeddings.embed_documents([t.page_content for t in splits])
+    
+    # Generate unique IDs for each document chunk
+    doc_ids = [str(uuid.uuid4()) for _ in range(len(splits))]
+    print(f"Number of IDs generated: {len(doc_ids)}")
+
+    # Create or load the Chroma vector store
+    persist_directory="../../chroma_db"
+
+    # Check if the directory exists
+    if os.path.exists(persist_directory):
+        # Remove the directory and its contents
+        #shutil.rmtree(persist_directory)
+        #print(f"Deleted {persist_directory}")
+
+        # Load the existing vector store
+        #chroma_store = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+
+        #print()
+
+        # Create a new vector store
+        chroma_store = Chroma.from_documents(documents=splits, embedding=embeddings, 
+                                             persist_directory=persist_directory)
+        
+        # Load the existing vector store
+        chroma_store = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+    
+        chroma_store.add_texts([t.page_content for t in splits], embeddings=embeddings_list, ids=doc_ids)
+
+
+    else:
+        print(f"{persist_directory} does not exist")
+        # Create a new vector store
+        chroma_store = Chroma.from_documents(documents=splits, embedding=embeddings, 
+                                             persist_directory=persist_directory)
+    
+    #Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings(),persist_directory="../../chroma_db")
+    #chroma_store = Chroma.from_documents(documents=splits, embedding=embeddings, persist_directory="../../chroma_db")
+    
+    # Is used to add new documents and their corresponding embeddings to an existing Chroma vector store.
+    #chroma_store.add_texts([t.page_content for t in splits], embeddings=embeddings_list, ids=doc_ids)
+    
+    print("Embeddings are added to vector store.")
+
+
+def main():
+    print(sys.argv)
+    load_dotenv()
+
+    #file_address = "../../../db_28_2_text/db_28_2_text/"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/"
+    file_address = "../../../International Job Dataset/allJobs.xlsx"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Technology Skills.xlsx" 
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Tools Used.xlsx"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Alternate Titles.xlsx"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Emerging Tasks.xlsx" Job Zone Reference
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Job Zone Reference.xlsx"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Job Zones.xlsx"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Occupation Data.xlsx"
+    #file_address = "../../../db_28_2_excel/db_28_2_excel/Related Occupations.xlsx"
+
+    # Check if the file_address exists
+    if not os.path.exists(file_address):
+        print("File address does not exist.")
+        return
+
+    # Determine the input type and load the documents accordingly
+    if 'http' in sys.argv[1].lower():
+        retriever_from_docs(from_web(sys.argv[1]))
+    elif '.xls' in sys.argv[1].lower():
+        retriever_from_docs(from_excel(sys.argv[1]))
+    elif '.csv' in sys.argv[1].lower():
+        retriever_from_docs(from_csv(sys.argv[1]))
+    elif '.pdf' in sys.argv[1].lower():
+        retriever_from_docs(from_pdf(sys.argv[1]))
+    elif '.txt' in sys.argv[1].lower():
+        retriever_from_docs(from_text_files(sys.argv[1]))
+    elif 'excel' in sys.argv[1].lower():
+        retriever_from_docs(from_excel(sys.argv[1]))
+    elif 'text' in sys.argv[1].lower():
+        retriever_from_docs(from_text_files(sys.argv[1]))
+    else:
+        print(f"Unsupported file format for file.")
+    
+if __name__ == "__main__":
     main()

chroma_db/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01014d173911620f29e57c04fc1ff8e29816f5044fe8d0045acfb9312e9965ed
-size 1619869696
+oid sha256:a18d7993408eda3ff5ce877beff0766b906a263a36108fcb788e7444827bb96e
+size 5914836992

langchain.md

@@ -1,62 +1,62 @@
-# Langchain 
-**Source: https://python.langchain.com/docs**  
-LangChain is a framework for developing applications powered by language models. It enables applications that:  
-**Are context-aware** i.e. connect a language model to sources of context (prompt instructions, few shot examples, content to ground its response in, etc.)  
-**Reason** i.e. rely on a language model to reason (about how to answer based on provided context, what actions to take, etc.)
-
-**Other Packages:**  
-* **LangServe:** A library for deploying LangChain chains as a REST API.  
-* **LangSmith:** A developer platform that lets you debug, test, evaluate, and monitor chains built on any LLM framework and seamlessly integrates with LangChain.
-
-**LangChain Libraries:**  
-The LangChain libraries themselves are made up of several different packages.  
-    - **langchain-core:** Base abstractions and LangChain Expression Language.  
-    - **langchain-community:** Third party integrations.  
-    - **langchain:** Chains, agents, and retrieval strategies that make up an application's cognitive architecture.
-
-**LangChain Components:**
-1. LangChain Expression Language
-2. Modules: 
-    * Model I/O: Interface with language models
-    * Retrieval: Interface with application-specific data
-    * Agents: Let chains choose which tools to use given high-level directives
-
-    Additional:
-    * Chains: Common, building block compositions
-    * Memory: Persist application state between runs of a chain
-    * Callbacks: Log and stream intermediate steps of any chain
-
-## LangChain Expression Language (LCEL)
-LangChain Expression Language, or LCEL, is a declarative way to easily compose chains together. LCEL was designed from day 1 to support putting prototypes in production, with no code changes, from the simplest “prompt + LLM” chain to the most complex chains (we’ve seen folks successfully run LCEL chains with 100s of steps in production). To highlight a few of the reasons you might want to use LCEL:  
-* Streaming support
-* Async support
-* Optimized parallel execution
-* Retries and fallbacks
-* Access intermediate results
-* Input and output schemas
-* Seamless LangSmith tracing integration
-* Seamless LangServe deployment integration
-
-## Modules:
-* **Model I/O:** The core element of any language model application is the model. LangChain gives you the building blocks to interface with any language model.
-    * **Prompts:** different types of prompt templates.
-    * **LLMs:** functionality related to the LLM class. This is a type of model that takes a text string as input and returns a text string.
-    * **ChatModels:** functionality related to the ChatModel class. This is a type of model that takes a list of messages as input and returns a message.
-    * **Output Parsers:** responsible for transforming the output of LLMs and ChatModels into more structured data.
-* **Retrieval:** Many LLM applications require user-specific data that is not part of the model's training set. The primary way of accomplishing this is through Retrieval Augmented Generation (RAG). In this process, external data is retrieved and then passed to the LLM when doing the generation step.
-    * **Document loaders:** load documents from many different sources. LangChain provides over 100 different document loaders as well as integrations with other major providers. LangChain provides integrations to load all types of documents (HTML, PDF, code).
-    * **Text Splitting:** splitting (or chunking) a large document into smaller chunks. LangChain provides several transformation algorithms for doing this.
-    * **Text embedding models:** integrations with over 25 different embedding providers and methods, from open-source to proprietary API. LangChain provides a standard interface, allowing you to easily swap between models.
-    * **Vector stores:** integrations with over 50 different vector stores, from open-source local ones to cloud-hosted proprietary ones. LangChain exposes a standard interface, allowing you to easily swap between vector stores.
-    * **Retrievers:** LangChain supports basic methods that are easy to get started - namely simple semantic search. However, we have also added a collection of algorithms on top of this to increase performance.
-    * **Indexing:** The LangChain Indexing API syncs your data from any source into a vector store.
-* **Agents:** The core idea of agents is to use a language model to choose a sequence of actions to take. In chains, a sequence of actions is hardcoded (in code). In agents, a language model is used as a reasoning engine to determine which actions to take and in which order. (Action Generation)
-    * **Agent Types:** There are many different types of agents to use. 
-    * **Tools:** Agents are only as good as the tools they have.
-* **Chains:** refer to sequences of calls - whether to an LLM, a tool, or a data preprocessing step. The primary supported way to do this is with LCEL.
-
-    LCEL is great for constructing your own chains, but it’s also nice to have chains that you can use off-the-shelf. There are two types of off-the-shelf chains that LangChain supports:
-
-    * Chains that are built with LCEL. In this case, LangChain offers a higher-level constructor method. However, all that is being done under the hood is constructing a chain with LCEL.
-
+# Langchain 
+**Source: https://python.langchain.com/docs**  
+LangChain is a framework for developing applications powered by language models. It enables applications that:  
+**Are context-aware** i.e. connect a language model to sources of context (prompt instructions, few shot examples, content to ground its response in, etc.)  
+**Reason** i.e. rely on a language model to reason (about how to answer based on provided context, what actions to take, etc.)
+
+**Other Packages:**  
+* **LangServe:** A library for deploying LangChain chains as a REST API.  
+* **LangSmith:** A developer platform that lets you debug, test, evaluate, and monitor chains built on any LLM framework and seamlessly integrates with LangChain.
+
+**LangChain Libraries:**  
+The LangChain libraries themselves are made up of several different packages.  
+    - **langchain-core:** Base abstractions and LangChain Expression Language.  
+    - **langchain-community:** Third party integrations.  
+    - **langchain:** Chains, agents, and retrieval strategies that make up an application's cognitive architecture.
+
+**LangChain Components:**
+1. LangChain Expression Language
+2. Modules: 
+    * Model I/O: Interface with language models
+    * Retrieval: Interface with application-specific data
+    * Agents: Let chains choose which tools to use given high-level directives
+
+    Additional:
+    * Chains: Common, building block compositions
+    * Memory: Persist application state between runs of a chain
+    * Callbacks: Log and stream intermediate steps of any chain
+
+## LangChain Expression Language (LCEL)
+LangChain Expression Language, or LCEL, is a declarative way to easily compose chains together. LCEL was designed from day 1 to support putting prototypes in production, with no code changes, from the simplest “prompt + LLM” chain to the most complex chains (we’ve seen folks successfully run LCEL chains with 100s of steps in production). To highlight a few of the reasons you might want to use LCEL:  
+* Streaming support
+* Async support
+* Optimized parallel execution
+* Retries and fallbacks
+* Access intermediate results
+* Input and output schemas
+* Seamless LangSmith tracing integration
+* Seamless LangServe deployment integration
+
+## Modules:
+* **Model I/O:** The core element of any language model application is the model. LangChain gives you the building blocks to interface with any language model.
+    * **Prompts:** different types of prompt templates.
+    * **LLMs:** functionality related to the LLM class. This is a type of model that takes a text string as input and returns a text string.
+    * **ChatModels:** functionality related to the ChatModel class. This is a type of model that takes a list of messages as input and returns a message.
+    * **Output Parsers:** responsible for transforming the output of LLMs and ChatModels into more structured data.
+* **Retrieval:** Many LLM applications require user-specific data that is not part of the model's training set. The primary way of accomplishing this is through Retrieval Augmented Generation (RAG). In this process, external data is retrieved and then passed to the LLM when doing the generation step.
+    * **Document loaders:** load documents from many different sources. LangChain provides over 100 different document loaders as well as integrations with other major providers. LangChain provides integrations to load all types of documents (HTML, PDF, code).
+    * **Text Splitting:** splitting (or chunking) a large document into smaller chunks. LangChain provides several transformation algorithms for doing this.
+    * **Text embedding models:** integrations with over 25 different embedding providers and methods, from open-source to proprietary API. LangChain provides a standard interface, allowing you to easily swap between models.
+    * **Vector stores:** integrations with over 50 different vector stores, from open-source local ones to cloud-hosted proprietary ones. LangChain exposes a standard interface, allowing you to easily swap between vector stores.
+    * **Retrievers:** LangChain supports basic methods that are easy to get started - namely simple semantic search. However, we have also added a collection of algorithms on top of this to increase performance.
+    * **Indexing:** The LangChain Indexing API syncs your data from any source into a vector store.
+* **Agents:** The core idea of agents is to use a language model to choose a sequence of actions to take. In chains, a sequence of actions is hardcoded (in code). In agents, a language model is used as a reasoning engine to determine which actions to take and in which order. (Action Generation)
+    * **Agent Types:** There are many different types of agents to use. 
+    * **Tools:** Agents are only as good as the tools they have.
+* **Chains:** refer to sequences of calls - whether to an LLM, a tool, or a data preprocessing step. The primary supported way to do this is with LCEL.
+
+    LCEL is great for constructing your own chains, but it’s also nice to have chains that you can use off-the-shelf. There are two types of off-the-shelf chains that LangChain supports:
+
+    * Chains that are built with LCEL. In this case, LangChain offers a higher-level constructor method. However, all that is being done under the hood is constructing a chain with LCEL.
+
     * [Legacy] Chains constructed by subclassing from a legacy Chain class. These chains do not use LCEL under the hood but are rather standalone classes.

notebooks/app.ipynb

@@ -1,116 +1,116 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "'''\n",
-    "requirements.txt file contents:\n",
-    " \n",
-    "langchain==0.0.154\n",
-    "PyPDF2==3.0.1\n",
-    "python-dotenv==1.0.0\n",
-    "streamlit==1.18.1\n",
-    "faiss-cpu==1.7.4\n",
-    "streamlit-extras\n",
-    "'''\n",
-    " \n",
-    " \n",
-    "import streamlit as st\n",
-    "from dotenv import load_dotenv\n",
-    "import pickle\n",
-    "from PyPDF2 import PdfReader\n",
-    "from streamlit_extras.add_vertical_space import add_vertical_space\n",
-    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
-    "from langchain.vectorstores import FAISS\n",
-    "from langchain.llms import OpenAI\n",
-    "from langchain.chains.question_answering import load_qa_chain\n",
-    "from langchain.callbacks import get_openai_callback\n",
-    "import os\n",
-    " \n",
-    "# Sidebar contents\n",
-    "with st.sidebar:\n",
-    "    st.title('🤗💬 LLM Chat App')\n",
-    "    st.markdown('''\n",
-    "    ## About\n",
-    "    This app is an LLM-powered chatbot built using:\n",
-    "    - [Streamlit](https://streamlit.io/)\n",
-    "    - [LangChain](https://python.langchain.com/)\n",
-    "    - [OpenAI](https://platform.openai.com/docs/models) LLM model\n",
-    " \n",
-    "    ''')\n",
-    "    add_vertical_space(5)\n",
-    "    st.write('Made with ❤️ by [Prompt Engineer](https://youtube.com/@engineerprompt)')\n",
-    " \n",
-    "load_dotenv()\n",
-    " \n",
-    "def main():\n",
-    "    st.header(\"Chat with PDF 💬\")\n",
-    " \n",
-    " \n",
-    "    # upload a PDF file\n",
-    "    pdf = st.file_uploader(\"Upload your PDF\", type='pdf')\n",
-    " \n",
-    "    # st.write(pdf)\n",
-    "    if pdf is not None:\n",
-    "        pdf_reader = PdfReader(pdf)\n",
-    "        \n",
-    "        text = \"\"\n",
-    "        for page in pdf_reader.pages:\n",
-    "            text += page.extract_text()\n",
-    " \n",
-    "        text_splitter = RecursiveCharacterTextSplitter(\n",
-    "            chunk_size=1000,\n",
-    "            chunk_overlap=200,\n",
-    "            length_function=len\n",
-    "            )\n",
-    "        chunks = text_splitter.split_text(text=text)\n",
-    " \n",
-    "        # # embeddings\n",
-    "        store_name = pdf.name[:-4]\n",
-    "        st.write(f'{store_name}')\n",
-    "        # st.write(chunks)\n",
-    " \n",
-    "        if os.path.exists(f\"{store_name}.pkl\"):\n",
-    "            with open(f\"{store_name}.pkl\", \"rb\") as f:\n",
-    "                VectorStore = pickle.load(f)\n",
-    "            # st.write('Embeddings Loaded from the Disk')s\n",
-    "        else:\n",
-    "            embeddings = OpenAIEmbeddings()\n",
-    "            VectorStore = FAISS.from_texts(chunks, embedding=embeddings)\n",
-    "            with open(f\"{store_name}.pkl\", \"wb\") as f:\n",
-    "                pickle.dump(VectorStore, f)\n",
-    " \n",
-    "        # embeddings = OpenAIEmbeddings()\n",
-    "        # VectorStore = FAISS.from_texts(chunks, embedding=embeddings)\n",
-    " \n",
-    "        # Accept user questions/query\n",
-    "        query = st.text_input(\"Ask questions about your PDF file:\")\n",
-    "        # st.write(query)\n",
-    " \n",
-    "        if query:\n",
-    "            docs = VectorStore.similarity_search(query=query, k=3)\n",
-    " \n",
-    "            llm = OpenAI()\n",
-    "            chain = load_qa_chain(llm=llm, chain_type=\"stuff\")\n",
-    "            with get_openai_callback() as cb:\n",
-    "                response = chain.run(input_documents=docs, question=query)\n",
-    "                print(cb)\n",
-    "            st.write(response)\n",
-    " \n",
-    "if __name__ == '__main__':\n",
-    "    main()"
-   ]
-  }
- ],
- "metadata": {
-  "language_info": {
-   "name": "python"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "'''\n",
+    "requirements.txt file contents:\n",
+    " \n",
+    "langchain==0.0.154\n",
+    "PyPDF2==3.0.1\n",
+    "python-dotenv==1.0.0\n",
+    "streamlit==1.18.1\n",
+    "faiss-cpu==1.7.4\n",
+    "streamlit-extras\n",
+    "'''\n",
+    " \n",
+    " \n",
+    "import streamlit as st\n",
+    "from dotenv import load_dotenv\n",
+    "import pickle\n",
+    "from PyPDF2 import PdfReader\n",
+    "from streamlit_extras.add_vertical_space import add_vertical_space\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
+    "from langchain.vectorstores import FAISS\n",
+    "from langchain.llms import OpenAI\n",
+    "from langchain.chains.question_answering import load_qa_chain\n",
+    "from langchain.callbacks import get_openai_callback\n",
+    "import os\n",
+    " \n",
+    "# Sidebar contents\n",
+    "with st.sidebar:\n",
+    "    st.title('🤗💬 LLM Chat App')\n",
+    "    st.markdown('''\n",
+    "    ## About\n",
+    "    This app is an LLM-powered chatbot built using:\n",
+    "    - [Streamlit](https://streamlit.io/)\n",
+    "    - [LangChain](https://python.langchain.com/)\n",
+    "    - [OpenAI](https://platform.openai.com/docs/models) LLM model\n",
+    " \n",
+    "    ''')\n",
+    "    add_vertical_space(5)\n",
+    "    st.write('Made with ❤️ by [Prompt Engineer](https://youtube.com/@engineerprompt)')\n",
+    " \n",
+    "load_dotenv()\n",
+    " \n",
+    "def main():\n",
+    "    st.header(\"Chat with PDF 💬\")\n",
+    " \n",
+    " \n",
+    "    # upload a PDF file\n",
+    "    pdf = st.file_uploader(\"Upload your PDF\", type='pdf')\n",
+    " \n",
+    "    # st.write(pdf)\n",
+    "    if pdf is not None:\n",
+    "        pdf_reader = PdfReader(pdf)\n",
+    "        \n",
+    "        text = \"\"\n",
+    "        for page in pdf_reader.pages:\n",
+    "            text += page.extract_text()\n",
+    " \n",
+    "        text_splitter = RecursiveCharacterTextSplitter(\n",
+    "            chunk_size=1000,\n",
+    "            chunk_overlap=200,\n",
+    "            length_function=len\n",
+    "            )\n",
+    "        chunks = text_splitter.split_text(text=text)\n",
+    " \n",
+    "        # # embeddings\n",
+    "        store_name = pdf.name[:-4]\n",
+    "        st.write(f'{store_name}')\n",
+    "        # st.write(chunks)\n",
+    " \n",
+    "        if os.path.exists(f\"{store_name}.pkl\"):\n",
+    "            with open(f\"{store_name}.pkl\", \"rb\") as f:\n",
+    "                VectorStore = pickle.load(f)\n",
+    "            # st.write('Embeddings Loaded from the Disk')s\n",
+    "        else:\n",
+    "            embeddings = OpenAIEmbeddings()\n",
+    "            VectorStore = FAISS.from_texts(chunks, embedding=embeddings)\n",
+    "            with open(f\"{store_name}.pkl\", \"wb\") as f:\n",
+    "                pickle.dump(VectorStore, f)\n",
+    " \n",
+    "        # embeddings = OpenAIEmbeddings()\n",
+    "        # VectorStore = FAISS.from_texts(chunks, embedding=embeddings)\n",
+    " \n",
+    "        # Accept user questions/query\n",
+    "        query = st.text_input(\"Ask questions about your PDF file:\")\n",
+    "        # st.write(query)\n",
+    " \n",
+    "        if query:\n",
+    "            docs = VectorStore.similarity_search(query=query, k=3)\n",
+    " \n",
+    "            llm = OpenAI()\n",
+    "            chain = load_qa_chain(llm=llm, chain_type=\"stuff\")\n",
+    "            with get_openai_callback() as cb:\n",
+    "                response = chain.run(input_documents=docs, question=query)\n",
+    "                print(cb)\n",
+    "            st.write(response)\n",
+    " \n",
+    "if __name__ == '__main__':\n",
+    "    main()"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/langchain_agent.ipynb

@@ -1,165 +1,165 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 19,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from typing import Optional, Type\n",
-    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
-    "from langchain_core.tools import BaseTool\n",
-    "\n",
-    "class MultiplierInput(BaseModel):\n",
-    "    a: int = Field(description=\"first number\")\n",
-    "    b: int = Field(description=\"second number\")\n",
-    "\n",
-    "class CustomMultiplierTool(BaseTool):\n",
-    "    name = \"Calculator\"\n",
-    "    description = \"useful for when you need to answer questions about math\"\n",
-    "    args_schema: Type[BaseModel] = MultiplierInput\n",
-    "\n",
-    "    def _run(\n",
-    "        self, a: int, b: int\n",
-    "    ) -> str:\n",
-    "        \"\"\"Use the tool.\"\"\"\n",
-    "        return a * b\n",
-    "\n",
-    "    async def _arun(\n",
-    "        self,\n",
-    "        a: int,\n",
-    "        b: int,\n",
-    "    ) -> str:\n",
-    "        \"\"\"Use the tool asynchronously.\"\"\"\n",
-    "        return self._run(a,b)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from langchain.agents import AgentExecutor, create_openai_tools_agent,Tool\n",
-    "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
-    "from langchain_openai import ChatOpenAI\n",
-    "from langchain.memory import ConversationBufferMemory\n",
-    "from langchain.prompts import MessagesPlaceholder\n",
-    "from langchain.globals import set_verbose\n",
-    "import dotenv\n",
-    "\n",
-    "dotenv.load_dotenv()\n",
-    "\n",
-    "def create_agent(model_name):\n",
-    "\n",
-    "    memory = ConversationBufferMemory(memory_key=\"chat_history\", return_messages=True)\n",
-    "    \n",
-    "    tools = [CustomMultiplierTool()]\n",
-    "    \n",
-    "    llm = ChatOpenAI(model=model_name, temperature=0)\n",
-    "\n",
-    "    system_message = \"\"\"You are a general AI assistant.\n",
-    "    You have access to several tool. Don't answer the question if you are not getting the answer from a tool.\n",
-    "    Don't put safety and cultural warnings. Only warn about security. \n",
-    "    \"\"\"\n",
-    "\n",
-    "    prompt = ChatPromptTemplate.from_messages(\n",
-    "        [\n",
-    "            (\"system\", system_message),\n",
-    "            MessagesPlaceholder(\"chat_history\", optional=True),\n",
-    "            (\"human\", \"{input}\"),\n",
-    "            MessagesPlaceholder(\"agent_scratchpad\"),\n",
-    "        ]\n",
-    "    )\n",
-    "    agent = create_openai_tools_agent(llm, tools, prompt)\n",
-    "    agent_exe = AgentExecutor(agent=agent, tools=tools,memory=memory)\n",
-    "    return agent_exe\n",
-    "\n",
-    "async def run_agent(agent,user_query):\n",
-    "    #print(agent.memory.chat_memory.messages[-2:] if len(agent.memory.chat_memory.messages) > 1 else \"\")\n",
-    "    #set_verbose(True)\n",
-    "    # print(agent.memory.chat_memory)\n",
-    "    print('********************')\n",
-    "    # print()\n",
-    "    return await agent.ainvoke(input={\"input\":user_query},verbose=True)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 29,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "agent_exe = create_agent(\"gpt-3.5-turbo\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 30,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "********************\n",
-      "\n",
-      "\n",
-      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
-      "\u001b[32;1m\u001b[1;3m\n",
-      "Invoking: `Calculator` with `{'a': 2, 'b': 3}`\n",
-      "\n",
-      "\n",
-      "\u001b[0m\u001b[36;1m\u001b[1;3m6\u001b[0m\u001b[32;1m\u001b[1;3m2 multiplied by 3 is equal to 6.\u001b[0m\n",
-      "\n",
-      "\u001b[1m> Finished chain.\u001b[0m\n"
-     ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "{'input': '2 multiply by 3',\n",
-       " 'chat_history': [HumanMessage(content='2 multiply by 3'),\n",
-       "  AIMessage(content='2 multiplied by 3 is equal to 6.')],\n",
-       " 'output': '2 multiplied by 3 is equal to 6.'}"
-      ]
-     },
-     "execution_count": 30,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "await run_agent(agent_exe,\"2 multiply by 3\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "stevens-chat-py311-v1",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import Optional, Type\n",
+    "from langchain_core.pydantic_v1 import BaseModel, Field\n",
+    "from langchain_core.tools import BaseTool\n",
+    "\n",
+    "class MultiplierInput(BaseModel):\n",
+    "    a: int = Field(description=\"first number\")\n",
+    "    b: int = Field(description=\"second number\")\n",
+    "\n",
+    "class CustomMultiplierTool(BaseTool):\n",
+    "    name = \"Calculator\"\n",
+    "    description = \"useful for when you need to answer questions about math\"\n",
+    "    args_schema: Type[BaseModel] = MultiplierInput\n",
+    "\n",
+    "    def _run(\n",
+    "        self, a: int, b: int\n",
+    "    ) -> str:\n",
+    "        \"\"\"Use the tool.\"\"\"\n",
+    "        return a * b\n",
+    "\n",
+    "    async def _arun(\n",
+    "        self,\n",
+    "        a: int,\n",
+    "        b: int,\n",
+    "    ) -> str:\n",
+    "        \"\"\"Use the tool asynchronously.\"\"\"\n",
+    "        return self._run(a,b)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.agents import AgentExecutor, create_openai_tools_agent,Tool\n",
+    "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
+    "from langchain_openai import ChatOpenAI\n",
+    "from langchain.memory import ConversationBufferMemory\n",
+    "from langchain.prompts import MessagesPlaceholder\n",
+    "from langchain.globals import set_verbose\n",
+    "import dotenv\n",
+    "\n",
+    "dotenv.load_dotenv()\n",
+    "\n",
+    "def create_agent(model_name):\n",
+    "\n",
+    "    memory = ConversationBufferMemory(memory_key=\"chat_history\", return_messages=True)\n",
+    "    \n",
+    "    tools = [CustomMultiplierTool()]\n",
+    "    \n",
+    "    llm = ChatOpenAI(model=model_name, temperature=0)\n",
+    "\n",
+    "    system_message = \"\"\"You are a general AI assistant.\n",
+    "    You have access to several tool. Don't answer the question if you are not getting the answer from a tool.\n",
+    "    Don't put safety and cultural warnings. Only warn about security. \n",
+    "    \"\"\"\n",
+    "\n",
+    "    prompt = ChatPromptTemplate.from_messages(\n",
+    "        [\n",
+    "            (\"system\", system_message),\n",
+    "            MessagesPlaceholder(\"chat_history\", optional=True),\n",
+    "            (\"human\", \"{input}\"),\n",
+    "            MessagesPlaceholder(\"agent_scratchpad\"),\n",
+    "        ]\n",
+    "    )\n",
+    "    agent = create_openai_tools_agent(llm, tools, prompt)\n",
+    "    agent_exe = AgentExecutor(agent=agent, tools=tools,memory=memory)\n",
+    "    return agent_exe\n",
+    "\n",
+    "async def run_agent(agent,user_query):\n",
+    "    #print(agent.memory.chat_memory.messages[-2:] if len(agent.memory.chat_memory.messages) > 1 else \"\")\n",
+    "    #set_verbose(True)\n",
+    "    # print(agent.memory.chat_memory)\n",
+    "    print('********************')\n",
+    "    # print()\n",
+    "    return await agent.ainvoke(input={\"input\":user_query},verbose=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "agent_exe = create_agent(\"gpt-3.5-turbo\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "********************\n",
+      "\n",
+      "\n",
+      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
+      "\u001b[32;1m\u001b[1;3m\n",
+      "Invoking: `Calculator` with `{'a': 2, 'b': 3}`\n",
+      "\n",
+      "\n",
+      "\u001b[0m\u001b[36;1m\u001b[1;3m6\u001b[0m\u001b[32;1m\u001b[1;3m2 multiplied by 3 is equal to 6.\u001b[0m\n",
+      "\n",
+      "\u001b[1m> Finished chain.\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'input': '2 multiply by 3',\n",
+       " 'chat_history': [HumanMessage(content='2 multiply by 3'),\n",
+       "  AIMessage(content='2 multiplied by 3 is equal to 6.')],\n",
+       " 'output': '2 multiplied by 3 is equal to 6.'}"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "await run_agent(agent_exe,\"2 multiply by 3\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "stevens-chat-py311-v1",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/streamlit_bot.ipynb

@@ -1,444 +1,444 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### Image Generation Response Analysis"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from openai import OpenAI\n",
-    "from dotenv import load_dotenv\n",
-    "\n",
-    "load_dotenv()\n",
-    "\n",
-    "client = OpenAI()\n",
-    "\n",
-    "response = client.images.generate(\n",
-    "    model=\"dall-e-2\",\n",
-    "    prompt=\"A Persian cat sitting on a grand Persian carpet\",\n",
-    "    size=\"1024x1024\",\n",
-    "    quality=\"standard\",\n",
-    "    n=1,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "openai.types.images_response.ImagesResponse"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "type(response)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 5,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['__abstractmethods__',\n",
-       " '__annotations__',\n",
-       " '__class__',\n",
-       " '__class_getitem__',\n",
-       " '__class_vars__',\n",
-       " '__copy__',\n",
-       " '__deepcopy__',\n",
-       " '__delattr__',\n",
-       " '__dict__',\n",
-       " '__dir__',\n",
-       " '__doc__',\n",
-       " '__eq__',\n",
-       " '__fields__',\n",
-       " '__fields_set__',\n",
-       " '__format__',\n",
-       " '__ge__',\n",
-       " '__get_pydantic_core_schema__',\n",
-       " '__get_pydantic_json_schema__',\n",
-       " '__getattr__',\n",
-       " '__getattribute__',\n",
-       " '__getstate__',\n",
-       " '__gt__',\n",
-       " '__hash__',\n",
-       " '__init__',\n",
-       " '__init_subclass__',\n",
-       " '__iter__',\n",
-       " '__le__',\n",
-       " '__lt__',\n",
-       " '__module__',\n",
-       " '__ne__',\n",
-       " '__new__',\n",
-       " '__pretty__',\n",
-       " '__private_attributes__',\n",
-       " '__pydantic_complete__',\n",
-       " '__pydantic_core_schema__',\n",
-       " '__pydantic_custom_init__',\n",
-       " '__pydantic_decorators__',\n",
-       " '__pydantic_extra__',\n",
-       " '__pydantic_fields_set__',\n",
-       " '__pydantic_generic_metadata__',\n",
-       " '__pydantic_init_subclass__',\n",
-       " '__pydantic_parent_namespace__',\n",
-       " '__pydantic_post_init__',\n",
-       " '__pydantic_private__',\n",
-       " '__pydantic_root_model__',\n",
-       " '__pydantic_serializer__',\n",
-       " '__pydantic_validator__',\n",
-       " '__reduce__',\n",
-       " '__reduce_ex__',\n",
-       " '__repr__',\n",
-       " '__repr_args__',\n",
-       " '__repr_name__',\n",
-       " '__repr_str__',\n",
-       " '__rich_repr__',\n",
-       " '__setattr__',\n",
-       " '__setstate__',\n",
-       " '__signature__',\n",
-       " '__sizeof__',\n",
-       " '__slots__',\n",
-       " '__str__',\n",
-       " '__subclasshook__',\n",
-       " '__weakref__',\n",
-       " '_abc_impl',\n",
-       " '_calculate_keys',\n",
-       " '_check_frozen',\n",
-       " '_copy_and_set_values',\n",
-       " '_get_value',\n",
-       " '_iter',\n",
-       " 'construct',\n",
-       " 'copy',\n",
-       " 'created',\n",
-       " 'data',\n",
-       " 'dict',\n",
-       " 'from_orm',\n",
-       " 'json',\n",
-       " 'model_computed_fields',\n",
-       " 'model_config',\n",
-       " 'model_construct',\n",
-       " 'model_copy',\n",
-       " 'model_dump',\n",
-       " 'model_dump_json',\n",
-       " 'model_extra',\n",
-       " 'model_fields',\n",
-       " 'model_fields_set',\n",
-       " 'model_json_schema',\n",
-       " 'model_parametrized_name',\n",
-       " 'model_post_init',\n",
-       " 'model_rebuild',\n",
-       " 'model_validate',\n",
-       " 'model_validate_json',\n",
-       " 'model_validate_strings',\n",
-       " 'parse_file',\n",
-       " 'parse_obj',\n",
-       " 'parse_raw',\n",
-       " 'schema',\n",
-       " 'schema_json',\n",
-       " 'update_forward_refs',\n",
-       " 'validate']"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "dir(response)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "ImagesResponse(created=1709567373, data=[Image(b64_json=None, revised_prompt=None, url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-bMkUKmvAIOpUOCuXviHRwAQQ/user-u8VOUy4ltWJcbWslWW6PhwXz/img-qHtW8sFZmN0zQAdnArkbjJTj.png?st=2024-03-04T14%3A49%3A33Z&se=2024-03-04T16%3A49%3A33Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-03-04T09%3A37%3A21Z&ske=2024-03-05T09%3A37%3A21Z&sks=b&skv=2021-08-06&sig=8FAlknmnBl7aCOyoZo%2BKsy%2BIrmSSrPv9lfQBo5mmDYk%3D')])"
-      ]
-     },
-     "execution_count": 6,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "response"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "{'b64_json': None,\n",
-       " 'revised_prompt': None,\n",
-       " 'url': 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-bMkUKmvAIOpUOCuXviHRwAQQ/user-u8VOUy4ltWJcbWslWW6PhwXz/img-3uw1d2UZQ6Dd8F4EAENGHZoS.png?st=2024-03-04T22%3A23%3A13Z&se=2024-03-05T00%3A23%3A13Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-03-04T14%3A54%3A11Z&ske=2024-03-05T14%3A54%3A11Z&sks=b&skv=2021-08-06&sig=b1h2vJTXMMFQaBa2vMc4eCyPIfJMEn3LfiV6WLxhto8%3D'}"
-      ]
-     },
-     "execution_count": 3,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "response.data[0].__dict__"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'https://oaidalleapiprodscus.blob.core.windows.net/private/org-bMkUKmvAIOpUOCuXviHRwAQQ/user-u8VOUy4ltWJcbWslWW6PhwXz/img-3uw1d2UZQ6Dd8F4EAENGHZoS.png?st=2024-03-04T22%3A23%3A13Z&se=2024-03-05T00%3A23%3A13Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-03-04T14%3A54%3A11Z&ske=2024-03-05T14%3A54%3A11Z&sks=b&skv=2021-08-06&sig=b1h2vJTXMMFQaBa2vMc4eCyPIfJMEn3LfiV6WLxhto8%3D'"
-      ]
-     },
-     "execution_count": 4,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "response.data[0].url"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import requests\n",
-    "import os \n",
-    "\n",
-    "image = requests.get(response.data[0].url,stream=True)\n",
-    "image_path = os.path.join(\"../bot/images\",\"test2.png\")\n",
-    "if image.status_code == 200:\n",
-    "    with open(image_path, 'wb') as f:\n",
-    "        for chunk in image:\n",
-    "            f.write(chunk)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "requests.models.Response"
-      ]
-     },
-     "execution_count": 13,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "type(image)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 17,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "200\n"
-     ]
-    }
-   ],
-   "source": [
-    "print(image.status_code)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "### OpenAI Package Test"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "['__annotations__',\n",
-       " '__class__',\n",
-       " '__class_getitem__',\n",
-       " '__delattr__',\n",
-       " '__dict__',\n",
-       " '__dir__',\n",
-       " '__doc__',\n",
-       " '__enter__',\n",
-       " '__eq__',\n",
-       " '__exit__',\n",
-       " '__format__',\n",
-       " '__ge__',\n",
-       " '__getattribute__',\n",
-       " '__gt__',\n",
-       " '__hash__',\n",
-       " '__init__',\n",
-       " '__init_subclass__',\n",
-       " '__le__',\n",
-       " '__lt__',\n",
-       " '__module__',\n",
-       " '__ne__',\n",
-       " '__new__',\n",
-       " '__orig_bases__',\n",
-       " '__parameters__',\n",
-       " '__reduce__',\n",
-       " '__reduce_ex__',\n",
-       " '__repr__',\n",
-       " '__setattr__',\n",
-       " '__sizeof__',\n",
-       " '__slots__',\n",
-       " '__str__',\n",
-       " '__subclasshook__',\n",
-       " '__weakref__',\n",
-       " '_base_url',\n",
-       " '_build_headers',\n",
-       " '_build_request',\n",
-       " '_calculate_retry_timeout',\n",
-       " '_client',\n",
-       " '_custom_headers',\n",
-       " '_custom_query',\n",
-       " '_default_stream_cls',\n",
-       " '_enforce_trailing_slash',\n",
-       " '_idempotency_header',\n",
-       " '_idempotency_key',\n",
-       " '_is_protocol',\n",
-       " '_limits',\n",
-       " '_make_status_error',\n",
-       " '_make_status_error_from_response',\n",
-       " '_maybe_override_cast_to',\n",
-       " '_parse_retry_after_header',\n",
-       " '_prepare_options',\n",
-       " '_prepare_request',\n",
-       " '_prepare_url',\n",
-       " '_process_response',\n",
-       " '_process_response_data',\n",
-       " '_proxies',\n",
-       " '_remaining_retries',\n",
-       " '_request',\n",
-       " '_request_api_list',\n",
-       " '_retry_request',\n",
-       " '_serialize_multipartform',\n",
-       " '_should_retry',\n",
-       " '_should_stream_response_body',\n",
-       " '_strict_response_validation',\n",
-       " '_transport',\n",
-       " '_validate_headers',\n",
-       " '_version',\n",
-       " 'api_key',\n",
-       " 'audio',\n",
-       " 'auth_headers',\n",
-       " 'base_url',\n",
-       " 'beta',\n",
-       " 'chat',\n",
-       " 'close',\n",
-       " 'completions',\n",
-       " 'copy',\n",
-       " 'custom_auth',\n",
-       " 'default_headers',\n",
-       " 'delete',\n",
-       " 'embeddings',\n",
-       " 'files',\n",
-       " 'fine_tuning',\n",
-       " 'get',\n",
-       " 'get_api_list',\n",
-       " 'images',\n",
-       " 'is_closed',\n",
-       " 'max_retries',\n",
-       " 'models',\n",
-       " 'moderations',\n",
-       " 'organization',\n",
-       " 'patch',\n",
-       " 'platform_headers',\n",
-       " 'post',\n",
-       " 'put',\n",
-       " 'qs',\n",
-       " 'request',\n",
-       " 'timeout',\n",
-       " 'user_agent',\n",
-       " 'with_options',\n",
-       " 'with_raw_response',\n",
-       " 'with_streaming_response']"
-      ]
-     },
-     "execution_count": 10,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "from openai import OpenAI\n",
-    "\n",
-    "client = OpenAI()\n",
-    "dir(client)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "stevens-chat-v2-py310",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Image Generation Response Analysis"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from openai import OpenAI\n",
+    "from dotenv import load_dotenv\n",
+    "\n",
+    "load_dotenv()\n",
+    "\n",
+    "client = OpenAI()\n",
+    "\n",
+    "response = client.images.generate(\n",
+    "    model=\"dall-e-2\",\n",
+    "    prompt=\"A Persian cat sitting on a grand Persian carpet\",\n",
+    "    size=\"1024x1024\",\n",
+    "    quality=\"standard\",\n",
+    "    n=1,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "openai.types.images_response.ImagesResponse"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['__abstractmethods__',\n",
+       " '__annotations__',\n",
+       " '__class__',\n",
+       " '__class_getitem__',\n",
+       " '__class_vars__',\n",
+       " '__copy__',\n",
+       " '__deepcopy__',\n",
+       " '__delattr__',\n",
+       " '__dict__',\n",
+       " '__dir__',\n",
+       " '__doc__',\n",
+       " '__eq__',\n",
+       " '__fields__',\n",
+       " '__fields_set__',\n",
+       " '__format__',\n",
+       " '__ge__',\n",
+       " '__get_pydantic_core_schema__',\n",
+       " '__get_pydantic_json_schema__',\n",
+       " '__getattr__',\n",
+       " '__getattribute__',\n",
+       " '__getstate__',\n",
+       " '__gt__',\n",
+       " '__hash__',\n",
+       " '__init__',\n",
+       " '__init_subclass__',\n",
+       " '__iter__',\n",
+       " '__le__',\n",
+       " '__lt__',\n",
+       " '__module__',\n",
+       " '__ne__',\n",
+       " '__new__',\n",
+       " '__pretty__',\n",
+       " '__private_attributes__',\n",
+       " '__pydantic_complete__',\n",
+       " '__pydantic_core_schema__',\n",
+       " '__pydantic_custom_init__',\n",
+       " '__pydantic_decorators__',\n",
+       " '__pydantic_extra__',\n",
+       " '__pydantic_fields_set__',\n",
+       " '__pydantic_generic_metadata__',\n",
+       " '__pydantic_init_subclass__',\n",
+       " '__pydantic_parent_namespace__',\n",
+       " '__pydantic_post_init__',\n",
+       " '__pydantic_private__',\n",
+       " '__pydantic_root_model__',\n",
+       " '__pydantic_serializer__',\n",
+       " '__pydantic_validator__',\n",
+       " '__reduce__',\n",
+       " '__reduce_ex__',\n",
+       " '__repr__',\n",
+       " '__repr_args__',\n",
+       " '__repr_name__',\n",
+       " '__repr_str__',\n",
+       " '__rich_repr__',\n",
+       " '__setattr__',\n",
+       " '__setstate__',\n",
+       " '__signature__',\n",
+       " '__sizeof__',\n",
+       " '__slots__',\n",
+       " '__str__',\n",
+       " '__subclasshook__',\n",
+       " '__weakref__',\n",
+       " '_abc_impl',\n",
+       " '_calculate_keys',\n",
+       " '_check_frozen',\n",
+       " '_copy_and_set_values',\n",
+       " '_get_value',\n",
+       " '_iter',\n",
+       " 'construct',\n",
+       " 'copy',\n",
+       " 'created',\n",
+       " 'data',\n",
+       " 'dict',\n",
+       " 'from_orm',\n",
+       " 'json',\n",
+       " 'model_computed_fields',\n",
+       " 'model_config',\n",
+       " 'model_construct',\n",
+       " 'model_copy',\n",
+       " 'model_dump',\n",
+       " 'model_dump_json',\n",
+       " 'model_extra',\n",
+       " 'model_fields',\n",
+       " 'model_fields_set',\n",
+       " 'model_json_schema',\n",
+       " 'model_parametrized_name',\n",
+       " 'model_post_init',\n",
+       " 'model_rebuild',\n",
+       " 'model_validate',\n",
+       " 'model_validate_json',\n",
+       " 'model_validate_strings',\n",
+       " 'parse_file',\n",
+       " 'parse_obj',\n",
+       " 'parse_raw',\n",
+       " 'schema',\n",
+       " 'schema_json',\n",
+       " 'update_forward_refs',\n",
+       " 'validate']"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dir(response)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ImagesResponse(created=1709567373, data=[Image(b64_json=None, revised_prompt=None, url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-bMkUKmvAIOpUOCuXviHRwAQQ/user-u8VOUy4ltWJcbWslWW6PhwXz/img-qHtW8sFZmN0zQAdnArkbjJTj.png?st=2024-03-04T14%3A49%3A33Z&se=2024-03-04T16%3A49%3A33Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-03-04T09%3A37%3A21Z&ske=2024-03-05T09%3A37%3A21Z&sks=b&skv=2021-08-06&sig=8FAlknmnBl7aCOyoZo%2BKsy%2BIrmSSrPv9lfQBo5mmDYk%3D')])"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'b64_json': None,\n",
+       " 'revised_prompt': None,\n",
+       " 'url': 'https://oaidalleapiprodscus.blob.core.windows.net/private/org-bMkUKmvAIOpUOCuXviHRwAQQ/user-u8VOUy4ltWJcbWslWW6PhwXz/img-3uw1d2UZQ6Dd8F4EAENGHZoS.png?st=2024-03-04T22%3A23%3A13Z&se=2024-03-05T00%3A23%3A13Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-03-04T14%3A54%3A11Z&ske=2024-03-05T14%3A54%3A11Z&sks=b&skv=2021-08-06&sig=b1h2vJTXMMFQaBa2vMc4eCyPIfJMEn3LfiV6WLxhto8%3D'}"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.data[0].__dict__"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'https://oaidalleapiprodscus.blob.core.windows.net/private/org-bMkUKmvAIOpUOCuXviHRwAQQ/user-u8VOUy4ltWJcbWslWW6PhwXz/img-3uw1d2UZQ6Dd8F4EAENGHZoS.png?st=2024-03-04T22%3A23%3A13Z&se=2024-03-05T00%3A23%3A13Z&sp=r&sv=2021-08-06&sr=b&rscd=inline&rsct=image/png&skoid=6aaadede-4fb3-4698-a8f6-684d7786b067&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-03-04T14%3A54%3A11Z&ske=2024-03-05T14%3A54%3A11Z&sks=b&skv=2021-08-06&sig=b1h2vJTXMMFQaBa2vMc4eCyPIfJMEn3LfiV6WLxhto8%3D'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "response.data[0].url"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "import os \n",
+    "\n",
+    "image = requests.get(response.data[0].url,stream=True)\n",
+    "image_path = os.path.join(\"../bot/images\",\"test2.png\")\n",
+    "if image.status_code == 200:\n",
+    "    with open(image_path, 'wb') as f:\n",
+    "        for chunk in image:\n",
+    "            f.write(chunk)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "requests.models.Response"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(image)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "200\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(image.status_code)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### OpenAI Package Test"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['__annotations__',\n",
+       " '__class__',\n",
+       " '__class_getitem__',\n",
+       " '__delattr__',\n",
+       " '__dict__',\n",
+       " '__dir__',\n",
+       " '__doc__',\n",
+       " '__enter__',\n",
+       " '__eq__',\n",
+       " '__exit__',\n",
+       " '__format__',\n",
+       " '__ge__',\n",
+       " '__getattribute__',\n",
+       " '__gt__',\n",
+       " '__hash__',\n",
+       " '__init__',\n",
+       " '__init_subclass__',\n",
+       " '__le__',\n",
+       " '__lt__',\n",
+       " '__module__',\n",
+       " '__ne__',\n",
+       " '__new__',\n",
+       " '__orig_bases__',\n",
+       " '__parameters__',\n",
+       " '__reduce__',\n",
+       " '__reduce_ex__',\n",
+       " '__repr__',\n",
+       " '__setattr__',\n",
+       " '__sizeof__',\n",
+       " '__slots__',\n",
+       " '__str__',\n",
+       " '__subclasshook__',\n",
+       " '__weakref__',\n",
+       " '_base_url',\n",
+       " '_build_headers',\n",
+       " '_build_request',\n",
+       " '_calculate_retry_timeout',\n",
+       " '_client',\n",
+       " '_custom_headers',\n",
+       " '_custom_query',\n",
+       " '_default_stream_cls',\n",
+       " '_enforce_trailing_slash',\n",
+       " '_idempotency_header',\n",
+       " '_idempotency_key',\n",
+       " '_is_protocol',\n",
+       " '_limits',\n",
+       " '_make_status_error',\n",
+       " '_make_status_error_from_response',\n",
+       " '_maybe_override_cast_to',\n",
+       " '_parse_retry_after_header',\n",
+       " '_prepare_options',\n",
+       " '_prepare_request',\n",
+       " '_prepare_url',\n",
+       " '_process_response',\n",
+       " '_process_response_data',\n",
+       " '_proxies',\n",
+       " '_remaining_retries',\n",
+       " '_request',\n",
+       " '_request_api_list',\n",
+       " '_retry_request',\n",
+       " '_serialize_multipartform',\n",
+       " '_should_retry',\n",
+       " '_should_stream_response_body',\n",
+       " '_strict_response_validation',\n",
+       " '_transport',\n",
+       " '_validate_headers',\n",
+       " '_version',\n",
+       " 'api_key',\n",
+       " 'audio',\n",
+       " 'auth_headers',\n",
+       " 'base_url',\n",
+       " 'beta',\n",
+       " 'chat',\n",
+       " 'close',\n",
+       " 'completions',\n",
+       " 'copy',\n",
+       " 'custom_auth',\n",
+       " 'default_headers',\n",
+       " 'delete',\n",
+       " 'embeddings',\n",
+       " 'files',\n",
+       " 'fine_tuning',\n",
+       " 'get',\n",
+       " 'get_api_list',\n",
+       " 'images',\n",
+       " 'is_closed',\n",
+       " 'max_retries',\n",
+       " 'models',\n",
+       " 'moderations',\n",
+       " 'organization',\n",
+       " 'patch',\n",
+       " 'platform_headers',\n",
+       " 'post',\n",
+       " 'put',\n",
+       " 'qs',\n",
+       " 'request',\n",
+       " 'timeout',\n",
+       " 'user_agent',\n",
+       " 'with_options',\n",
+       " 'with_raw_response',\n",
+       " 'with_streaming_response']"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from openai import OpenAI\n",
+    "\n",
+    "client = OpenAI()\n",
+    "dir(client)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "stevens-chat-v2-py310",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

notebooks/web_scrapping.ipynb

@@ -1,348 +1,348 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "from selenium import webdriver\n",
-    "from selenium.webdriver.common.by import By\n",
-    "import time\n",
-    "import spacy"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def tokenize(text):\n",
-    "    nlp = spacy.load('en_core_web_sm')\n",
-    "    tokens = [token.text for token in nlp(text) if not token.is_punct and not token.is_space]\n",
-    "    \n",
-    "    return tokens"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def add_target_columns(data, category, target):\n",
-    "    # Column to store category of questions\n",
-    "    data[\"Category\"] = category\n",
-    "    # Target column to store whether human answer or chatgpt answer\n",
-    "    data[\"Human vs ChatGPT\"] = target\n",
-    "    \n",
-    "    return data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def store_excel(data, prev_data = None):\n",
-    "    if prev_data:\n",
-    "        # Loading old data into dataframe\n",
-    "        old_data = pd.read_excel(prev_data)\n",
-    "        # Concatenating the two dataframes vertically\n",
-    "        complete_data = pd.concat([old_data, data], ignore_index=True)\n",
-    "        # Storing the combined data to the excel file\n",
-    "        complete_data.to_excel('scraped_data.xlsx', index=False)\n",
-    "    else:\n",
-    "        data.to_excel(\"scraped_data.xlsx\", index=False)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def getData(page_urls, driver, min_ans_len = 15, limit = 10, scroll_num = 10):\n",
-    "    \n",
-    "    # Empty dataframe to store the scraped content\n",
-    "    scraped_data = pd.DataFrame()\n",
-    "    \n",
-    "    # Initializing variable to track the number of data samples collected \n",
-    "    len_data = 0\n",
-    "    \n",
-    "    # Initializing lists to store the scraped content\n",
-    "    questions = []\n",
-    "    answers = []\n",
-    "    \n",
-    "    count = 1\n",
-    "    count1 = 0\n",
-    "    for page_url in page_urls:\n",
-    "        print(f\"Page {count} of {len(page_urls)}\")\n",
-    "        # Sending a get request to the web page (Navigating to the webpage)\n",
-    "        driver.get(page_url)\n",
-    "        # Wait\n",
-    "        driver.implicitly_wait(10)\n",
-    "\n",
-    "        # Initializing variables to iterate through the try except block\n",
-    "        max_tries = 10\n",
-    "        retry = 0   \n",
-    "\n",
-    "        # Initializing variable to check if we've reached the end of the page \n",
-    "        old_content = None\n",
-    "        new_content = None\n",
-    "\n",
-    "        # Scrolling to get enough answers\n",
-    "        for i in range(scroll_num):\n",
-    "            # Scrolling to access the next page of questions\n",
-    "            driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
-    "            # Wait for the content to be loaded        \n",
-    "            time.sleep(5) \n",
-    "            # Checking if page is same before and after scrolling\n",
-    "            new_content = driver.page_source\n",
-    "            if new_content == old_content:\n",
-    "                break\n",
-    "            old_content = new_content\n",
-    "\n",
-    "        # Used while loop to avoid \"StaleElementReferenceException\" error\n",
-    "        while retry<max_tries:\n",
-    "            try:\n",
-    "                # Scraping the question answer blocks on Quora\n",
-    "                data_elements = driver.find_elements(By.CSS_SELECTOR, \"div.dom_annotate_multifeed_bundle_AnswersBundle\")\n",
-    "                retry = 0\n",
-    "                break\n",
-    "            except:\n",
-    "                retry += 1\n",
-    "\n",
-    "\n",
-    "        # Used while loop to avoid \"StaleElementReferenceException\" error\n",
-    "        while retry<max_tries:\n",
-    "            try:\n",
-    "                for block in data_elements:\n",
-    "\n",
-    "                    ### --- Questions --- ###\n",
-    "                    # Scraping question from the webpage \n",
-    "                    ques = block.find_element(By.CSS_SELECTOR, \"div.q-text.puppeteer_test_question_title span\")\n",
-    "\n",
-    "                    ### --- Answers --- ###\n",
-    "                    # Checking if \"more\" button is present for an answer\n",
-    "                    try: \n",
-    "                        # Selecting the \"more\" button\n",
-    "                        read_more = block.find_element(By.CSS_SELECTOR, \"div.q-absolute div.qt_read_more\")\n",
-    "                        # Checking if the button is clickable\n",
-    "                        try:\n",
-    "                            # Expanding answer by clicking \"more\" button\n",
-    "                            read_more.click()\n",
-    "                        except:\n",
-    "                            # Discarding data where complete answer cannot be obtained\n",
-    "                            continue\n",
-    "                    except:\n",
-    "                        None\n",
-    "                    # Scraping answers from the webpage \n",
-    "                    ans = block.find_element(By.CSS_SELECTOR, \"div.q-box.spacing_log_answer_content.puppeteer_test_answer_content span.q-box\")\n",
-    "\n",
-    "                    if ques.text and ans.text:\n",
-    "                        # Skipping questions that are already present\n",
-    "                        if ques.text in questions:\n",
-    "                            continue\n",
-    "                        # Skipping the questions where length of answers are less than a given threshold\n",
-    "                        ans_tokens = len(tokenize(ans.text))\n",
-    "                        if ans_tokens<min_ans_len:\n",
-    "                            continue\n",
-    "                        # Appending the scraped question\n",
-    "                        questions.append(ques.text)\n",
-    "                        # Appending the scraped answer\n",
-    "                        answers.append(ans.text)\n",
-    "                        count1+=1\n",
-    "                        print(f\"{count1} of 250\")\n",
-    "                    else:\n",
-    "                        continue\n",
-    "\n",
-    "                    # Updating the number of data samples collected\n",
-    "                    len_data = len(questions) \n",
-    "                    # Collecting data until limit is reached \n",
-    "                    if len_data == limit:\n",
-    "                        break\n",
-    "                retry = 0\n",
-    "                break\n",
-    "            except:\n",
-    "                retry += 1 \n",
-    "        count+=1\n",
-    "        if len_data == limit:\n",
-    "            break\n",
-    "        \n",
-    "    # Warning to give more urls if desired amount of data is not scraped\n",
-    "    if len_data < limit:\n",
-    "        print(\"Warning: Need to provide more webpages to get desired amount of data!\")\n",
-    "        \n",
-    "    # Storing the scraped information in a dataframe  \n",
-    "    scraped_data[\"Questions\"] = questions\n",
-    "    scraped_data[\"Answers\"] = answers\n",
-    "    \n",
-    "    return scraped_data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import requests\n",
-    "from bs4 import BeautifulSoup"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "url = \"https://boards.greenhouse.io/enveritas/jobs/4001717008\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Send a GET request to the URL\n",
-    "response = requests.get(url)\n",
-    "\n",
-    "# Parse the HTML content with BeautifulSoup\n",
-    "soup = BeautifulSoup(response.content, \"html.parser\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Job Title: Data Scientist\n",
-      "Company: Enveritas\n",
-      "Job Location: Global / Remote\n",
-      "Job Description: \n",
-      "Data Scientist, Engineering & Data Group\n",
-      "Do you want to work for a mission-driven non-profit, analyzing data and writing software that will contribute to helping millions of coffee farmers out of poverty? Enveritas is a 501(c)3 non-profit and Y Combinator-backed startup looking to hire a Data Scientist for our Data Team. \n",
-      "We are looking for a Data Scientist with extensive professional experience to join our Engineering and Data Group on a remote, full-time basis. This position is open globally, based on locations supported by our EOR partner, Deel. You can learn more about this role at https://www.enveritas.org/jobs/data-scientist.\n",
-      "Our Engineering and Data Group is a quirky, talented, and humble group of about twenty with diverse backgrounds ranging from journalism to academia to international industry.\n",
-      "About Our Data Team\n",
-      "The Data Team's mission is to leverage data analytics to drive Enveritas' efforts in improving the livelihoods of smallholder farmers around the world. We are responsible for ensuring that we collect high quality data, we leverage it to generate actionable insights that support smallholder farmers worldwide, and we make the data and insights accessible to other teams, such as the Operations and Partnerships Team.\n",
-      "To improve the quality of our data collection process, we create tools to support our Country Operations teams. These tools enable the detection of outliers and automating quality control processes while collecting our survey data. \n",
-      "We are also responsible for the transformation of our raw data into insights. This includes writing the code that scores our standards, and creating models that give new insights about smallholder farming. For example, we estimate the cost farmers face when growing coffee to know if they earn a livable income.\n",
-      "Our programming languages of choice are Python and SQL (we use PostgreSQL), although some of our analysis is also done in R. We use git for version control, Github for hosting our repositories, and pytest for automated testing. Our internal BI tool is Looker.\n",
-      "What You’ll Be Doing\n",
-      "Providing technical oversight and mentorship. You will provide technical oversight to the models the Data Team produces. Additionally, you will offer guidance on and document statistical best practices and methodologies, helping to elevate the overall analytical capabilities within the Data Team and the organization.\n",
-      "Build models to create new insights from our data. You will identify opportunities for creating models using the data we collect. Engage with the Operations and Partnerships teams to generate ideas for models that would be beneficial for them, and the coffee and cocoa community. This includes using Python to craft these models, incorporating them into our data pipeline, and effectively synthesizing, visualizing, and conveying the results.\n",
-      "Serve as internal statistics consultant. Undertake the role of an internal statistical advisor, providing expertise across the organization for all statistics-related matters. This includes aiding teams in developing innovative sampling methods, calculating appropriate sample sizes, interpreting data analysis results, and ensuring the statistical integrity of projects and data collection. \n",
-      "Who You Are\n",
-      "Our team is fully distributed, so you should be comfortable with remote work. This role is a full-time individual contributor role. While you can be located anywhere, our core hours are 10am to 2pm Eastern Time Monday through Friday, with team members choosing either an early start or later stop as suits them.\n",
-      "Our work is often ambiguous, so you should have a love for environments with uncertainty. You should have a deep empathy for users of our tools and understand the importance of supporting the work of other teams. You should also be willing to engage in a broad range of tasks beyond standard data science functions, and you are not afraid of using a simple model that gets the job done.\n",
-      " \n",
-      "Qualifications\n",
-      "Read this first: Research shows that people of different backgrounds read job postings differently. If you don’t think you meet all of the qualifications but do think you’d be a great match for us, please consider applying and sharing more in your cover letter. We’d love to talk with you to see what skills you can bring to our team. This said, we are most likely to be interested in your candidacy if you can demonstrate the majority of the qualifications listed below:\n",
-      "\n",
-      "Extensive professional experience as a data scientist, a statistician, or equivalent.\n",
-      "Hands-on experience building and deploying statistical models. Strong theoretical knowledge of probability and statistics.\n",
-      "Intermediate Python. Experience with good software engineering practices such as reproducible environments, testing, and code quality.\n",
-      "A desire to apply your skills at a non-profit working to improve the livelihoods of smallholder farmers.\n",
-      "Have experience or expertise in one or more of the following:\n",
-      "\n",
-      "Building sustainability or agricultural models.\n",
-      "Advanced statistical modeling.\n",
-      "Survey sampling.\n",
-      "Survey methodology.\n",
-      "Bayesian statistical modeling.\n",
-      "Data Management.\n",
-      "Causal inference.\n",
-      "Managing teams.\n",
-      "\n",
-      "\n",
-      "\n",
-      " \n",
-      "About Working With Us & Compensation\n",
-      "Enveritas has teams around the world: we are about 90 people spread over almost two dozen countries, and of all backgrounds, faiths, and identities. To learn more about working at Enveritas, see https://www.enveritas.org/jobs/\n",
-      "For a US-Based hire, base salary for this position will be between $110,000-$130,000 annually (paid semi-monthly). This is a full-time exempt position. Full benefits include 401k with matching contributions, Medical/Dental/Vision, and Flexible Spending Account (FSA), 4 weeks vacation in addition to 12 standard holidays, and personal/sick time.\n",
-      "For a hire outside the US, our offer will be competitive; the specific benefits and compensation details will vary as required to account for your region’s laws and requirements. Salary for this position will be paid in relevant local currency.\n",
-      "For all staff, we are able to offer:\n",
-      "\n",
-      "Annual education budget for conferences, books, and other professional development opportunities.\n",
-      "Annual all-company retreat and annual group retreat.\n",
-      "Field visits to our Country Ops teams in coffee-growing countries such as Colombia, Costa Rica, Ethiopia, and Indonesia.\n",
-      "\n",
-      "Interview Process\n",
-      "We are committed to fair and equitable hiring. To honor this commitment, we are being transparent about our interview process. We are interested in learning what working with you would be like and believe the below is the fairest method for us to see you at your best — and for you to learn about us! If you feel that a different method would be better for us to learn what working together would be like, please tell us in your application. \n",
-      "After your introductory interview, we expect your interview process to take four to six weeks (but will depend on scheduling), and consist of four conversations that total about five hours of time. You should plan to also spend about four hours in total preparing for interviews.  See the hiring page at https://www.enveritas.org/jobs/data-scientist/ for details about each of these interviews.\n",
-      "\n",
-      "Introductory Interview (30 minutes; Google Meet; audio-only)\n",
-      "First Technical Interview (60-90 minutes; Google Meet video)\n",
-      "Second Technical Interview (60-90 minutes; Google Meet video)\n",
-      "Manager Interview (45 minutes; Google Meet video)\n",
-      "\n",
-      "How to Apply\n",
-      "Please apply here. Feel free to contact us at jobs@enveritas.org should you have any questions about the position, the interview process, or if you require any adjustments to ensure a fair and equitable application process. Questions about this opportunity or process will not reflect negatively on your application.\n",
-      "A few notes about our communications: We are not able to reply to messages sent to staff outside of either our application process or our jobs email address, as this is unfair to other candidates. Also, Enveritas has been made aware of fake job postings by individuals pretending to hire persons seeking employment. These individuals are looking to collect personal information about you for fraudulent purposes. All legitimate Enveritas job openings are posted on our Greenhouse board at https://boards.greenhouse.io/enveritas or Lever board at https://jobs.lever.co/Enveritas. All recruiting emails from Enveritas team members will come through Lever or from @enveritas.org. If you have any concerns about employment opportunities or contact from someone supposedly representing Enveritas, please reach out to us at jobs@enveritas.org\n"
-     ]
-    }
-   ],
-   "source": [
-    "# Extract the job description content\n",
-    "job_title = soup.find(\"h1\", {\"class\": \"app-title\"})\n",
-    "\n",
-    "company_name = soup.find(\"span\",{\"class\": \"company-name\"})\n",
-    "\n",
-    "job_location = soup.find(\"div\", {\"class\": \"location\"})\n",
-    "\n",
-    "job_description = soup.find(\"div\", {\"id\": \"content\"})\n",
-    "\n",
-    "# Print the job description\n",
-    "if job_description:\n",
-    "    print(\"Job Title: \"+job_title.get_text()+\"\\n\"+\"Company: \"+company_name.get_text().strip().split(\"at \")[1]+\"\\n\"+\"Job Location: \"+job_location.get_text().strip()+\"\\n\"+\"Job Description: \\n\"+job_description.get_text().strip())\n",
-    "else:\n",
-    "    print(\"Job description not found.\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "my_env",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.6"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from selenium import webdriver\n",
+    "from selenium.webdriver.common.by import By\n",
+    "import time\n",
+    "import spacy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def tokenize(text):\n",
+    "    nlp = spacy.load('en_core_web_sm')\n",
+    "    tokens = [token.text for token in nlp(text) if not token.is_punct and not token.is_space]\n",
+    "    \n",
+    "    return tokens"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def add_target_columns(data, category, target):\n",
+    "    # Column to store category of questions\n",
+    "    data[\"Category\"] = category\n",
+    "    # Target column to store whether human answer or chatgpt answer\n",
+    "    data[\"Human vs ChatGPT\"] = target\n",
+    "    \n",
+    "    return data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def store_excel(data, prev_data = None):\n",
+    "    if prev_data:\n",
+    "        # Loading old data into dataframe\n",
+    "        old_data = pd.read_excel(prev_data)\n",
+    "        # Concatenating the two dataframes vertically\n",
+    "        complete_data = pd.concat([old_data, data], ignore_index=True)\n",
+    "        # Storing the combined data to the excel file\n",
+    "        complete_data.to_excel('scraped_data.xlsx', index=False)\n",
+    "    else:\n",
+    "        data.to_excel(\"scraped_data.xlsx\", index=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def getData(page_urls, driver, min_ans_len = 15, limit = 10, scroll_num = 10):\n",
+    "    \n",
+    "    # Empty dataframe to store the scraped content\n",
+    "    scraped_data = pd.DataFrame()\n",
+    "    \n",
+    "    # Initializing variable to track the number of data samples collected \n",
+    "    len_data = 0\n",
+    "    \n",
+    "    # Initializing lists to store the scraped content\n",
+    "    questions = []\n",
+    "    answers = []\n",
+    "    \n",
+    "    count = 1\n",
+    "    count1 = 0\n",
+    "    for page_url in page_urls:\n",
+    "        print(f\"Page {count} of {len(page_urls)}\")\n",
+    "        # Sending a get request to the web page (Navigating to the webpage)\n",
+    "        driver.get(page_url)\n",
+    "        # Wait\n",
+    "        driver.implicitly_wait(10)\n",
+    "\n",
+    "        # Initializing variables to iterate through the try except block\n",
+    "        max_tries = 10\n",
+    "        retry = 0   \n",
+    "\n",
+    "        # Initializing variable to check if we've reached the end of the page \n",
+    "        old_content = None\n",
+    "        new_content = None\n",
+    "\n",
+    "        # Scrolling to get enough answers\n",
+    "        for i in range(scroll_num):\n",
+    "            # Scrolling to access the next page of questions\n",
+    "            driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
+    "            # Wait for the content to be loaded        \n",
+    "            time.sleep(5) \n",
+    "            # Checking if page is same before and after scrolling\n",
+    "            new_content = driver.page_source\n",
+    "            if new_content == old_content:\n",
+    "                break\n",
+    "            old_content = new_content\n",
+    "\n",
+    "        # Used while loop to avoid \"StaleElementReferenceException\" error\n",
+    "        while retry<max_tries:\n",
+    "            try:\n",
+    "                # Scraping the question answer blocks on Quora\n",
+    "                data_elements = driver.find_elements(By.CSS_SELECTOR, \"div.dom_annotate_multifeed_bundle_AnswersBundle\")\n",
+    "                retry = 0\n",
+    "                break\n",
+    "            except:\n",
+    "                retry += 1\n",
+    "\n",
+    "\n",
+    "        # Used while loop to avoid \"StaleElementReferenceException\" error\n",
+    "        while retry<max_tries:\n",
+    "            try:\n",
+    "                for block in data_elements:\n",
+    "\n",
+    "                    ### --- Questions --- ###\n",
+    "                    # Scraping question from the webpage \n",
+    "                    ques = block.find_element(By.CSS_SELECTOR, \"div.q-text.puppeteer_test_question_title span\")\n",
+    "\n",
+    "                    ### --- Answers --- ###\n",
+    "                    # Checking if \"more\" button is present for an answer\n",
+    "                    try: \n",
+    "                        # Selecting the \"more\" button\n",
+    "                        read_more = block.find_element(By.CSS_SELECTOR, \"div.q-absolute div.qt_read_more\")\n",
+    "                        # Checking if the button is clickable\n",
+    "                        try:\n",
+    "                            # Expanding answer by clicking \"more\" button\n",
+    "                            read_more.click()\n",
+    "                        except:\n",
+    "                            # Discarding data where complete answer cannot be obtained\n",
+    "                            continue\n",
+    "                    except:\n",
+    "                        None\n",
+    "                    # Scraping answers from the webpage \n",
+    "                    ans = block.find_element(By.CSS_SELECTOR, \"div.q-box.spacing_log_answer_content.puppeteer_test_answer_content span.q-box\")\n",
+    "\n",
+    "                    if ques.text and ans.text:\n",
+    "                        # Skipping questions that are already present\n",
+    "                        if ques.text in questions:\n",
+    "                            continue\n",
+    "                        # Skipping the questions where length of answers are less than a given threshold\n",
+    "                        ans_tokens = len(tokenize(ans.text))\n",
+    "                        if ans_tokens<min_ans_len:\n",
+    "                            continue\n",
+    "                        # Appending the scraped question\n",
+    "                        questions.append(ques.text)\n",
+    "                        # Appending the scraped answer\n",
+    "                        answers.append(ans.text)\n",
+    "                        count1+=1\n",
+    "                        print(f\"{count1} of 250\")\n",
+    "                    else:\n",
+    "                        continue\n",
+    "\n",
+    "                    # Updating the number of data samples collected\n",
+    "                    len_data = len(questions) \n",
+    "                    # Collecting data until limit is reached \n",
+    "                    if len_data == limit:\n",
+    "                        break\n",
+    "                retry = 0\n",
+    "                break\n",
+    "            except:\n",
+    "                retry += 1 \n",
+    "        count+=1\n",
+    "        if len_data == limit:\n",
+    "            break\n",
+    "        \n",
+    "    # Warning to give more urls if desired amount of data is not scraped\n",
+    "    if len_data < limit:\n",
+    "        print(\"Warning: Need to provide more webpages to get desired amount of data!\")\n",
+    "        \n",
+    "    # Storing the scraped information in a dataframe  \n",
+    "    scraped_data[\"Questions\"] = questions\n",
+    "    scraped_data[\"Answers\"] = answers\n",
+    "    \n",
+    "    return scraped_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from bs4 import BeautifulSoup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = \"https://boards.greenhouse.io/enveritas/jobs/4001717008\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Send a GET request to the URL\n",
+    "response = requests.get(url)\n",
+    "\n",
+    "# Parse the HTML content with BeautifulSoup\n",
+    "soup = BeautifulSoup(response.content, \"html.parser\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Job Title: Data Scientist\n",
+      "Company: Enveritas\n",
+      "Job Location: Global / Remote\n",
+      "Job Description: \n",
+      "Data Scientist, Engineering & Data Group\n",
+      "Do you want to work for a mission-driven non-profit, analyzing data and writing software that will contribute to helping millions of coffee farmers out of poverty? Enveritas is a 501(c)3 non-profit and Y Combinator-backed startup looking to hire a Data Scientist for our Data Team. \n",
+      "We are looking for a Data Scientist with extensive professional experience to join our Engineering and Data Group on a remote, full-time basis. This position is open globally, based on locations supported by our EOR partner, Deel. You can learn more about this role at https://www.enveritas.org/jobs/data-scientist.\n",
+      "Our Engineering and Data Group is a quirky, talented, and humble group of about twenty with diverse backgrounds ranging from journalism to academia to international industry.\n",
+      "About Our Data Team\n",
+      "The Data Team's mission is to leverage data analytics to drive Enveritas' efforts in improving the livelihoods of smallholder farmers around the world. We are responsible for ensuring that we collect high quality data, we leverage it to generate actionable insights that support smallholder farmers worldwide, and we make the data and insights accessible to other teams, such as the Operations and Partnerships Team.\n",
+      "To improve the quality of our data collection process, we create tools to support our Country Operations teams. These tools enable the detection of outliers and automating quality control processes while collecting our survey data. \n",
+      "We are also responsible for the transformation of our raw data into insights. This includes writing the code that scores our standards, and creating models that give new insights about smallholder farming. For example, we estimate the cost farmers face when growing coffee to know if they earn a livable income.\n",
+      "Our programming languages of choice are Python and SQL (we use PostgreSQL), although some of our analysis is also done in R. We use git for version control, Github for hosting our repositories, and pytest for automated testing. Our internal BI tool is Looker.\n",
+      "What You’ll Be Doing\n",
+      "Providing technical oversight and mentorship. You will provide technical oversight to the models the Data Team produces. Additionally, you will offer guidance on and document statistical best practices and methodologies, helping to elevate the overall analytical capabilities within the Data Team and the organization.\n",
+      "Build models to create new insights from our data. You will identify opportunities for creating models using the data we collect. Engage with the Operations and Partnerships teams to generate ideas for models that would be beneficial for them, and the coffee and cocoa community. This includes using Python to craft these models, incorporating them into our data pipeline, and effectively synthesizing, visualizing, and conveying the results.\n",
+      "Serve as internal statistics consultant. Undertake the role of an internal statistical advisor, providing expertise across the organization for all statistics-related matters. This includes aiding teams in developing innovative sampling methods, calculating appropriate sample sizes, interpreting data analysis results, and ensuring the statistical integrity of projects and data collection. \n",
+      "Who You Are\n",
+      "Our team is fully distributed, so you should be comfortable with remote work. This role is a full-time individual contributor role. While you can be located anywhere, our core hours are 10am to 2pm Eastern Time Monday through Friday, with team members choosing either an early start or later stop as suits them.\n",
+      "Our work is often ambiguous, so you should have a love for environments with uncertainty. You should have a deep empathy for users of our tools and understand the importance of supporting the work of other teams. You should also be willing to engage in a broad range of tasks beyond standard data science functions, and you are not afraid of using a simple model that gets the job done.\n",
+      " \n",
+      "Qualifications\n",
+      "Read this first: Research shows that people of different backgrounds read job postings differently. If you don’t think you meet all of the qualifications but do think you’d be a great match for us, please consider applying and sharing more in your cover letter. We’d love to talk with you to see what skills you can bring to our team. This said, we are most likely to be interested in your candidacy if you can demonstrate the majority of the qualifications listed below:\n",
+      "\n",
+      "Extensive professional experience as a data scientist, a statistician, or equivalent.\n",
+      "Hands-on experience building and deploying statistical models. Strong theoretical knowledge of probability and statistics.\n",
+      "Intermediate Python. Experience with good software engineering practices such as reproducible environments, testing, and code quality.\n",
+      "A desire to apply your skills at a non-profit working to improve the livelihoods of smallholder farmers.\n",
+      "Have experience or expertise in one or more of the following:\n",
+      "\n",
+      "Building sustainability or agricultural models.\n",
+      "Advanced statistical modeling.\n",
+      "Survey sampling.\n",
+      "Survey methodology.\n",
+      "Bayesian statistical modeling.\n",
+      "Data Management.\n",
+      "Causal inference.\n",
+      "Managing teams.\n",
+      "\n",
+      "\n",
+      "\n",
+      " \n",
+      "About Working With Us & Compensation\n",
+      "Enveritas has teams around the world: we are about 90 people spread over almost two dozen countries, and of all backgrounds, faiths, and identities. To learn more about working at Enveritas, see https://www.enveritas.org/jobs/\n",
+      "For a US-Based hire, base salary for this position will be between $110,000-$130,000 annually (paid semi-monthly). This is a full-time exempt position. Full benefits include 401k with matching contributions, Medical/Dental/Vision, and Flexible Spending Account (FSA), 4 weeks vacation in addition to 12 standard holidays, and personal/sick time.\n",
+      "For a hire outside the US, our offer will be competitive; the specific benefits and compensation details will vary as required to account for your region’s laws and requirements. Salary for this position will be paid in relevant local currency.\n",
+      "For all staff, we are able to offer:\n",
+      "\n",
+      "Annual education budget for conferences, books, and other professional development opportunities.\n",
+      "Annual all-company retreat and annual group retreat.\n",
+      "Field visits to our Country Ops teams in coffee-growing countries such as Colombia, Costa Rica, Ethiopia, and Indonesia.\n",
+      "\n",
+      "Interview Process\n",
+      "We are committed to fair and equitable hiring. To honor this commitment, we are being transparent about our interview process. We are interested in learning what working with you would be like and believe the below is the fairest method for us to see you at your best — and for you to learn about us! If you feel that a different method would be better for us to learn what working together would be like, please tell us in your application. \n",
+      "After your introductory interview, we expect your interview process to take four to six weeks (but will depend on scheduling), and consist of four conversations that total about five hours of time. You should plan to also spend about four hours in total preparing for interviews.  See the hiring page at https://www.enveritas.org/jobs/data-scientist/ for details about each of these interviews.\n",
+      "\n",
+      "Introductory Interview (30 minutes; Google Meet; audio-only)\n",
+      "First Technical Interview (60-90 minutes; Google Meet video)\n",
+      "Second Technical Interview (60-90 minutes; Google Meet video)\n",
+      "Manager Interview (45 minutes; Google Meet video)\n",
+      "\n",
+      "How to Apply\n",
+      "Please apply here. Feel free to contact us at jobs@enveritas.org should you have any questions about the position, the interview process, or if you require any adjustments to ensure a fair and equitable application process. Questions about this opportunity or process will not reflect negatively on your application.\n",
+      "A few notes about our communications: We are not able to reply to messages sent to staff outside of either our application process or our jobs email address, as this is unfair to other candidates. Also, Enveritas has been made aware of fake job postings by individuals pretending to hire persons seeking employment. These individuals are looking to collect personal information about you for fraudulent purposes. All legitimate Enveritas job openings are posted on our Greenhouse board at https://boards.greenhouse.io/enveritas or Lever board at https://jobs.lever.co/Enveritas. All recruiting emails from Enveritas team members will come through Lever or from @enveritas.org. If you have any concerns about employment opportunities or contact from someone supposedly representing Enveritas, please reach out to us at jobs@enveritas.org\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Extract the job description content\n",
+    "job_title = soup.find(\"h1\", {\"class\": \"app-title\"})\n",
+    "\n",
+    "company_name = soup.find(\"span\",{\"class\": \"company-name\"})\n",
+    "\n",
+    "job_location = soup.find(\"div\", {\"class\": \"location\"})\n",
+    "\n",
+    "job_description = soup.find(\"div\", {\"id\": \"content\"})\n",
+    "\n",
+    "# Print the job description\n",
+    "if job_description:\n",
+    "    print(\"Job Title: \"+job_title.get_text()+\"\\n\"+\"Company: \"+company_name.get_text().strip().split(\"at \")[1]+\"\\n\"+\"Job Location: \"+job_location.get_text().strip()+\"\\n\"+\"Job Description: \\n\"+job_description.get_text().strip())\n",
+    "else:\n",
+    "    print(\"Job description not found.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "my_env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

openai.md

@@ -1,56 +1,56 @@
-# OpenAI APIs
-OpenAI API is a simple restful api. But the have **Python** and **Node.js** packages built on top of the api. We are using the Python package.
-
-## Installation and Setup
-* **Python:** 
-    * **Windows & MacOS:** Go to [Python3.11](https://www.python.org/downloads/release/python-3118/) and find the link in the page and follow the installer.
-    * **WSL2 & Ubuntu:** 
-        1. run ```sudo apt update && sudo apt upgrade```.
-        2. Check the Python version: ```python -V```.
-        3. If you need to update the python, run 
-        ```sudo apt upgrade python3```.
-* **Virtual Environment:** 
-    1. Create a venv: 
-    ```python -m venv path-to-venv```  
-    venv stands for Virtual Environment.
-    2. Activate the venv:  
-    **Linux & MacOS:** ```source path-to-venv/bin/activate```.
-    **Windows:** ```path-to-venv\Scripts\activate```.
-* **VS Code:** 
-    1. Install Python Extension:  
-    ```Extensions > search Python > Install microsoft.com Python extension```.  
-    2. Choose the Python interpreter from your venv:  
-    ```Settings > Command Palette > Python: Select interpreter > Enter interpreter path > Find > Path-to-env/bin/activate```.
-    3. Choose the kernel for notebooks.
-* **Set up API Key:** We use ```dotenv``` package and ```.env``` file. This way can work for all similar APIs.
-    1. In each venv install ```dotenv``` package by ```pip install dotenv```. 
-    2. In each project folder create ```.env``` file. In this file add the following line: ```OPENAI_API_KEY=key```.
-    3. At the top of the main file of your project, add the following two lines:  
-    ```from dotenv import load_dotenv```  
-    ```load_dotenv()```
-
-## Models
-* **API Structure:** The API interface is very simple. You just need a client to call any of the models. The following two lines import the needed class and create an instance of it:
-```
-from openai import OpenAI
-client = OpenAI()
-```
-* **Models List**
-    * **GPT-4 Turbo** accessed by handle ```gpt-4-turbo-preview``` or any of the following handles.
-        1. ```gpt-4-0125-preview``` Context Window: 128 k, Training Data: Up to Dec 2023.
-        2. ```gpt-4-1106-preview``` Context Window: 128 k, Training Data: Up to Apr 2023.
-        3. ```gpt-4-1106-vision-preview``` Context Window: 128 k, Training Data: Up to Apr 2023. Understands images.
-    * **GPT-3.5 Turbo** accessed by handle ```gpt-3.5-turbo```
-        1. ```gpt-3.5-turbo-0125``` Context Window: 32 k, Training Data: Up to Sep 2021.
-    * **DALL-E**
-        1. ```dall-e-3``` released Nov 2023
-        2. ```dall-e-2``` released Nov 2022
-    * **TTS** Text to natural sounding spoken text.
-        1. ```tts-1```
-        2. ```tts-1-hd```
-    * **Whisper** general-purpose speech recognition model.
-    * **Embeddings** numerical representation of text for semantic similarity and semantic search.
-        1. ```text-embeddings-3-large``` Output dimension: 3072
-        2. ```text-embeddings-3-small``` Output dimension: 1536
-        3. ```text-embeddings-ada-002``` Output dimension: 1536 (oldest)
-
+# OpenAI APIs
+OpenAI API is a simple restful api. But the have **Python** and **Node.js** packages built on top of the api. We are using the Python package.
+
+## Installation and Setup
+* **Python:** 
+    * **Windows & MacOS:** Go to [Python3.11](https://www.python.org/downloads/release/python-3118/) and find the link in the page and follow the installer.
+    * **WSL2 & Ubuntu:** 
+        1. run ```sudo apt update && sudo apt upgrade```.
+        2. Check the Python version: ```python -V```.
+        3. If you need to update the python, run 
+        ```sudo apt upgrade python3```.
+* **Virtual Environment:** 
+    1. Create a venv: 
+    ```python -m venv path-to-venv```  
+    venv stands for Virtual Environment.
+    2. Activate the venv:  
+    **Linux & MacOS:** ```source path-to-venv/bin/activate```.
+    **Windows:** ```path-to-venv\Scripts\activate```.
+* **VS Code:** 
+    1. Install Python Extension:  
+    ```Extensions > search Python > Install microsoft.com Python extension```.  
+    2. Choose the Python interpreter from your venv:  
+    ```Settings > Command Palette > Python: Select interpreter > Enter interpreter path > Find > Path-to-env/bin/activate```.
+    3. Choose the kernel for notebooks.
+* **Set up API Key:** We use ```dotenv``` package and ```.env``` file. This way can work for all similar APIs.
+    1. In each venv install ```dotenv``` package by ```pip install dotenv```. 
+    2. In each project folder create ```.env``` file. In this file add the following line: ```OPENAI_API_KEY=key```.
+    3. At the top of the main file of your project, add the following two lines:  
+    ```from dotenv import load_dotenv```  
+    ```load_dotenv()```
+
+## Models
+* **API Structure:** The API interface is very simple. You just need a client to call any of the models. The following two lines import the needed class and create an instance of it:
+```
+from openai import OpenAI
+client = OpenAI()
+```
+* **Models List**
+    * **GPT-4 Turbo** accessed by handle ```gpt-4-turbo-preview``` or any of the following handles.
+        1. ```gpt-4-0125-preview``` Context Window: 128 k, Training Data: Up to Dec 2023.
+        2. ```gpt-4-1106-preview``` Context Window: 128 k, Training Data: Up to Apr 2023.
+        3. ```gpt-4-1106-vision-preview``` Context Window: 128 k, Training Data: Up to Apr 2023. Understands images.
+    * **GPT-3.5 Turbo** accessed by handle ```gpt-3.5-turbo```
+        1. ```gpt-3.5-turbo-0125``` Context Window: 32 k, Training Data: Up to Sep 2021.
+    * **DALL-E**
+        1. ```dall-e-3``` released Nov 2023
+        2. ```dall-e-2``` released Nov 2022
+    * **TTS** Text to natural sounding spoken text.
+        1. ```tts-1```
+        2. ```tts-1-hd```
+    * **Whisper** general-purpose speech recognition model.
+    * **Embeddings** numerical representation of text for semantic similarity and semantic search.
+        1. ```text-embeddings-3-large``` Output dimension: 3072
+        2. ```text-embeddings-3-small``` Output dimension: 1536
+        3. ```text-embeddings-ada-002``` Output dimension: 1536 (oldest)
+

rag.md

@@ -1,49 +1,49 @@
-# RAG: Retrieval-Augmented Generation
-Paper: [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401v4.pdf)  
-Code: [https://python.langchain.com/docs/use_cases/question_answering/quickstart](https://python.langchain.com/docs/use_cases/question_answering/quickstart)  
-Similarity Search: [https://arxiv.org/pdf/2403.05440.pdf](https://arxiv.org/pdf/2403.05440.pdf)  
-Prompt Hub: [https://smith.langchain.com/hub](https://smith.langchain.com/hub)
-
-## Premise
-LLMs store factual knowledge in their parameters. But accessing and manipulating this knowledge in a precise way is not easy. Instead, that specific knowledge can be accessed through the weights and similarity and then added to the prompt of another model for answer generation:
-
-![RAG Architecture](readme_data/rag1.png)
-The original paper considers training end-to-end retriever and generator models in one pipeline.   
-Models like GPT3.5 and GPT4 don't need that training piece, if their are used with the open AI embedding models.
-
-**Where RAG Is Used?** 
-* Mitigate hallucination generated by LLMs
-    * Hallucinations are factually incorrect information generated by an LLM in response to an instruction or question from a user. 
-    * Hallucinations are very hard to capture and need other methodologies to catch them.
-    * Even with RAG the probability of hallucination is not zero.
-* Allow LLMs the consume the data that is not part of their training in their inference. 
-    * LLMs are pre-training on huge amounts of data from public sources. 
-    * Proprietary data is not available to any general pre-trained LLM.
-
-**How RAG Works**  
-1. One can vectorize the semantics of a piece of text using specialized LLMs, called embedding models. 
-2. A collection of text can be vectorized to be used for answering the incoming questions. 
-3. A question is embedded using the same embedding model, and similar documents from a vector database is retrieved using a similarity search algorithm, like cosine similarity.
-4. Found documents with the question are passed to generator LLM to generate and answer.
-
-
-## RAG Components
-Here are different components present in a RAG pipeline:
-1. **Embedding Model:** Vectorization model which for each string outputs a vector of fixed length. 
-    * The length is the dimension of latent space for this model.
-3. **Vector DB:** Specialized database for saving pairs of (text,embeddings). Each of these pairs are called a document. Usually we put related documents in one collection. This makes the similarity search easier.
-2. **Similarity Metric:** Given two document pairs $(t_1,e_1)$ and $(t_2,e_2), the metric calculates the similarity of $t1$ and $t_2$ by performing some geometric calculation on their respective embeddings. 
-    * **Cosine Similarity:** Calculates the cosine of the angle between embedding1 and embedding2:
-    $$\cos(\theta)=\frac{e_1 \cdot e_2}{||e_1||\;||e_2||}.$$
-    * **Inner Product:** Calculates the inner product of $e_1$ and $e_2$:
-    $$e1\cdot e_2.$$
-    * **Distance:** Calculates the distance of $e_1$ from $e_2$ using $L_p$ norms:
-    $$||e_1-e_2||_p.$$
-4. **Generator Model:** Generates the final answer based on the question and found similar text in the database that may contain the answer.
-
-## Cosine Similarity Problems
-* The motivation for using cosine similarity is that the norm of the learned embedding-vectors is not as important as the directional alignment between the embedding-vectors.
-* But cosine similarity "work better but sometimes also worse than the unnormalized dot-product between embedded vectors in practice."
-* The paper derives "analytically how cosine-similarity can yield arbitrary and therefore meaningless ‘similarities.’" 
-* To do this, they "study embeddings derived from regularized linear models, where closed-form solutions facilitate analytical insights."
+# RAG: Retrieval-Augmented Generation
+Paper: [Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks](https://arxiv.org/pdf/2005.11401v4.pdf)  
+Code: [https://python.langchain.com/docs/use_cases/question_answering/quickstart](https://python.langchain.com/docs/use_cases/question_answering/quickstart)  
+Similarity Search: [https://arxiv.org/pdf/2403.05440.pdf](https://arxiv.org/pdf/2403.05440.pdf)  
+Prompt Hub: [https://smith.langchain.com/hub](https://smith.langchain.com/hub)
+
+## Premise
+LLMs store factual knowledge in their parameters. But accessing and manipulating this knowledge in a precise way is not easy. Instead, that specific knowledge can be accessed through the weights and similarity and then added to the prompt of another model for answer generation:
+
+![RAG Architecture](readme_data/rag1.png)
+The original paper considers training end-to-end retriever and generator models in one pipeline.   
+Models like GPT3.5 and GPT4 don't need that training piece, if their are used with the open AI embedding models.
+
+**Where RAG Is Used?** 
+* Mitigate hallucination generated by LLMs
+    * Hallucinations are factually incorrect information generated by an LLM in response to an instruction or question from a user. 
+    * Hallucinations are very hard to capture and need other methodologies to catch them.
+    * Even with RAG the probability of hallucination is not zero.
+* Allow LLMs the consume the data that is not part of their training in their inference. 
+    * LLMs are pre-training on huge amounts of data from public sources. 
+    * Proprietary data is not available to any general pre-trained LLM.
+
+**How RAG Works**  
+1. One can vectorize the semantics of a piece of text using specialized LLMs, called embedding models. 
+2. A collection of text can be vectorized to be used for answering the incoming questions. 
+3. A question is embedded using the same embedding model, and similar documents from a vector database is retrieved using a similarity search algorithm, like cosine similarity.
+4. Found documents with the question are passed to generator LLM to generate and answer.
+
+
+## RAG Components
+Here are different components present in a RAG pipeline:
+1. **Embedding Model:** Vectorization model which for each string outputs a vector of fixed length. 
+    * The length is the dimension of latent space for this model.
+3. **Vector DB:** Specialized database for saving pairs of (text,embeddings). Each of these pairs are called a document. Usually we put related documents in one collection. This makes the similarity search easier.
+2. **Similarity Metric:** Given two document pairs $(t_1,e_1)$ and $(t_2,e_2), the metric calculates the similarity of $t1$ and $t_2$ by performing some geometric calculation on their respective embeddings. 
+    * **Cosine Similarity:** Calculates the cosine of the angle between embedding1 and embedding2:
+    $$\cos(\theta)=\frac{e_1 \cdot e_2}{||e_1||\;||e_2||}.$$
+    * **Inner Product:** Calculates the inner product of $e_1$ and $e_2$:
+    $$e1\cdot e_2.$$
+    * **Distance:** Calculates the distance of $e_1$ from $e_2$ using $L_p$ norms:
+    $$||e_1-e_2||_p.$$
+4. **Generator Model:** Generates the final answer based on the question and found similar text in the database that may contain the answer.
+
+## Cosine Similarity Problems
+* The motivation for using cosine similarity is that the norm of the learned embedding-vectors is not as important as the directional alignment between the embedding-vectors.
+* But cosine similarity "work better but sometimes also worse than the unnormalized dot-product between embedded vectors in practice."
+* The paper derives "analytically how cosine-similarity can yield arbitrary and therefore meaningless ‘similarities.’" 
+* To do this, they "study embeddings derived from regularized linear models, where closed-form solutions facilitate analytical insights."
 * "The underlying reason is not cosine similarity itself, but the fact that the learned embeddings have a degree of freedom that can render arbitrary cosine-similarities even though their (unnormalized) dot-products are well-defined and unique."

requirements.txt

@@ -1,17 +1,17 @@
-langchain==0.1.8
-streamlit==1.31.1
-langchain-openai==0.0.8
-python-dotenv==1.0.1
-openai==1.12.0
-beautifulsoup4==4.12.3
-langchain_text_splitters==0.0.1
-chromadb==0.4.24
-langchainhub==0.1.15
-langchain-community==0.0.37
-langchain-core==0.1.52
-sentence-transformers
-selenium==4.20.0
-openpyxl
-bs4
-unstructured==0.13.7
+langchain==0.1.20
+streamlit==1.31.1
+langchain-openai==0.0.8
+python-dotenv==1.0.1
+openai==1.12.0
+beautifulsoup4==4.12.3
+langchain_text_splitters==0.0.1
+chromadb==0.4.3
+langchainhub==0.1.15
+langchain-community==0.0.38
+langchain-core==0.1.52
+sentence-transformers
+selenium==4.20.0
+openpyxl
+bs4
+unstructured==0.13.7
 openpyxl