version 2 commit

Code/app.py

@@ -0,0 +1,44 @@
+import random
+
+# Sample dataset and transition matrix
+dataset = [
+    "Once upon a time",
+    "there was a cat",
+    "The cat loved to",
+    "explore the forest",
+    "One day, it found",
+    "a hidden treasure",
+    "The treasure was",
+    "guarded by a dragon",
+]
+
+transition_matrix = {
+    "Once upon a time": {"there was a cat": 1.0},
+    "there was a cat": {"The cat loved to": 1.0},
+    "The cat loved to": {"explore the forest": 1.0},
+    "explore the forest": {"One day, it found": 1.0},
+    "One day, it found": {"a hidden treasure": 1.0},
+    "a hidden treasure": {"The treasure was": 1.0},
+    "The treasure was": {"guarded by a dragon": 1.0},
+    "guarded by a dragon": {"The end": 1.0},
+}
+
+# Generate a story
+def generate_story(transition_matrix, initial_state, length=5):
+    current_state = initial_state
+    story = [current_state]
+
+    for _ in range(length):
+        next_state_options = transition_matrix.get(current_state, {})
+        if not next_state_options:
+            break
+        next_state = random.choices(list(next_state_options.keys()), list(next_state_options.values()))[0]
+        story.append(next_state)
+        current_state = next_state
+
+    return " ".join(story)
+
+# Generate a story starting from a specific state
+initial_state = "Once upon a time"
+generated_story = generate_story(transition_matrix, initial_state)
+print(generated_story)

app.py

@@ -0,0 +1,73 @@
+import streamlit as st
+import random
+import time
+import git
+import os
+from util import clone_repo
+from util import generate_assistant_response
+from google.api_core.exceptions import InternalServerError
+
+flag = False
+repo = st.text_input(label="Paste the .git link of repository.")
+btn = st.button(label="Submit")
+if btn and not flag:
+    clone_repo(repo=repo)
+    # if os.path.exists("githubCode") and os.path.isdir("githubCode"):
+    #     print("File already exists!!")
+    #     pass
+    # else:
+    #     git.Repo.clone_from(repo,"githubCode")
+    flag = True
+
+# def response_generator():
+#     response = random.choice(
+#         [
+#             "Hello there! How can I assist you today?",
+#             "Hi, human! Is there anything I can help you with?",
+#             "Do you need help?",
+#         ]
+#     )
+#     for word in response.split():
+#         yield word + " "
+#         time.sleep(0.05)
+
+
+st.title("Simple chat")
+
+# Initialize chat history
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+# Display chat messages from history on app rerun
+for message in st.session_state.messages:
+    with st.chat_message(message["role"]):
+        st.markdown(message["content"])
+
+# Accept user input
+if prompt := st.chat_input("What's your question ?"):
+    # Add user message to chat history
+    st.session_state.messages.append({"role": "user", "content": prompt})
+    # Display user message in chat message container
+    with st.chat_message("user"):
+        st.markdown(prompt)
+
+    # Display assistant response in chat message container
+    with st.chat_message("assistant"):
+
+        try:
+            response = generate_assistant_response(prompt)
+        except InternalServerError as err:
+            retry_attempts = 0
+            MAX_RETRIES = 2
+            if retry_attempts < MAX_RETRIES:
+                retry_attempts += 1
+                time.sleep(2)
+                response = generate_assistant_response(prompt)
+            else:
+                # If retries are exhausted, log the error for further investigation
+                response = "500 An internal error has occurred. Please retry or report in https://developers.generativeai.google/guide/troubleshooting"
+
+        print(response)
+        st.markdown(response)
+    # Add assistant response to chat history
+    st.session_state.messages.append({"role": "assistant", "content": response})

requirements.txt

@@ -0,0 +1,5 @@
+langchain
+langchain_community
+google_generativeai
+sentence_transformers
+chromadb

util.py

@@ -0,0 +1,115 @@
+import os
+import streamlit as st
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import Chroma
+# from langchain.llms.huggingface_pipeline import HuggingFacePipeline
+# from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, BitsAndBytesConfig
+from langchain.chains.question_answering import load_qa_chain
+from langchain.prompts import PromptTemplate
+
+# import os
+import google.generativeai as genai
+
+import git  # pip install gitpython
+
+os.environ['GOOGLE_API_KEY'] = "AIzaSyCS1bV6_bfizb1tAcQEB9BvCTtqCCLlGFo"
+genai.configure(api_key = os.environ['GOOGLE_API_KEY'])
+
+# quantization_config = BitsAndBytesConfig(
+# load_in_4bit=True,
+# bnb_4bit_compute_dtype=torch.bfloat16
+# )
+
+
+model_kwargs = {'device': 'cpu'}
+embeddings = HuggingFaceEmbeddings(model_kwargs=model_kwargs)
+
+# tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2")
+# model = AutoModelForCausalLM.from_pretrained("mistralai/Mistral-7B-Instruct-v0.2", device_map='auto', quantization_config = quantization_config)
+
+# pipe = pipeline("text-generation", model=model, tokenizer=tokenizer, max_new_tokens = 1000)
+# llm = HuggingFacePipeline(pipeline=pipe)
+
+
+def clone_repo(repo):
+    if os.path.exists("githubCode") and os.path.isdir("githubCode"):
+        print("File already exists!!")
+        pass
+    else:
+        print("Cloning repo!!")
+        git.Repo.clone_from(repo,"githubCode")
+
+# git.Repo.clone_from("https://github.com/Divyansh3021/Github_code_assistant.git", "githubCode")
+
+llm = genai.GenerativeModel('gemini-pro')
+
+def get_folder_paths(directory = "githubCode"):
+    folder_paths = []
+    for root, dirs, files in os.walk(directory):
+        if '.git' in dirs:
+            # Skip the directory if a .git folder is found
+            dirs.remove('.git')
+        for dir_name in dirs:
+            folder_paths.append(os.path.join(root, dir_name))
+    return folder_paths
+
+directory_paths = get_folder_paths()
+directory_paths.append("Code")
+
+with open("Code.txt", "w", encoding='utf-8') as output:
+    for directory_path in directory_paths:
+        for filename in os.listdir(directory_path):
+            if filename.endswith((".py",".ipynb",".js", ".ts")):
+                filepath = os.path.join(directory_path, filename)
+                with open(filepath, "r", encoding='utf-8') as file:
+                    code = file.read()
+                    output.write(f"Filepath: {filepath}:\n\n")
+                    output.write(code + "\n\n")
+            elif filename.endswith((".txt")):
+                filepath = os.path.join(directory_path, filename)
+                with open(filepath, "r", encoding="utf-8") as file:
+                    code = file.read()
+                    output.write(f"Documentation list:\n\n")
+                    output.write(code + "\n\n")
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import TextLoader
+
+# for filename in os.listdir(directory_path):
+#     if filename.endswith(".txt"):  # Only process PD files
+#         file_path = os.path.join(directory_path, filename)
+loader = TextLoader("Code.txt", encoding="utf-8")
+pages = loader.load_and_split()
+
+# Split data into chunks
+text_splitter = RecursiveCharacterTextSplitter(
+chunk_size = 4000,
+chunk_overlap  = 200,
+length_function = len,
+add_start_index = True,
+)
+chunks = text_splitter.split_documents(pages)
+
+# Store data into database
+db=Chroma.from_documents(chunks,embedding=embeddings,persist_directory="test_index")
+db.persist()
+
+# Load the database
+vectordb = Chroma(persist_directory="test_index", embedding_function = embeddings)
+
+# Load the retriver
+retriever = vectordb.as_retriever()
+
+# Function to generate assistant's response using ask function
+def generate_assistant_response(question):
+    context = retriever.get_relevant_documents(question)
+    qna_prompt_template= f"""### [INST] Instruction: You will be provided with questions and context. Your task is to find the answers to the questions using the given data. If the data doesn't contain the answer to the question, then you must return 'Not enough information.'
+    Context: ```
+        {context}
+    ```
+        ### Question: {question} [/INST]"""
+    print("Context: ", context)
+    answer = llm.generate_content(qna_prompt_template).text
+    return answer
+
+# print(generate_assistant_response("Tell me about the instructor_embeddings function."))