cleanup

.gitignore

@@ -3,4 +3,5 @@
 downloaded/*
 __pycache__/*
 launch.json
-.DS_Store
+.DS_Store
+devcode.py

app.py

@@ -1,20 +1,13 @@
+# chat-pykg/app.py
 import datetime
 import os
 import gradio as gr
-from abc import ABC
-from typing import List, Optional, Any
-import asyncio
-import langchain
 import chromadb
 from chromadb.config import Settings
 # logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 # logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
-from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings
 from langchain.vectorstores import Chroma
-from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter, PythonCodeTextSplitter
-from langchain.document_loaders import TextLoader
 from langchain.docstore.document import Document
-from langchain.embeddings.base import Embeddings
 import shutil
 import random, string
 from chain import get_new_chain1

chain.py

@@ -1,10 +1,6 @@
-import json
-import os
-import pathlib
-from typing import Dict, List, Tuple
+# chat-pykg/chain.py
+
 from langchain.chains.base import Chain
-import os
-import langchain
 # logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 # logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))
 from langchain import HuggingFaceHub
@@ -17,45 +13,12 @@ from langchain.callbacks.base import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 from langchain.chains.conversational_retrieval.prompts import CONDENSE_QUESTION_PROMPT, QA_PROMPT
 
-from abc import ABC
-from typing import List, Optional, Any
-
-from langchain.vectorstores import Chroma
-
 def get_new_chain1(vectorstore, model_selector, k_textbox) -> Chain:
     max_tokens_dict = {'gpt-4': 2000, 'gpt-3.5-turbo': 1000}
 
-    # These templates aren't used for the moment.
-    _eg_template = """## Example:
-
-    Chat History:
-    {chat_history}
-    Follow Up Input: {question}
-    Standalone question: {answer}"""
-    _prefix = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question. You should assume that the question is related to PyCBC."""
-    _suffix = """## Example:
-
-    Chat History:
-    {chat_history}
-    Follow Up Input: {question}
-    Standalone question:"""
-
-    template = """You are an AI assistant for various open source libraries.
-    You are given the following extracted parts of a long document and a question. Provide a conversational answer to the question.
-    You should only use hyperlinks that are explicitly listed as a source in the context. Do NOT make up a hyperlink that is not listed.
-    If you don't know the answer, just say "Hmm, I'm not sure." Don't try to make up an answer.
-    If the question is not about the package documentation, politely inform them that you are tuned to only answer questions about the package documentationz.
-    Question: {question}
-    =========
-    {context}
-    =========
-    Answer in Markdown:"""
-
-    # Construct a ChatVectorDBChain with a streaming llm for combine docs
-    # and a separate, non-streaming llm for question generation
     if model_selector in ['gpt-4', 'gpt-3.5-turbo']:
         llm = ChatOpenAI(client = None, temperature=0.7, model_name=model_selector)
-        doc_chain_llm = ChatOpenAI(client = None, streaming=True, callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]), verbose=True, temperature=0.7, model_name=model_selector, max_tokens=1000)
+        doc_chain_llm = ChatOpenAI(client = None, streaming=True, callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]), verbose=True, temperature=0.7, model_name=model_selector, max_tokens=max_tokens_dict[model_selector])
     if model_selector == 'other':
         llm = HuggingFaceHub(repo_id="chavinlo/gpt4-x-alpaca")#, model_kwargs={"temperature":0, "max_length":64})
         doc_chain_llm = HuggingFaceHub(repo_id="chavinlo/gpt4-x-alpaca")

ingest.py

@@ -1,22 +1,14 @@
-import pickle
+# chat-pykg/ingest.py
 import tempfile
 from langchain.document_loaders import SitemapLoader, ReadTheDocsLoader, TextLoader
 from langchain.embeddings import OpenAIEmbeddings, HuggingFaceEmbeddings
-from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter, PythonCodeTextSplitter, MarkdownTextSplitter
+from langchain.text_splitter import RecursiveCharacterTextSplitter, PythonCodeTextSplitter, MarkdownTextSplitter
 from langchain.vectorstores.faiss import FAISS
-import chromadb
 import os
 from langchain.vectorstores import Chroma
 import shutil
 from pathlib import Path
 import subprocess
-import tarfile
-# import chromadb
-from abc import ABC
-from typing import List, Optional, Any
-from langchain.docstore.document import Document
-from langchain.embeddings.base import Embeddings
-from chromadb.config import Settings
 
 # class CachedChroma(Chroma, ABC):
 #     """