Merge pull request #1 from andreped/linting

Added listing; refactored code; change environ-import order

.github/workflows/linting.yml

@@ -0,0 +1,26 @@
+name: Linting
+
+on:
+  push:
+    branches:
+      - '*'
+  pull_request:
+    branches:
+      - '*'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      
+      - name: Install lint dependencies
+        run: pip install wheel setuptools black==22.3.0 isort==5.10.1 flake8==4.0.1
+
+      - name: Lint the code
+        run: sh shell/lint.sh

README.md

@@ -23,6 +23,7 @@ app_file: knowledge_gpt/main.py
 | - | - |
 | **HF Deploy** | [![Deploy](https://github.com/andreped/referencebot/workflows/Deploy/badge.svg)](https://github.com/andreped/referencebot/actions) |
 | **File size check** | [![Filesize](https://github.com/andreped/referencebot/workflows/Check%20file%20size/badge.svg)](https://github.com/andreped/referencebot/actions) |
+| **Formatting check** | [![Filesize](https://github.com/andreped/ReferenceBot/workflows/Linting/badge.svg)](https://github.com/andreped/ReferenceBot/actions) |
 
 ## [Installation](https://github.com/andreped/referencebot#installation)
 

knowledge_gpt/components/sidebar.py

@@ -1,5 +1,4 @@
 import streamlit as st
-
 from dotenv import load_dotenv
 
 load_dotenv()

knowledge_gpt/core/caching.py

@@ -1,9 +1,9 @@
 import streamlit as st
 from streamlit.runtime.caching.hashing import HashFuncsDict
 
-import knowledge_gpt.core.parsing as parsing
 import knowledge_gpt.core.chunking as chunking
 import knowledge_gpt.core.embedding as embedding
+import knowledge_gpt.core.parsing as parsing
 from knowledge_gpt.core.parsing import File
 
 
@@ -18,16 +18,10 @@ def bootstrap_caching():
 
     # Get all substypes of File from module
     file_subtypes = [
-        cls
-        for cls in vars(parsing).values()
-        if isinstance(cls, type) and issubclass(cls, File) and cls != File
+        cls for cls in vars(parsing).values() if isinstance(cls, type) and issubclass(cls, File) and cls != File
     ]
     file_hash_funcs: HashFuncsDict = {cls: file_hash_func for cls in file_subtypes}
 
     parsing.read_file = st.cache_data(show_spinner=False)(parsing.read_file)
-    chunking.chunk_file = st.cache_data(show_spinner=False, hash_funcs=file_hash_funcs)(
-        chunking.chunk_file
-    )
-    embedding.embed_files = st.cache_data(
-        show_spinner=False, hash_funcs=file_hash_funcs
-    )(embedding.embed_files)
+    chunking.chunk_file = st.cache_data(show_spinner=False, hash_funcs=file_hash_funcs)(chunking.chunk_file)
+    embedding.embed_files = st.cache_data(show_spinner=False, hash_funcs=file_hash_funcs)(embedding.embed_files)

knowledge_gpt/core/chunking.py

@@ -1,11 +1,10 @@
 from langchain.docstore.document import Document
 from langchain.text_splitter import RecursiveCharacterTextSplitter
+
 from knowledge_gpt.core.parsing import File
 
 
-def chunk_file(
-    file: File, chunk_size: int, chunk_overlap: int = 0, model_name="gpt-3.5-turbo"
-) -> File:
+def chunk_file(file: File, chunk_size: int, chunk_overlap: int = 0, model_name="gpt-3.5-turbo") -> File:
     """Chunks each document in a file into smaller documents
     according to the specified chunk size and overlap
     where the size is determined by the number of tokens for the specified model.

knowledge_gpt/core/debug.py

@@ -1,10 +1,13 @@
-from langchain.vectorstores import VectorStore
-from typing import Iterable, List, Any
+from typing import Any
+from typing import Iterable
+from typing import List
+from typing import Optional
+
+from langchain.chat_models.fake import FakeListChatModel
 from langchain.docstore.document import Document
 from langchain.embeddings.base import Embeddings
 from langchain.embeddings.fake import FakeEmbeddings as FakeEmbeddingsBase
-from langchain.chat_models.fake import FakeListChatModel
-from typing import Optional
+from langchain.vectorstores import VectorStore
 
 
 class FakeChatModel(FakeListChatModel):
@@ -24,9 +27,7 @@ class FakeVectorStore(VectorStore):
     def __init__(self, texts: List[str]):
         self.texts: List[str] = texts
 
-    def add_texts(
-        self, texts: Iterable[str], metadatas: List[dict] | None = None, **kwargs: Any
-    ) -> List[str]:
+    def add_texts(self, texts: Iterable[str], metadatas: List[dict] | None = None, **kwargs: Any) -> List[str]:
         self.texts.extend(texts)
         return self.texts
 
@@ -40,10 +41,5 @@ class FakeVectorStore(VectorStore):
     ) -> "FakeVectorStore":
         return cls(texts=list(texts))
 
-    def similarity_search(
-        self, query: str, k: int = 4, **kwargs: Any
-    ) -> List[Document]:
-        return [
-            Document(page_content=text, metadata={"source": f"{i+1}-{1}"})
-            for i, text in enumerate(self.texts)
-        ]
+    def similarity_search(self, query: str, k: int = 4, **kwargs: Any) -> List[Document]:
+        return [Document(page_content=text, metadata={"source": f"{i+1}-{1}"}) for i, text in enumerate(self.texts)]

knowledge_gpt/core/embedding.py

@@ -1,11 +1,15 @@
-from langchain.vectorstores import VectorStore
-from knowledge_gpt.core.parsing import File
-from langchain.vectorstores.faiss import FAISS
+from typing import List
+from typing import Type
+
+from langchain.docstore.document import Document
 from langchain.embeddings import OpenAIEmbeddings
 from langchain.embeddings.base import Embeddings
-from typing import List, Type
-from langchain.docstore.document import Document
-from knowledge_gpt.core.debug import FakeVectorStore, FakeEmbeddings
+from langchain.vectorstores import VectorStore
+from langchain.vectorstores.faiss import FAISS
+
+from knowledge_gpt.core.debug import FakeEmbeddings
+from knowledge_gpt.core.debug import FakeVectorStore
+from knowledge_gpt.core.parsing import File
 
 
 class FolderIndex:
@@ -30,9 +34,7 @@ class FolderIndex:
         return all_texts
 
     @classmethod
-    def from_files(
-        cls, files: List[File], embeddings: Embeddings, vector_store: Type[VectorStore]
-    ) -> "FolderIndex":
+    def from_files(cls, files: List[File], embeddings: Embeddings, vector_store: Type[VectorStore]) -> "FolderIndex":
         """Creates an index from files."""
 
         all_docs = cls._combine_files(files)
@@ -45,9 +47,7 @@ class FolderIndex:
         return cls(files=files, index=index)
 
 
-def embed_files(
-    files: List[File], embedding: str, vector_store: str, **kwargs
-) -> FolderIndex:
+def embed_files(files: List[File], embedding: str, vector_store: str, **kwargs) -> FolderIndex:
     """Embeds a collection of files and stores them in a FolderIndex."""
 
     supported_embeddings: dict[str, Type[Embeddings]] = {
@@ -69,6 +69,4 @@ def embed_files(
     else:
         raise NotImplementedError(f"Vector store {vector_store} not supported.")
 
-    return FolderIndex.from_files(
-        files=files, embeddings=_embeddings, vector_store=_vector_store
-    )
+    return FolderIndex.from_files(files=files, embeddings=_embeddings, vector_store=_vector_store)

knowledge_gpt/core/parsing.py

@@ -1,14 +1,16 @@
-from io import BytesIO
-from typing import List, Any, Optional
 import re
+from abc import ABC
+from abc import abstractmethod
+from copy import deepcopy
+from hashlib import md5
+from io import BytesIO
+from typing import Any
+from typing import List
+from typing import Optional
 
 import docx2txt
-from langchain.docstore.document import Document
 import fitz
-from hashlib import md5
-
-from abc import abstractmethod, ABC
-from copy import deepcopy
+from langchain.docstore.document import Document
 
 
 class File(ABC):
@@ -32,10 +34,7 @@ class File(ABC):
         """Creates a File from a BytesIO object"""
 
     def __repr__(self) -> str:
-        return (
-            f"File(name={self.name}, id={self.id},"
-            " metadata={self.metadata}, docs={self.docs})"
-        )
+        return f"File(name={self.name}, id={self.id}," " metadata={self.metadata}, docs={self.docs})"
 
     def __str__(self) -> str:
         return f"File(name={self.name}, id={self.id}, metadata={self.metadata})"

knowledge_gpt/core/prompts.py

@@ -26,6 +26,4 @@ QUESTION: {question}
 =========
 FINAL ANSWER:"""
 
-STUFF_PROMPT = PromptTemplate(
-    template=template, input_variables=["summaries", "question"]
-)
+STUFF_PROMPT = PromptTemplate(template=template, input_variables=["summaries", "question"])

knowledge_gpt/core/qa.py

@@ -1,10 +1,12 @@
 from typing import List
+
 from langchain.chains.qa_with_sources import load_qa_with_sources_chain
-from knowledge_gpt.core.prompts import STUFF_PROMPT
+from langchain.chat_models.base import BaseChatModel
 from langchain.docstore.document import Document
-from knowledge_gpt.core.embedding import FolderIndex
 from pydantic import BaseModel
-from langchain.chat_models.base import BaseChatModel
+
+from knowledge_gpt.core.embedding import FolderIndex
+from knowledge_gpt.core.prompts import STUFF_PROMPT
 
 
 class AnswerWithSources(BaseModel):
@@ -39,9 +41,7 @@ def query_folder(
     )
 
     relevant_docs = folder_index.index.similarity_search(query, k=5)
-    result = chain(
-        {"input_documents": relevant_docs, "question": query}, return_only_outputs=True
-    )
+    result = chain({"input_documents": relevant_docs, "question": query}, return_only_outputs=True)
     sources = relevant_docs
 
     if not return_all:

knowledge_gpt/core/utils.py

@@ -1,15 +1,14 @@
 from typing import List
+
 from langchain.chains.combine_documents.stuff import StuffDocumentsChain
+from langchain.chat_models import ChatOpenAI
+from langchain.chat_models.base import BaseChatModel
 from langchain.docstore.document import Document
 
-from langchain.chat_models import ChatOpenAI
 from knowledge_gpt.core.debug import FakeChatModel
-from langchain.chat_models.base import BaseChatModel
 
 
-def pop_docs_upto_limit(
-    query: str, chain: StuffDocumentsChain, docs: List[Document], max_len: int
-) -> List[Document]:
+def pop_docs_upto_limit(query: str, chain: StuffDocumentsChain, docs: List[Document], max_len: int) -> List[Document]:
     """Pops documents from a list until the final prompt length is less
     than the max length."""
 

knowledge_gpt/main.py

@@ -1,31 +1,27 @@
 import os
-os.environ["OPENAI_API_TYPE"] = "azure"  # configure API to Azure OpenAI
 
 import streamlit as st
-st.set_page_config(page_title="ReferenceBot", page_icon="📖", layout="wide")
-
-# add all secrets into environmental variables
-if os.path.exists(os.path.dirname(os.path.abspath(__file__)) + "/../.streamlit/secrets.toml"):  # to avoid redundant print by calling st.secrets
-    for key, value in st.secrets.items():
-        os.environ[key] = value
+from langchain.chat_models import AzureChatOpenAI
 
 from knowledge_gpt.components.sidebar import sidebar
-
-from knowledge_gpt.ui import (
-    wrap_doc_in_html,
-    is_query_valid,
-    is_file_valid,
-    display_file_read_error,
-)
-
 from knowledge_gpt.core.caching import bootstrap_caching
-
-from knowledge_gpt.core.parsing import read_file
 from knowledge_gpt.core.chunking import chunk_file
 from knowledge_gpt.core.embedding import embed_files
+from knowledge_gpt.core.parsing import read_file
 from knowledge_gpt.core.qa import query_folder
+from knowledge_gpt.ui import display_file_read_error
+from knowledge_gpt.ui import is_file_valid
+from knowledge_gpt.ui import is_query_valid
+from knowledge_gpt.ui import wrap_doc_in_html
 
-from langchain.chat_models import AzureChatOpenAI
+st.set_page_config(page_title="ReferenceBot", page_icon="📖", layout="wide")
+
+# add all secrets into environmental variables
+if os.path.exists(
+    os.path.dirname(os.path.abspath(__file__)) + "/../.streamlit/secrets.toml"
+):  # to avoid redundant print by calling st.secrets
+    for key, value in st.secrets.items():
+        os.environ[key] = value
 
 
 def main():
@@ -78,7 +74,7 @@ def main():
             openai_api_key=os.environ["OPENAI_API_KEY"],
             openai_api_base=os.environ["OPENAI_API_BASE"],
             openai_api_type="azure",
-            chunk_size = 1,
+            chunk_size=1,
         )
 
     with st.form(key="qa_form"):
@@ -106,7 +102,7 @@ def main():
                 openai_api_type="azure",
                 temperature=0,
             )
-        
+
         with st.spinner("Querying folder to get result..."):
             result = query_folder(
                 folder_index=folder_index,

knowledge_gpt/ui.py

@@ -1,9 +1,11 @@
 from typing import List
+from typing import NoReturn
+
 import streamlit as st
 from langchain.docstore.document import Document
-from knowledge_gpt.core.parsing import File
 from streamlit.logger import get_logger
-from typing import NoReturn
+
+from knowledge_gpt.core.parsing import File
 
 logger = get_logger(__name__)
 
@@ -25,10 +27,7 @@ def is_query_valid(query: str) -> bool:
 
 
 def is_file_valid(file: File) -> bool:
-    if (
-        len(file.docs) == 0
-        or "".join([doc.page_content for doc in file.docs]).strip() == ""
-    ):
+    if len(file.docs) == 0 or "".join([doc.page_content for doc in file.docs]).strip() == "":
         st.error("Cannot read document! Make sure the document has selectable text")
         logger.error("Cannot read document")
         return False

setup.cfg

@@ -0,0 +1,14 @@
+[metadata]
+description-file = README.md
+
+[isort]
+force_single_line=True
+known_first_party=aeropath
+line_length=160
+profile=black
+
+[flake8]
+# imported but unused in __init__.py, that's ok.
+per-file-ignores=*__init__.py:F401
+ignore=E203,W503,W605,F632,E266,E731,E712,E741
+max-line-length=120

shell/format.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+isort --sl knowledge_gpt/
+black --line-length 120 knowledge_gpt/
+flake8 knowledge_gpt/

shell/lint.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+isort --check --sl -c knowledge_gpt/
+if ! [ $? -eq 0 ]
+then
+  echo "Please run \"sh shell/format.sh\" to format the code."
+  exit 1
+fi
+echo "no issues with isort"
+flake8 knowledge_gpt/
+if ! [ $? -eq 0 ]
+then
+  echo "Please fix the code style issue."
+  exit 1
+fi
+echo "no issues with flake8"
+black --check --line-length 120 knowledge_gpt/
+if ! [ $? -eq 0 ]
+then
+  echo "Please run \"sh shell/format.sh\" to format the code."
+    exit 1
+fi
+echo "no issues with black"
+echo "linting success!"