initial commit

.github/workflows/tests-integration.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  push:
+    branches: 
+      - main
+      - feature/**
+      - issue/**
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.11]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Switch to Current Branch
+        run: git checkout ${{ env.BRANCH }}
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          pip install --no-cache-dir -r requirements.txt
+          cp config/.env.example config/.env
+      - name: Run unit tests
+        run: |
+          pytest -o "testpaths=tests" --noconftest

.gitignore

@@ -0,0 +1,68 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Environments
+.env
+.venv
+env/
+venv/
+
+# OS
+.DS_Store
+
+# IDE
+.vscode
+
+# Project
+wandb/
+indexes/
+*.out
+*.log
+
+# Data
+data/datasets/*
+!data/datasets/hf_repositories_urls.json
+!data/datasets/hf_repositories_urls_scraped.json
+
+# Local models
+qa_engine/local_models

Dockerfile.api

@@ -0,0 +1,19 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get -y install git python3.10 python3-pip
+
+COPY requirements.txt .
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+WORKDIR /hugging-face-qa-bot
+COPY config/api/ config/api/
+COPY api/ api/
+
+EXPOSE 8000
+
+ENTRYPOINT [ "python3", "-m", "api" ]

Dockerfile.bot

@@ -0,0 +1,17 @@
+FROM ubuntu:latest
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get -y update && \
+    apt-get -y upgrade && \
+    apt-get -y install git python3.10 python3-pip
+
+COPY requirements.txt .
+RUN pip install --upgrade pip && \
+    pip install --no-cache-dir -r requirements.txt
+
+WORKDIR /hugging-face-qa-bot
+COPY config/bot/ config/bot/
+COPY bot/ bot/
+
+ENTRYPOINT [ "python3", "-m", "bot" ]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

api/__main__.py

@@ -0,0 +1,39 @@
+import uvicorn
+from fastapi import FastAPI
+
+from qa_engine import logger, Config, QAEngine
+
+
+config = Config()
+app = FastAPI()
+qa_engine = QAEngine(
+    llm_model_id=config.question_answering_model_id,
+    embedding_model_id=config.embedding_model_id,
+    index_repo_id=config.index_repo_id,
+    prompt_template=config.prompt_template,
+    use_docs_for_context=config.use_docs_for_context,
+    num_relevant_docs=config.num_relevant_docs,
+    add_sources_to_response=config.add_sources_to_response,
+    use_messages_for_context=config.use_messages_in_context,
+    debug=config.debug
+)
+
+
+@app.get('/')
+def get_answer(question: str, messages_context: str = ''):
+    logger.info(
+        f'Received request with question: {question}' \
+        f'and context: {messages_context}'
+    )
+    response = qa_engine.get_response(
+        question=question,
+        messages_context=messages_context
+    )
+    return {
+        'answer': response.get_answer(),
+        'sources': response.get_sources_as_text()
+    }
+
+
+if __name__ == '__main__':
+    uvicorn.run(app, host='0.0.0.0', port=8000)

app.py

@@ -0,0 +1,33 @@
+import gradio as gr
+
+from qa_engine import logger, Config, QAEngine
+
+
+config = Config()
+model = QAEngine(
+    llm_model_id=config.question_answering_model_id,
+    embedding_model_id=config.embedding_model_id,
+    index_repo_id=config.index_repo_id,
+    prompt_template=config.prompt_template,
+    use_docs_for_context=config.use_docs_for_context,
+    add_sources_to_response=config.add_sources_to_response,
+    use_messages_for_context=config.use_messages_in_context,
+    debug=config.debug
+)
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot()
+    msg = gr.Textbox()
+    clear = gr.ClearButton([msg, chatbot])
+
+    def respond(message, chat_history):
+        context = ''.join(f'User: {msg} \nBot:{bot_msg}\n' for msg, bot_msg in chat_history)
+        logger.info(f'Context: {context}')
+        response = model.get_response(message, context)
+        bot_message = response.get_answer() + response.get_sources_as_text() + '\n'
+        chat_history.append((message, bot_message))
+        return '', chat_history
+
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+
+demo.launch(share=True)

benchmarker.py

@@ -0,0 +1,63 @@
+import gradio as gr
+from dotenv import load_dotenv
+from api.config import Config
+from api.logger import logger
+from api.question_answering import QAModel
+import time
+
+
+load_dotenv(dotenv_path='config/api/.env')
+
+config = Config()
+model = QAModel(
+    llm_model_id=config.question_answering_model_id,
+    embedding_model_id=config.embedding_model_id,
+    index_repo_id=config.index_repo_id,
+    prompt_template=config.prompt_template,
+    use_docs_for_context=config.use_docs_for_context,
+    add_sources_to_response=config.add_sources_to_response,
+    use_messages_for_context=config.use_messages_in_context,
+    debug=config.debug
+)
+
+QUESTIONS_FILENAME = 'data/benchmark/questions.json'
+ANSWERS_FILENAME = 'data/benchmark/answers.json'
+
+
+def main():
+    benchmark_name = \
+        f'model: {config.question_answering_model_id}' \
+        f'index: {config.index_repo_id}'
+
+    wandb.init(
+        project='HF-Docs-QA',
+        name=f'model: {config.question_answering_model_id}',
+        mode='run', # run/disabled
+        config=config.asdict()
+    )
+    # log config to wandb
+
+    with open(QUESTIONS_FILENAME, 'r') as f: # json
+        questions = f.readlines()
+
+    with open(ANSWERS_FILENAME, 'w') as f:
+        for q in questions:
+            question = q['question']
+            messages_contex = q['messages_context']
+
+            t_start = time.perf_counter()
+            response = model.get_response(
+                question=question, 
+                messages_context=messages_context
+            )
+            t_end = time.perf_counter()
+            # write to json
+            {
+                "answer": response.get_answer(),
+                "sources": response.get_sources_as_text(),
+                'time': t_end - t_start
+            }
+
+
+if __name__ == '__main__':
+    main()

config/.env.example

@@ -0,0 +1,16 @@
+# QA engine settings
+QUESTION_ANSWERING_MODEL_ID=hf-question-answering-model-ID
+EMBEDDING_MODEL_ID=hf-embedding-model-ID
+INDEX_REPO_ID=hf-index-ID
+PROMPT_TEMPLATE_NAME=llama
+USE_DOCS_FOR_CONTEXT=True
+NUM_RELEVANT_DOCS=4
+ADD_SOURCES_TO_RESPONSE=True
+USE_MESSAGES_IN_CONTEXT=True
+DEBUG=True
+
+# Discord settings
+DISCORD_TOKEN=your-bot-token
+NUM_LAST_MESSAGES=1
+USE_NAMES_IN_CONTEXT=False
+ENABLE_COMMANDS=True

config/prompt_templates/llama.txt

@@ -0,0 +1,7 @@
+### Instruction:
+Give an answer that contains all the necessary information for the question.
+If the context contains necessary information to answer question, use it to generate an appropriate response.
+{context}
+### Input:
+{question}
+### Response:

config/prompt_templates/llama2.txt

@@ -0,0 +1,6 @@
+<<SYS>>
+**CONTEXT FOR THE QUESTION**:
+{context}
+<</SYS>>
+
+[INST] Respond for the user question as factualy as possible, using given context. [/INST] User: {question}

config/prompt_templates/sythia_v1.3.txt

@@ -0,0 +1,3 @@
+SYSTEM: {context}
+USER: {question}
+ASSISTANT:

data/datasets/hf_repositories_urls.json

@@ -0,0 +1,22 @@
+{
+    "urls": [
+        "https://github.com/huggingface/transformers",
+        "https://github.com/huggingface/diffusers",
+        "https://github.com/huggingface/datasets",
+        "https://github.com/gradio-app/gradio",
+        "https://github.com/huggingface/huggingface_hub",
+        "https://github.com/huggingface/peft",
+        "https://github.com/huggingface/blog",
+        "https://github.com/huggingface/optimum",
+        "https://github.com/huggingface/tokenizers",
+        "https://github.com/huggingface/course",
+        "https://github.com/huggingface/deep-rl-class",
+        "https://github.com/huggingface/evaluate",
+        "https://github.com/huggingface/datasets-server",
+        "https://github.com/huggingface/simulate",
+        "https://github.com/huggingface/hub-docs",
+        "https://github.com/huggingface/pytorch-image-models",
+        "https://github.com/huggingface/safetensors",
+        "https://github.com/huggingface/hf-endpoints-documentation"
+    ]
+}

data/datasets/hf_repositories_urls_scraped.json

@@ -0,0 +1,92 @@
+{
+    "urls": [
+        "https://github.com/huggingface/tokenizers",
+        "https://github.com/huggingface/datablations",
+        "https://github.com/huggingface/peft",
+        "https://github.com/huggingface/tflite-android-transformers",
+        "https://github.com/huggingface/simulate",
+        "https://github.com/huggingface/transformers",
+        "https://github.com/huggingface/deep-rl-class",
+        "https://github.com/huggingface/awesome-huggingface",
+        "https://github.com/huggingface/datasets-server",
+        "https://github.com/huggingface/setfit",
+        "https://github.com/huggingface/olm-training",
+        "https://github.com/huggingface/huggingface_sb3",
+        "https://github.com/huggingface/optimum-neuron",
+        "https://github.com/huggingface/blog",
+        "https://github.com/huggingface/100-times-faster-nlp",
+        "https://github.com/huggingface/bloom-jax-inference",
+        "https://github.com/huggingface/speechbox",
+        "https://github.com/huggingface/olm-datasets",
+        "https://github.com/huggingface/hub-docs",
+        "https://github.com/huggingface/torchMoji",
+        "https://github.com/huggingface/hffs",
+        "https://github.com/huggingface/trl",
+        "https://github.com/huggingface/text-generation-inference",
+        "https://github.com/huggingface/Mongoku",
+        "https://github.com/huggingface/education-toolkit",
+        "https://github.com/huggingface/datasets",
+        "https://github.com/huggingface/optimum-benchmark",
+        "https://github.com/huggingface/course",
+        "https://github.com/huggingface/accelerate",
+        "https://github.com/huggingface/pytorch-image-models",
+        "https://github.com/huggingface/fuego",
+        "https://github.com/huggingface/diffusion-models-class",
+        "https://github.com/huggingface/disaggregators",
+        "https://github.com/huggingface/unity-api",
+        "https://github.com/huggingface/workshops",
+        "https://github.com/huggingface/llm-ls",
+        "https://github.com/huggingface/llm-vscode",
+        "https://github.com/huggingface/community-events",
+        "https://github.com/huggingface/tune",
+        "https://github.com/huggingface/candle",
+        "https://github.com/huggingface/paper-style-guide",
+        "https://github.com/huggingface/huggingface.js",
+        "https://github.com/huggingface/neuralcoref",
+        "https://github.com/huggingface/hfapi",
+        "https://github.com/huggingface/data-measurements-tool",
+        "https://github.com/huggingface/personas",
+        "https://github.com/huggingface/instruction-tuned-sd",
+        "https://github.com/huggingface/swift-transformers",
+        "https://github.com/huggingface/api-inference-community",
+        "https://github.com/huggingface/diffusers",
+        "https://github.com/huggingface/safetensors",
+        "https://github.com/huggingface/optimum-graphcore",
+        "https://github.com/huggingface/OBELICS",
+        "https://github.com/huggingface/swift-coreml-diffusers",
+        "https://github.com/huggingface/naacl_transfer_learning_tutorial",
+        "https://github.com/huggingface/nn_pruning",
+        "https://github.com/huggingface/awesome-papers",
+        "https://github.com/huggingface/optimum-intel",
+        "https://github.com/huggingface/autotrain-advanced",
+        "https://github.com/huggingface/pytorch-openai-transformer-lm",
+        "https://github.com/huggingface/node-question-answering",
+        "https://github.com/huggingface/optimum",
+        "https://github.com/huggingface/knockknock",
+        "https://github.com/huggingface/optimum-habana",
+        "https://github.com/huggingface/transfer-learning-conv-ai",
+        "https://github.com/huggingface/notebooks",
+        "https://github.com/huggingface/hmtl",
+        "https://github.com/huggingface/block_movement_pruning",
+        "https://github.com/huggingface/huggingface_hub",
+        "https://github.com/huggingface/transformers-bloom-inference",
+        "https://github.com/huggingface/hf_transfer",
+        "https://github.com/huggingface/doc-builder",
+        "https://github.com/huggingface/large_language_model_training_playbook",
+        "https://github.com/huggingface/that_is_good_data",
+        "https://github.com/huggingface/swift-coreml-transformers",
+        "https://github.com/huggingface/datasets-viewer",
+        "https://github.com/huggingface/open-muse",
+        "https://github.com/huggingface/evaluate",
+        "https://github.com/huggingface/llm_training_handbook",
+        "https://github.com/huggingface/pytorch_block_sparse",
+        "https://github.com/huggingface/chat-ui",
+        "https://github.com/huggingface/llm.nvim",
+        "https://github.com/huggingface/swift-chat",
+        "https://github.com/huggingface/pytorch-pretrained-BigGAN",
+        "https://github.com/huggingface/exporters",
+        "https://github.com/huggingface/audio-transformers-course",
+        "https://github.com/huggingface/hf-endpoints-documentation",
+        "https://github.com/gradio-app/gradio"
+    ]
+}

data/get_hf_repositories_urls.py

@@ -0,0 +1,49 @@
+import json
+import argparse
+import requests
+from typing import List
+
+
+def get_repositories_names(token: str, min_stars: int) -> List[str]:
+    repos_per_page = 100
+    repo_names = []
+    i = 0
+    while True:
+        url = \
+            f'https://api.github.com/orgs/huggingface/repos?' \
+            f'per_page={repos_per_page}&page={i}'
+        headers = {'Authorization': f'token {token}'}
+        response = requests.get(url, headers=headers)
+        if response.status_code == 200:
+            repos = json.loads(response.content)
+            repo_names += [
+                repo['full_name'] for repo in repos
+                if repo['stargazers_count'] >= min_stars
+            ]
+            if len(repos) < repos_per_page:
+                break
+            i += 1
+        else:
+            return 'Error: '+str(response.status_code)
+    return list(set(repo_names))
+
+
+def save_repositories_urls(repositories_names: List[str], output_filename: str):
+    urls = [f'https://github.com/{repo_name}' for repo_name in repositories_names]
+    data = {'urls': urls}
+    with open(output_filename, 'w') as f:
+        json.dump(data, f, indent=4)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--token', type=str)
+    parser.add_argument('--stars', type=str)
+    args = parser.parse_args()
+    repositories = get_repositories_names(token=args.token, min_stars=int(args.stars))
+    repositories += [
+        'huggingface/hf-endpoints-documentation',
+        'gradio-app/gradio'
+    ]
+    print(f'Found {len(repositories)} repositories with at least {args.stars} stars')
+    save_repositories_urls(repositories, 'datasets/hf_repositories_urls_scraped.json')

data/hugging_face_docs_dataset.py

@@ -0,0 +1,190 @@
+import glob
+import json
+import os
+import re
+import subprocess
+from typing import List
+
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+from markdown import markdown
+import nbformat
+from nbconvert import MarkdownExporter
+from nbconvert.preprocessors import Preprocessor, ClearOutputPreprocessor
+from tqdm import tqdm
+
+
+VALIDATE_URLS = False
+
+
+def download_repositories(repo_urls_file: str, repo_dir: str):
+    """
+    Downloads the Hugging Face repositories.
+    """
+    if not os.path.exists(repo_dir):
+        os.makedirs(repo_dir)
+    with open(repo_urls_file, "r") as f:
+        repositories_urls = json.load(f)["urls"]
+        print(f'Downloading {len(repositories_urls)} repositories')
+        for url in repositories_urls:
+            try:
+                subprocess.run(["git", "clone", url], cwd=repo_dir)
+            except subprocess.CalledProcessError as e:
+                print("Command failed with error:", e.stderr)
+
+
+class EmptyCellPreprocessor(Preprocessor):
+    def preprocess_cell(self, cell, resources, index):
+        if cell.source.strip() == '':
+            cell.source = ''
+            cell.cell_type = 'raw'
+        return cell, resources
+
+
+def convert_notebook_to_txt(filename: str):
+    """
+    Converts a notebook to a markdown file.
+    """
+    with open(filename) as f:
+        notebook = nbformat.read(f, as_version=4)
+    # id validation error fix
+    for cell in notebook['cells']:
+        cell['id'] = str(cell['id'])
+
+    clear_output = ClearOutputPreprocessor()
+    notebook, resources = clear_output.preprocess(notebook, {})
+
+    exporter = MarkdownExporter()
+    exporter.register_preprocessor(EmptyCellPreprocessor, enabled=True)
+    output_notebook_text, resources = exporter.from_notebook_node(notebook)
+
+    new_filename = filename.replace('.ipynb', '_ipynb.md')
+    with open(new_filename, 'w') as f:
+        f.write(output_notebook_text)
+    return new_filename
+
+
+def extract_files_from_directories(
+    repo_urls_file: str,
+    repo_dir: str,
+    docs_dir: str,
+    files_extensions: List[str]
+) -> None:
+
+    """
+    This function reads markdown and markdownx files from the repositories directory,
+    filters out non-English files, and adds the source GitHub URL as the first line of each file.
+    The resulting files are saved in the docs_dir.
+    """
+    languages = pd.read_csv("language-codes.csv").loc[:,"alpha2"].tolist()
+    languages.remove("en")
+
+    files = [
+        filename
+        for extension in files_extensions
+        for filename in glob.glob(repo_dir + f"**/*{extension}", recursive=True)
+    ]
+    print(f'Used extensions: {", ".join(files_extensions)}')
+    print(f'Found {len(files)} files')
+
+    repo_urls = []
+    with open(repo_urls_file, "r") as f:
+        repo_urls = json.load(f)["urls"]
+
+    # filter out the files that are not in english
+    filtered_files = []
+    for filename in files:
+        sep_file = filename.split("/")
+        for seq in sep_file:
+            if seq in languages:
+                break
+        else:
+            filtered_files.append(filename)
+    print(f'Found {len(filtered_files)} files in English')
+
+    # generate a GitHub URL for a file based on its name and a list of possible repository URLs
+    def get_github_url(filename: str, repo_urls: str, repo_dir: str) -> str:
+        source = filename.replace(repo_dir, '')
+        repo_name, file_path = source.split('/', 1)
+        repo_url_prefix = None
+        for repo_url in repo_urls:
+            if repo_name == repo_url.split('/')[-1]:
+                repo_url_prefix = repo_url
+                break
+        if not repo_url_prefix:
+            raise ValueError(f"Repo URL not found for {repo_name}")
+        url = f'{repo_url_prefix}/blob/main/{file_path}'
+        if VALIDATE_URLS:
+            try:
+                response = requests.get(url)
+                response.raise_for_status()
+            except:
+                print(f'filename: {filename}')
+                print(f'repo: {repo_name}, file: {file_path}')
+                print(f'url: {url}')
+                raise
+        return url
+
+    # creates a valid filename by replacing certain characters and removing the repo_dir path
+    def create_filename_from_path(filename: str, repo_dir: str) -> str:
+        filename = filename.replace(repo_dir, '')
+        chars_to_replace = ['/', '{', '}', '-', '.']
+        filename = ''.join(['_' if c in chars_to_replace else c for c in filename])
+        return filename
+
+    # copy the files with the source added in the first line
+    if not os.path.exists(docs_dir):
+        os.makedirs(docs_dir)
+    copied_files = []
+    for filename in tqdm(filtered_files):
+        source_url = get_github_url(filename, repo_urls, repo_dir)
+        data = f"source: {source_url}\n\n"
+        # convert jupyter notebooks to txt files
+        try:
+            if filename.endswith('.ipynb'):
+                filename = convert_notebook_to_txt(filename)
+            # rename and copy files
+            with open(filename, 'r') as f:
+                data += f.read()
+            output_filename = docs_dir + create_filename_from_path(filename, repo_dir)
+            with open(output_filename, 'w') as f:
+                f.write(data)
+            if not os.path.isfile(output_filename):
+                raise ValueError(f"Failed to create the output file: {output_filename}")
+            copied_files.append(output_filename)
+        except Exception as ex:
+            print(f'Failed to copy file {filename}: {ex}')
+
+    print(f'Successfully copied {len(set(copied_files))}/{len(filtered_files)} files')
+
+
+def markdown_cleaner(data: str):
+    """
+    Clean markdown text.
+
+    Args:
+        data (str): The markdown text to be cleaned.
+
+    Returns:
+        str: The cleaned markdown text.
+    """
+    soupped = BeautifulSoup(markdown(data), "html.parser")
+    raw_text = ''.join(soupped.findAll(string=True))
+    clean_text = re.sub(r"<!--.*?-->", "", raw_text, flags=re.DOTALL)
+    # remove any special tokens e.g <|endoftext|>
+    clean_text = re.sub(r"<\|endoftext\|>", "", clean_text, flags=re.DOTALL)
+    # discard non english text
+    clean_text = re.sub(r"[^a-zA-Z0-9\s]", "", clean_text, flags=re.DOTALL)
+    return "\n".join([t for t in clean_text.split("\n") if t])
+
+
+if __name__ == '__main__':
+    repo_urls_file = "./datasets/hf_repositories_urls.json"
+    repo_dir = "./datasets/huggingface_repositories/"
+    docs_dir = "./datasets/huggingface_docs/"
+    download_repositories(repo_urls_file, repo_dir)
+    extract_files_from_directories(
+        repo_urls_file, repo_dir, docs_dir,
+        files_extensions=['.md', '.mdx', '.ipynb']
+    )

data/indexer.ipynb

@@ -0,0 +1,238 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import math\n",
+    "import numpy as np\n",
+    "from pathlib import Path\n",
+    "from tqdm import tqdm\n",
+    "from typing import List, Any\n",
+    "from langchain.chains import RetrievalQA\n",
+    "from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings\n",
+    "from langchain.document_loaders import TextLoader\n",
+    "from langchain.indexes import VectorstoreIndexCreator\n",
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "from langchain.vectorstores import FAISS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "docs = []\n",
+    "metadata = []\n",
+    "for p in Path(\"./datasets/huggingface_docs/\").iterdir():\n",
+    "    if not p.is_dir():\n",
+    "        with open(p) as f:\n",
+    "            # the first line is the source of the text\n",
+    "            source = f.readline().strip().replace('source: ', '')\n",
+    "            docs.append(f.read())\n",
+    "            metadata.append({\"source\": source})\n",
+    "    # break\n",
+    "\n",
+    "print(f'number of documents: {len(docs)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chunk_size = 512\n",
+    "text_splitter = CharacterTextSplitter(\n",
+    "    separator=\"\",\n",
+    "    chunk_size=chunk_size,\n",
+    "    chunk_overlap=100,\n",
+    "    length_function=len,\n",
+    ")\n",
+    "docs = text_splitter.create_documents(docs, metadata)\n",
+    "print(f'number of chunks: {len(docs)}')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_name = \"hkunlp/instructor-large\"\n",
+    "embed_instruction = \"Represent the Hugging Face library documentation\"\n",
+    "query_instruction = \"Query the most relevant piece of information from the Hugging Face documentation\"\n",
+    "\n",
+    "embedding_model = HuggingFaceInstructEmbeddings(\n",
+    "    model_name=model_name,\n",
+    "    embed_instruction=embed_instruction,\n",
+    "    query_instruction=query_instruction,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "class AverageInstructEmbeddings(HuggingFaceInstructEmbeddings):\n",
+    "    max_length: int = None\n",
+    "\n",
+    "    def __init__(self, max_length: int = 512, **kwargs: Any):\n",
+    "        super().__init__(**kwargs)\n",
+    "        self.max_length = max_length\n",
+    "        if self.max_length < 0:\n",
+    "            print('max_length is not specified, using model default max_seq_length')\n",
+    "\n",
+    "    def embed_documents(self, texts: List[str]) -> List[List[float]]:\n",
+    "        all_embeddings = []\n",
+    "        for text in tqdm(texts, desc=\"Embedding documents\"):\n",
+    "            if len(text) > self.max_length and self.max_length > -1:\n",
+    "                n_chunks = math.ceil(len(text)/self.max_length)\n",
+    "                chunks = [\n",
+    "                    text[i*self.max_length:(i+1)*self.max_length]\n",
+    "                    for i in range(n_chunks)\n",
+    "                ]\n",
+    "                instruction_pairs = [[self.embed_instruction, chunk] for chunk in chunks]\n",
+    "                chunk_embeddings = self.client.encode(instruction_pairs)\n",
+    "                avg_embedding = np.mean(chunk_embeddings, axis=0)\n",
+    "                all_embeddings.append(avg_embedding.tolist())\n",
+    "            else:\n",
+    "                instruction_pairs = [[self.embed_instruction, text]]\n",
+    "                embeddings = self.client.encode(instruction_pairs)\n",
+    "                all_embeddings.append(embeddings[0].tolist())\n",
+    "\n",
+    "        return all_embeddings\n",
+    "\n",
+    "\n",
+    "# max length fed to the model, if longer than max then chunks + averaging\n",
+    "max_length = 512\n",
+    "embedding_model = AverageInstructEmbeddings(  \n",
+    "    model_name=model_name,\n",
+    "    embed_instruction=embed_instruction,\n",
+    "    query_instruction=query_instruction,\n",
+    "    max_length=max_length,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings = embedding_model.embed_documents(texts=[d.page_content for d in docs[:10]])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = FAISS.from_documents(docs, embedding_model)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index_name = f'index-{model_name}-{chunk_size}-m{max_length}-notebooks'\n",
+    "index_name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.save_local(f'../indexes/{index_name}/')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index = FAISS.load_local(f'../indexes/{index_name}/', embedding_model)\n",
+    "docs = index.similarity_search(query='how to create a pipeline object?', k=5)\n",
+    "docs[0].page_content\n",
+    "docs[0].metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i, doc in enumerate(docs, start=1):\n",
+    "    print(f\"\\n{'='*100}\\n\")\n",
+    "    print(f\"Document {i} of {len(docs)}\")\n",
+    "    print(\"Page Content:\")\n",
+    "    print(f\"\\n{'-'*100}\\n\")\n",
+    "    print(doc.page_content, '\\n')\n",
+    "    print(doc.metadata)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from huggingface_hub import HfApi\n",
+    "\n",
+    "api = HfApi()\n",
+    "api.create_repo(\n",
+    "    repo_id=f'KonradSzafer/{index_name}',\n",
+    "    repo_type='dataset',\n",
+    "    private=False,\n",
+    "    exist_ok=True\n",
+    ")\n",
+    "api.upload_folder(\n",
+    "    folder_path=f'../indexes/{index_name}',\n",
+    "    repo_id=f'KonradSzafer/{index_name}',\n",
+    "    repo_type='dataset',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_qa_bot",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data/language-codes.csv

@@ -0,0 +1,190 @@
+alpha2,English
+aa,Afar
+ab,Abkhazian
+ae,Avestan
+af,Afrikaans
+ak,Akan
+am,Amharic
+an,Aragonese
+ar,Arabic
+as,Assamese
+av,Avaric
+ay,Aymara
+az,Azerbaijani
+ba,Bashkir
+be,Belarusian
+bg,Bulgarian
+bh,Bihari languages
+bi,Bislama
+bm,Bambara
+bn,Bengali
+bo,Tibetan
+br,Breton
+bs,Bosnian
+ca,Catalan; Valencian
+ce,Chechen
+ch,Chamorro
+co,Corsican
+cr,Cree
+cs,Czech
+cu,Church Slavic; Old Slavonic; Church Slavonic; Old Bulgarian; Old Church Slavonic
+cv,Chuvash
+cy,Welsh
+da,Danish
+de,German
+dv,Divehi; Dhivehi; Maldivian
+dz,Dzongkha
+ee,Ewe
+el,"Greek, Modern (1453-)"
+en,English
+eo,Esperanto
+es,Spanish; Castilian
+et,Estonian
+eu,Basque
+fa,Persian
+ff,Fulah
+fi,Finnish
+fj,Fijian
+fo,Faroese
+fr,French
+fy,Western Frisian
+ga,Irish
+gd,Gaelic; Scottish Gaelic
+gj,Gujarati
+gl,Galician
+gn,Guarani
+gu,Gujarati
+gv,Manx
+ha,Hausa
+hd,Hindi
+he,Hebrew
+hi,Hindi
+ho,Hiri Motu
+hr,Croatian
+ht,Haitian; Haitian Creole
+hu,Hungarian
+hy,Armenian
+hz,Herero
+ia,Interlingua (International Auxiliary Language Association)
+id,Indonesian
+ie,Interlingue; Occidental
+ig,Igbo
+ii,Sichuan Yi; Nuosu
+ik,Inupiaq
+io,Ido
+is,Icelandic
+it,Italian
+iu,Inuktitut
+ja,Japanese
+jv,Javanese
+ka,Georgian
+kg,Kongo
+ki,Kikuyu; Gikuyu
+kj,Kuanyama; Kwanyama
+kk,Kazakh
+kl,Kalaallisut; Greenlandic
+km,Central Khmer
+kn,Kannada
+ko,Korean
+kr,Kanuri
+ks,Kashmiri
+ku,Kurdish
+kv,Komi
+kw,Cornish
+ky,Kirghiz; Kyrgyz
+la,Latin
+lb,Luxembourgish; Letzeburgesch
+lg,Ganda
+li,Limburgan; Limburger; Limburgish
+ln,Lingala
+lo,Lao
+lt,Lithuanian
+lu,Luba-Katanga
+lv,Latvian
+mg,Malagasy
+mh,Marshallese
+mi,Maori
+mk,Macedonian
+ml,Malayalam
+mn,Mongolian
+mr,Marathi
+ms,Malay
+mt,Maltese
+my,Burmese
+na,Nauru
+nb,"Bokmål, Norwegian; Norwegian Bokmål"
+nd,"Ndebele, North; North Ndebele"
+ne,Nepali
+ng,Ndonga
+nl,Dutch; Flemish
+nn,"Norwegian Nynorsk; Nynorsk, Norwegian"
+no,Norwegian
+nr,"Ndebele, South; South Ndebele"
+nv,Navajo; Navaho
+ny,Chichewa; Chewa; Nyanja
+oc,Occitan (post 1500)
+oj,Ojibwa
+om,Oromo
+or,Oriya
+os,Ossetian; Ossetic
+pa,Panjabi; Punjabi
+pi,Pali
+pl,Polish
+ps,Pushto; Pashto
+pt,Portuguese
+qu,Quechua
+rm,Romansh
+rn,Rundi
+ro,Romanian; Moldavian; Moldovan
+ru,Russian
+rw,Kinyarwanda
+sa,Sanskrit
+sc,Sardinian
+sd,Sindhi
+se,Northern Sami
+sg,Sango
+si,Sinhala; Sinhalese
+sk,Slovak
+sl,Slovenian
+sm,Samoan
+sn,Shona
+so,Somali
+sq,Albanian
+sr,Serbian
+ss,Swati
+st,"Sotho, Southern"
+su,Sundanese
+sv,Swedish
+sw,Swahili
+ta,Tamil
+te,Telugu
+tg,Tajik
+th,Thai
+ti,Tigrinya
+tk,Turkmen
+tl,Tagalog
+tn,Tswana
+to,Tonga (Tonga Islands)
+tr,Turkish
+ts,Tsonga
+tt,Tatar
+tw,Twi
+ty,Tahitian
+ug,Uighur; Uyghur
+uk,Ukrainian
+ur,Urdu
+uz,Uzbek
+ve,Venda
+vi,Vietnamese
+vo,Volapük
+wa,Walloon
+wo,Wolof
+xh,Xhosa
+yi,Yiddish
+yo,Yoruba
+za,Zhuang; Chuang
+zh,Chinese; General
+zh-CN,Chinese; Simplified
+zh-TW,Chinese; Traditional
+zh-hans,Chinese; Simplified
+zu,Zulu

data/scrapers/stack_overflow_scraper.py

@@ -0,0 +1,91 @@
+import re
+import csv
+import time
+import requests
+from typing import List
+import pandas as pd
+from tqdm import tqdm
+from bs4 import BeautifulSoup
+
+
+def scrape_question_with_answers(question_url: str) -> List[str]:
+    url = 'https://stackoverflow.com/' + question_url
+    response = requests.get(url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+
+    title = soup.find('title').text.replace(' - Stack Overflow', '')
+    question_div = soup.find('div', {'class': 'postcell post-layout--right'})
+    question = question_div.find('p').text
+    answers_div = soup.find('div', {'class': 'answercell post-layout--right'})
+    answer = answers_div.find('div', {'class': 's-prose js-post-body'}).text
+    return [title, question, answer, url]
+
+
+def scrape_questions_page(url: str, min_votes: int, min_answers: int) -> List[List[str]]:
+    response = requests.get(url)
+    soup = BeautifulSoup(response.content, 'html.parser')
+    posts_summaries = soup.find_all('div', {'class':'s-post-summary js-post-summary'})
+
+    qa_data = []
+    for summary in posts_summaries:
+        stats_div = summary.find('div', {'class': 's-post-summary--stats'})
+        vote_div = stats_div.find('div', {
+            'class': 's-post-summary--stats-item s-post-summary--stats-item__emphasized',
+            'title': re.compile(r'^Score of \d+$')})
+        if vote_div:
+            vote_number = int(vote_div.find('span', {'class': 's-post-summary--stats-item-number'}).text)
+        else:
+            vote_number = 0
+        answer_div = stats_div.find('div', {
+            'class': 's-post-summary--stats-item',
+            'title': re.compile(r'^\d+ answers$')})
+        if answer_div:
+            answer_number = int(answer_div.find('span', {'class': 's-post-summary--stats-item-number'}).text)
+        else:
+            answer_number = 0
+
+        question_href = summary.find('a', {'class': 's-link'})['href']
+        if vote_number >= min_votes and answer_number >= min_answers:
+            try:
+                qa_data.append(scrape_question_with_answers(question_href))
+            except Exception as error:
+                print(error)
+
+        time.sleep(1.5)
+    return qa_data
+
+
+def crawl_and_save_qa(
+    filename: str,
+    base_url: str,
+    start_page: int,
+    n_pages: int=10,
+    min_votes: int=1,
+    min_answers: int=1
+):
+    with open(filename, 'a', newline='') as f:
+        writer = csv.writer(f)
+        if start_page == 1:
+            writer.writerow(['title', 'question', 'answer', 'url'])
+        for page_num in tqdm(range(start_page, start_page+n_pages)):
+            page_data = scrape_questions_page(
+                base_url.format(page_num),
+                min_votes,
+                min_answers
+            )
+            if page_data:
+                for qa_data in page_data:
+                    writer.writerow(qa_data)
+
+
+if __name__ == '__main__':
+    filename = '../datasets/stackoverflow_linux.csv'
+    url = 'https://stackoverflow.com/questions/tagged/linux?tab=votes&page={}&pagesize=15'
+    crawl_and_save_qa(
+        filename=filename,
+        base_url=url,
+        start_page=21,
+        n_pages=10,
+        min_votes=1,
+        min_answers=1
+    )

data/stackoverflow_python_dataset.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+from datasets import load_dataset
+from bs4 import BeautifulSoup
+
+
+def preprocess_dataset():
+    """
+    Preprocesses the 'koutch/stackoverflow_python' dataset.
+
+    Returns:
+        datasets.arrow_dataset.Dataset: The preprocessed dataset.
+    """
+    dataset = load_dataset('koutch/stackoverflow_python', split='train')
+    dataset = dataset.filter(
+        lambda example:
+            example['question_score'] > 100 and
+            example['answer_score'] > 5 and
+            datetime.strptime(example['answer_date'], '%Y-%m-%dT%H:%M:%SZ').year > 2010
+        )
+
+    def html2text(example):
+        soup = BeautifulSoup(example, 'html.parser')
+        return ''.join(soup.findAll(string=True))
+
+    def transforms(example):
+        example['answer'] = html2text(example['answer_body'])
+        example['question'] = html2text(example['question_body'])
+        return example
+
+    dataset = dataset.map(lambda example: transforms(example))
+    dataset = dataset.remove_columns([
+        'question_score', 'question_date', 'question_id',
+        'answer_date', 'answer_id', 'answer_score', 'tags',
+        'question_body', 'answer_body'
+    ])
+    return dataset
+
+
+def show_info(dataset):
+    """
+    Print information about the dataset.
+
+    Args:
+        dataset (datasets.arrow_dataset.Dataset): The dataset.
+    """
+    print(dataset.info, '\n')
+    print(f'dataset len: {len(dataset)}')
+    print(f"example question: {dataset[0]['question']}")
+    print(f"example answer: {dataset[0]['answer']}")
+
+
+if __name__ == '__main__':
+    dataset = preprocess_dataset()
+    dataset.push_to_hub('KonradSzafer/stackoverflow_python_preprocessed', private=False)
+    show_info(dataset)

data/upload_csv_dataset.py

@@ -0,0 +1,24 @@
+import sys
+import pandas as pd
+from datasets import Dataset, DatasetDict
+from sklearn.model_selection import train_test_split
+
+
+
+def main():
+    dataset_name = sys.argv[1]
+    test_size = float(sys.argv[2]) if len(sys.argv) > 2 else 0.1
+    print(f'dataset: {dataset_name}, test size: {test_size}')
+
+    filename = f'datasets/{dataset_name}.csv'
+    df = pd.read_csv(filename)
+    dataset = Dataset.from_pandas(df)
+    train_dataset, test_dataset = train_test_split(dataset, test_size=test_size)
+    train_dataset = Dataset.from_dict(train_dataset)
+    test_dataset = Dataset.from_dict(test_dataset)
+    dataset_dict = DatasetDict({'train': train_dataset, 'test': test_dataset})
+    dataset_dict.push_to_hub(f'KonradSzafer/{dataset_name}', private=False)
+
+
+if __name__ == '__main__':
+    main()

discord_bot/__main__.py

@@ -0,0 +1,28 @@
+from qa_engine import logger, Config, QAEngine
+from discord_bot.client import DiscordClient
+
+
+config = Config()
+qa_engine = QAEngine(
+    llm_model_id=config.question_answering_model_id,
+    embedding_model_id=config.embedding_model_id,
+    index_repo_id=config.index_repo_id,
+    prompt_template=config.prompt_template,
+    use_docs_for_context=config.use_docs_for_context,
+    num_relevant_docs=config.num_relevant_docs,
+    add_sources_to_response=config.add_sources_to_response,
+    use_messages_for_context=config.use_messages_in_context,
+    debug=config.debug
+)
+client = DiscordClient(
+    qa_engine=qa_engine,
+    num_last_messages=config.num_last_messages,
+    use_names_in_context=config.use_names_in_context,
+    enable_commands=config.enable_commands,
+    debug=config.debug
+)
+
+
+if __name__ == '__main__':
+    logger.info('Starting Application...')
+    client.run(config.discord_token)

discord_bot/client/__init__.py

@@ -0,0 +1 @@
+from .client import DiscordClient

discord_bot/client/client.py

@@ -0,0 +1,130 @@
+import json
+import requests
+from urllib.parse import quote
+import discord
+from typing import List
+
+from qa_engine import logger, QAEngine
+from discord_bot.client.utils import split_text_into_chunks
+
+
+class DiscordClient(discord.Client):
+    """
+    Discord Client class, used for interacting with a Discord server.
+
+    Args:
+        qa_service_url (str): The URL of the question answering service.
+        num_last_messages (int, optional): The number of previous messages to use as context for generating answers.
+        Defaults to 5.
+        use_names_in_context (bool, optional): Whether to include user names in the message context. Defaults to True.
+        enable_commands (bool, optional): Whether to enable commands for the bot. Defaults to True.
+
+    Attributes:
+        qa_service_url (str): The URL of the question answering service.
+        num_last_messages (int): The number of previous messages to use as context for generating answers.
+        use_names_in_context (bool): Whether to include user names in the message context.
+        enable_commands (bool): Whether to enable commands for the bot.
+        max_message_len (int): The maximum length of a message.
+        system_prompt (str): The system prompt to be used.
+
+    """
+    def __init__(
+        self,
+        qa_engine: QAEngine,
+        num_last_messages: int = 5,
+        use_names_in_context: bool = True,
+        enable_commands: bool = True,
+        debug: bool = False
+    ):
+        logger.info('Initializing Discord client...')
+        intents = discord.Intents.all()
+        intents.message_content = True
+        super().__init__(intents=intents, command_prefix='!')
+
+        assert num_last_messages >= 1, \
+            'The number of last messages in context should be at least 1'
+
+        self.qa_engine: QAEngine = qa_engine
+        self.num_last_messages: int = num_last_messages
+        self.use_names_in_context: bool = use_names_in_context
+        self.enable_commands: bool = enable_commands
+        self.debug: bool = debug
+        self.min_messgae_len: int = 1800
+        self.max_message_len: int = 2000
+
+
+    async def on_ready(self):
+        """
+        Callback function to be called when the client is ready.
+        """
+        logger.info('Successfully logged in as: {0.user}'.format(self))
+        await self.change_presence(activity=discord.Game(name='Chatting...'))
+
+
+    async def get_last_messages(self, message) -> List[str]:
+        """
+        Method to fetch recent messages from a message's channel.
+
+        Args:
+            message (Message): The discord Message object used to identify the channel.
+
+        Returns:
+            List[str]: Reversed list of recent messages from the channel,
+            excluding the input message. Messages may be prefixed with the author's name 
+            if `self.use_names_in_context` is True.
+        """
+        last_messages: List[str] = []
+        async for msg in message.channel.history(
+            limit=self.num_last_messages):
+            if self.use_names_in_context:
+                last_messages.append(f'{msg.author}: {msg.content}')
+            else:
+                last_messages.append(msg.content)
+        last_messages.reverse()
+        last_messages.pop() # remove last message from context
+        return last_messages
+
+
+    async def send_message(self, message, answer: str, sources: str):
+        chunks = split_text_into_chunks(
+            text=answer,
+            split_characters=['. ', ', ', '\n'],
+            min_size=self.min_messgae_len,
+            max_size=self.max_message_len
+        )
+        for chunk in chunks:
+            await message.channel.send(chunk)
+        await message.channel.send(sources)
+
+
+    async def on_message(self, message):
+        """
+        Callback function to be called when a message is received.
+
+        Args:
+            message (discord.Message): The received message.
+        """
+        if message.author == self.user:
+            return
+        if self.enable_commands and message.content.startswith('!'):
+            if message.content == '!clear':
+                await message.channel.purge()
+                return
+
+        last_messages = await self.get_last_messages(message)
+        context = '\n'.join(last_messages)
+
+        logger.info('Received message: {0.content}'.format(message))
+        response = self.qa_engine.get_response(
+            question=message.content,
+            messages_context=context
+        )
+        logger.info('Sending response: {0}'.format(response))
+        try:
+            await self.send_message(
+                message,
+                response.get_answer(),
+                response.get_sources_as_text()
+            )
+        except Exception as e:
+            logger.error('Failed to send response: {0}'.format(e))

discord_bot/client/utils.py

@@ -0,0 +1,54 @@
+from typing import List
+
+
+def find_max_split_index(text: str, char: str) -> int:
+    char_idx = text.rfind(char)
+    if char_idx > 0:
+        # If a character is found, return the index after the splitting character
+        split_idx = char_idx + len(char)
+        if split_idx >= len(text):
+            return len(text)
+        else:
+            return split_idx
+    return -1
+
+
+def find_max_split_index_from_sequence(text: str, split_characters: List[str]) -> int:
+    split_index = max((
+        find_max_split_index(text, sequence)
+        for sequence in split_characters
+    ), default=-1)
+    return split_index
+
+
+def split_text_into_chunks(
+    text: str,
+    split_characters: List[str] = [],
+    min_size: int = 20,
+    max_size: int = 250,
+    ) -> List[str]:
+
+    chunks = []
+    start_idx = 0
+    end_idx = max_size
+    text_len = len(text)
+    while start_idx < text_len:
+        search_chunk = text[start_idx+min_size:end_idx]
+        split_idx = find_max_split_index_from_sequence(
+            text=search_chunk,
+            split_characters=split_characters
+        )
+        # if no spliting element found, set the maximal size
+        if split_idx < 1:
+            split_idx = end_idx
+        # if found - offset it by the starting idx of the chunk
+        else:
+            split_idx += start_idx + min_size
+
+        chunk = text[start_idx:split_idx]
+        chunks.append(chunk)
+
+        start_idx = split_idx
+        end_idx = split_idx + max_size
+
+    return chunks

docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+services:
+  api:
+    build:
+      context: .
+      dockerfile: Dockerfile.api
+    ports:
+      - 8000:8000
+    networks:
+      - mynetwork
+  bot:
+    build:
+      context: .
+      dockerfile: Dockerfile.bot
+    ports:
+      - 80:80
+    depends_on:
+      - api
+    networks:
+      - mynetwork
+networks:
+  mynetwork:
+    driver: bridge

models/inference.ipynb

@@ -0,0 +1,103 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import torch\n",
+    "import transformers\n",
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "\n",
+    "PROMPT_TEMPLATES_DIR = os.path.dirname(os.path.abspath(os.getcwd()))\n",
+    "PROMPT_TEMPLATES_DIR += '/config/api/prompt_templates/'\n",
+    "\n",
+    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt_template = 'sythia_v1.3'\n",
+    "with open(PROMPT_TEMPLATES_DIR + f'{prompt_template}.txt', 'r') as f:\n",
+    "    prompt_template = f.read()\n",
+    "\n",
+    "context = ''\n",
+    "question = 'How to fix a bike?'\n",
+    "\n",
+    "prompt = prompt_template.format(context=context, question=question)\n",
+    "print(f'prompt len: {len(prompt)}\\n')\n",
+    "print(prompt)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "model_id = 'migtissera/SynthIA-7B-v1.3'\n",
+    "\n",
+    "tokenizer = AutoTokenizer.from_pretrained(model_id)\n",
+    "model = AutoModelForCausalLM.from_pretrained(\n",
+    "    model_id,\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    trust_remote_code=True,\n",
+    "    load_in_8bit=False,\n",
+    "    device_map='auto',\n",
+    "    resume_download=True,\n",
+    ")\n",
+    "\n",
+    "pipeline = transformers.pipeline(\n",
+    "    'text-generation',\n",
+    "    model=model,\n",
+    "    tokenizer=tokenizer,\n",
+    "    device_map='auto',\n",
+    "    torch_dtype=torch.bfloat16,\n",
+    "    eos_token_id=tokenizer.eos_token_id,\n",
+    "    pad_token_id=tokenizer.eos_token_id,\n",
+    "    min_new_tokens=64,\n",
+    "    max_new_tokens=800,\n",
+    "    temperature=0.5,\n",
+    "    do_sample=True,\n",
+    ")\n",
+    "\n",
+    "output_text = pipeline(prompt)[0]['generated_text']\n",
+    "output_text = output_text.replace(prompt+'\\n', '')\n",
+    "print(output_text)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "hf_qa_bot",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "e769ac600d1c65682759767682b2a946c0eaa09d353302f712fe4c2e822e15df"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

qa_engine/__init__.py

@@ -0,0 +1,9 @@
+from dotenv import load_dotenv
+from qa_engine.logger import setup_logger
+
+setup_logger()
+load_dotenv(dotenv_path='config/.env')
+
+from .logger import setup_logger, logger
+from .config import Config
+from .qa_engine import QAEngine

qa_engine/config.py

@@ -0,0 +1,64 @@
+import os
+from dataclasses import dataclass, asdict
+from typing import Any, Union
+
+from qa_engine import logger
+
+
+def get_env(env_name: str, default: Any = None, warn: bool = True) -> str:
+    env = os.getenv(env_name)
+    if not env:
+        if default:
+            if warn:
+                logger.warning(
+                    f'Environment variable {env_name} not found.' \
+                    f'Using the default value: {default}.'
+                )
+            return default
+        else:
+            raise ValueError(f'Cannot parse: {env_name}')
+    else:
+        return env
+
+
+@dataclass
+class Config:
+    # QA Engine config
+    question_answering_model_id: str = get_env('QUESTION_ANSWERING_MODEL_ID')
+    embedding_model_id: str = get_env('EMBEDDING_MODEL_ID')
+    index_repo_id: str = get_env('INDEX_REPO_ID')
+    prompt_template_name: str = get_env('PROMPT_TEMPLATE_NAME')
+    use_docs_for_context: bool = eval(get_env('USE_DOCS_FOR_CONTEXT', 'True'))
+    num_relevant_docs: bool = eval(get_env('NUM_RELEVANT_DOCS', 3))
+    add_sources_to_response: bool = eval(get_env('ADD_SOURCES_TO_RESPONSE', 'True'))
+    use_messages_in_context: bool = eval(get_env('USE_MESSAGES_IN_CONTEXT', 'True'))
+    debug: bool = eval(get_env('DEBUG', 'True'))
+
+    # Discord bot config - optional
+    discord_token: str = get_env('DISCORD_TOKEN', '', warn=False)
+    num_last_messages: int = int(get_env('NUM_LAST_MESSAGES', 2, warn=False))
+    use_names_in_context: bool = eval(get_env('USE_NAMES_IN_CONTEXT', 'False', warn=False))
+    enable_commands: bool = eval(get_env('ENABLE_COMMANDS', 'True', warn=False))
+
+    def __post_init__(self):
+        prompt_template_file = f'config/prompt_templates/{self.prompt_template_name}.txt'
+        with open(prompt_template_file, 'r') as f:
+            self.prompt_template = f.read()
+        # validate config
+        if 'context' not in self.prompt_template:
+            raise ValueError("Prompt Template does not contain the 'context' field.")
+        if 'question' not in self.prompt_template:
+            raise ValueError("Prompt Template does not contain the 'question' field.")
+        if not self.use_docs_for_context and self.add_sources_to_response:
+            raise ValueError('Cannot add sources to response if not using docs in context')
+        if self.num_relevant_docs < 1:
+            raise ValueError('num_relevant_docs must be greater than 0')
+        self.log()
+
+    def asdict(self) -> dict:
+        return asdict(self)
+
+    def log(self) -> None:
+        logger.info('Config:')
+        for key, value in self.asdict().items():
+            logger.info(f'{key}: {value}')

qa_engine/logger.py

@@ -0,0 +1,14 @@
+import logging
+
+
+logger = logging.getLogger(__name__)
+
+def setup_logger() -> None:
+    """
+    Logger setup.
+    """
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    handler = logging.StreamHandler()
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)

qa_engine/mocks.py

@@ -0,0 +1,41 @@
+import os
+from typing import Mapping, Optional, Any
+
+from langchain.llms.base import LLM
+
+
+class MockLocalBinaryModel(LLM):
+    """
+    Mock Local Binary Model class, used for generating the string "a".
+
+    Args:
+        model_id (str): The ID of the model to be mocked.
+
+    Attributes:
+        model_path (str): The path to the model to be mocked.
+        llm (str): The string "a".
+
+    Raises:
+        ValueError: If the model_path does not exist.
+    """
+
+    model_path: str = None
+    llm: str = 'READY TO MOCK'
+
+    def __init__(self, model_id: str = None):
+        super().__init__()
+        self.model_path = f'bot/question_answering/{model_id}'
+        if not os.path.exists(self.model_path):
+            raise ValueError(f'{self.model_path} does not exist')
+
+
+    def _call(self, prompt: str, stop: Optional[list[str]] = None) -> str:
+        return self.llm
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {'name_of_model': self.model_path}
+
+    @property
+    def _llm_type(self) -> str:
+        return self.model_path

qa_engine/qa_engine.py

@@ -0,0 +1,286 @@
+import os
+import json
+import requests
+import subprocess
+from typing import Mapping, Optional, Any
+
+import torch
+import transformers
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from huggingface_hub import snapshot_download
+from urllib.parse import quote
+from langchain import PromptTemplate, HuggingFaceHub, LLMChain
+from langchain.llms import HuggingFacePipeline
+from langchain.llms.base import LLM
+from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceHubEmbeddings, HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+from sentence_transformers import CrossEncoder
+
+from qa_engine import logger
+from qa_engine.response import Response
+
+
+class LocalBinaryModel(LLM):
+    model_id: str = None
+    llm: None = None
+
+    def __init__(self, model_id: str = None):
+        super().__init__()
+        # pip install llama_cpp_python==0.1.39
+        from llama_cpp import Llama
+
+        model_path = f'qa_engine/{model_id}'
+        if not os.path.exists(model_path):
+            raise ValueError(f'{model_path} does not exist')
+        self.model_id = model_id
+        self.llm = Llama(model_path=model_path, n_ctx=4096)
+
+    def _call(self, prompt: str, stop: Optional[list[str]] = None) -> str:
+        output = self.llm(
+            prompt,
+            max_tokens=1024,
+            stop=['Q:'],
+            echo=False
+        )
+        return output['choices'][0]['text']
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {'name_of_model': self.model_id}
+
+    @property
+    def _llm_type(self) -> str:
+        return self.model_id
+
+
+class TransformersPipelineModel(LLM):
+    model_id: str = None
+    pipeline: str = None
+
+    def __init__(self, model_id: str = None):
+        super().__init__()
+        self.model_id = model_id
+
+        tokenizer = AutoTokenizer.from_pretrained(model_id)
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id,
+            torch_dtype=torch.bfloat16,
+            trust_remote_code=True,
+            load_in_8bit=False,
+            device_map='auto',
+            resume_download=True,
+        )
+        self.pipeline = transformers.pipeline(
+            'text-generation',
+            model=model,
+            tokenizer=tokenizer,
+            torch_dtype=torch.bfloat16,
+            device_map='auto',
+            eos_token_id=tokenizer.eos_token_id,
+            pad_token_id=tokenizer.eos_token_id,
+            min_new_tokens=64,
+            max_new_tokens=800,
+            temperature=0.5,
+            do_sample=True,
+        )
+
+    def _call(self, prompt: str, stop: Optional[list[str]] = None) -> str:
+        output_text = self.pipeline(prompt)[0]['generated_text']
+        output_text = output_text.replace(prompt+'\n', '')
+        return output_text
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {'name_of_model': self.model_id}
+
+    @property
+    def _llm_type(self) -> str:
+        return self.model_id
+
+
+class APIServedModel(LLM):
+    model_url: str = None
+    debug: bool = None
+
+    def __init__(self, model_url: str = None, debug: bool = None):
+        super().__init__()
+        if model_url[-1] == '/':
+            raise ValueError('URL should not end with a slash - "/"')
+        self.model_url = model_url
+        self.debug = debug
+
+    def _call(self, prompt: str, stop: Optional[list[str]] = None) -> str:
+        prompt_encoded = quote(prompt, safe='')
+        url = f'{self.model_url}/?prompt={prompt_encoded}'
+        if self.debug:
+            logger.info(f'URL: {url}')
+        try:
+            response = requests.get(url, timeout=1200, verify=False)
+            response.raise_for_status() 
+            return json.loads(response.content)['output_text']
+        except Exception as err:
+            logger.error(f'Error: {err}')
+            return f'Error: {err}'
+
+    @property
+    def _identifying_params(self) -> Mapping[str, Any]:
+        return {'name_of_model': f'model url: {self.model_url}'}
+
+    @property
+    def _llm_type(self) -> str:
+        return 'api_model'
+
+
+
+class QAEngine():
+    """
+    QAEngine class, used for generating answers to questions.
+
+    Args:
+        llm_model_id (str): The ID of the LLM model to be used.
+        embedding_model_id (str): The ID of the embedding model to be used.
+        index_repo_id (str): The ID of the index repository to be used.
+        run_locally (bool, optional): Whether to run the models locally or on the Hugging Face hub. Defaults to True.
+        use_docs_for_context (bool, optional): Whether to use relevant documents as context for generating answers.
+        Defaults to True.
+        use_messages_for_context (bool, optional): Whether to use previous messages as context for generating answers.
+        Defaults to True.
+        debug (bool, optional): Whether to log debug information. Defaults to False.
+
+    Attributes:
+        use_docs_for_context (bool): Whether to use relevant documents as context for generating answers.
+        use_messages_for_context (bool): Whether to use previous messages as context for generating answers.
+        debug (bool): Whether to log debug information.
+        llm_model (Union[LocalBinaryModel, HuggingFacePipeline, HuggingFaceHub]): The LLM model to be used.
+        embedding_model (Union[HuggingFaceInstructEmbeddings, HuggingFaceHubEmbeddings]): The embedding model to be used.
+        prompt_template (PromptTemplate): The prompt template to be used.
+        llm_chain (LLMChain): The LLM chain to be used.
+        knowledge_index (FAISS): The FAISS index to be used.
+
+    """
+    def __init__(
+        self,
+        llm_model_id: str,
+        embedding_model_id: str,
+        index_repo_id: str,
+        prompt_template: str,
+        use_docs_for_context: bool = True,
+        num_relevant_docs: int = 3,
+        add_sources_to_response: bool = True,
+        use_messages_for_context: bool = True,
+        first_stage_docs: int = 50,
+        debug: bool = False
+    ):
+        super().__init__()
+        self.prompt_template = prompt_template
+        self.use_docs_for_context = use_docs_for_context
+        self.num_relevant_docs = num_relevant_docs
+        self.add_sources_to_response = add_sources_to_response
+        self.use_messages_for_context = use_messages_for_context
+        self.first_stage_docs = first_stage_docs
+        self.debug = debug
+
+        if 'local_models/' in llm_model_id:
+            logger.info('using local binary model')
+            self.llm_model = LocalBinaryModel(
+                model_id=llm_model_id
+            )
+        elif 'api_models/' in llm_model_id:
+            logger.info('using api served model')
+            self.llm_model = APIServedModel(
+                model_url=llm_model_id.replace('api_models/', ''),
+                debug=self.debug
+            )
+        else:
+            logger.info('using transformers pipeline model')
+            self.llm_model = TransformersPipelineModel(
+                model_id=llm_model_id
+            )
+
+        prompt = PromptTemplate(
+            template=prompt_template,
+            input_variables=['question', 'context']
+        )
+        self.llm_chain = LLMChain(prompt=prompt, llm=self.llm_model)
+
+        if self.use_docs_for_context:
+            logger.info(f'Downloading {index_repo_id}')
+            snapshot_download(
+                repo_id=index_repo_id,
+                allow_patterns=['*.faiss', '*.pkl'], 
+                repo_type='dataset',
+                local_dir='indexes/run/'
+            )
+            logger.info('Loading embedding model')
+            embed_instruction = 'Represent the Hugging Face library documentation'
+            query_instruction = 'Query the most relevant piece of information from the Hugging Face documentation'
+            embedding_model = HuggingFaceInstructEmbeddings(
+                model_name=embedding_model_id,
+                embed_instruction=embed_instruction,
+                query_instruction=query_instruction
+            )
+            logger.info('Loading index')
+            self.knowledge_index = FAISS.load_local('./indexes/run/', embedding_model)
+            self.reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-12-v2')
+
+
+    def get_response(self, question: str, messages_context: str = '') -> Response:
+        """
+        Generate an answer to the specified question.
+
+        Args:
+            question (str): The question to be answered.
+            messages_context (str, optional): The context to be used for generating the answer. Defaults to ''.
+
+        Returns:
+            response (Response): The Response object containing the generated answer and the sources of information 
+            used to generate the response.
+        """
+
+        response = Response()
+        context = ''
+        relevant_docs = ''
+        if self.use_messages_for_context and messages_context:
+            messages_context = f'\nPrevious questions and answers:\n{messages_context}'
+            context += messages_context
+        if self.use_docs_for_context:
+            logger.info('Retriving documents')
+            # messages context is used for better retrival
+            retrival_query = messages_context + question 
+            relevant_docs = self.knowledge_index.similarity_search(
+                query=retrival_query,
+                k=self.first_stage_docs
+            )
+            cross_encoding_predictions = self.reranker.predict(
+                [(retrival_query, doc.page_content) for doc in relevant_docs]
+            )
+            relevant_docs = [
+                doc for _, doc in sorted(
+                    zip(cross_encoding_predictions, relevant_docs),
+                    reverse=True, key = lambda x: x[0]
+                )
+            ]
+            relevant_docs = relevant_docs[:self.num_relevant_docs]
+            context += '\nExtracted documents:\n'
+            context += ''.join([doc.page_content for doc in relevant_docs])
+            metadata = [doc.metadata for doc in relevant_docs]
+            response.set_sources(sources=[str(m['source']) for m in metadata])
+
+        logger.info('Running LLM chain')
+        answer = self.llm_chain.run(question=question, context=context)
+        response.set_answer(answer)
+        logger.info('Received answer')
+
+        if self.debug:
+            logger.info('\n' + '=' * 100)
+            sep = '\n' + '-' * 100
+            logger.info(f'question len: {len(question)} {sep}')
+            logger.info(f'question: {question} {sep}')
+            logger.info(f'answer len: {len(response.get_answer())} {sep}')
+            logger.info(f'answer: {response.get_answer()} {sep}')
+            logger.info(f'{response.get_sources_as_text()} {sep}')
+            logger.info(f'messages_contex: {messages_context} {sep}')
+            logger.info(f'relevant_docs: {relevant_docs} {sep}')
+            logger.info(f'context len: {len(context)} {sep}')
+            logger.info(f'context: {context} {sep}')
+        return response

qa_engine/response.py

@@ -0,0 +1,33 @@
+from typing import List
+
+
+class Response:
+    def __init__(self):
+        self.answer = ''
+        self.sources = []
+
+    def set_answer(self, answer: str) -> None:
+        self.answer = answer
+
+    def set_sources(self, sources: List) -> None:
+        self.sources = list(set(map(str, sources)))
+
+    def get_sources(self) -> List[str]:
+        return self.sources
+
+    def get_sources_as_text(self) -> str:
+        if not self.sources:
+            return ''
+        sources_text = '\n\nSources:'
+        for i, (source) in enumerate(self.sources):
+            sources_text += f'\n [{i+1}] {source}'
+        return sources_text
+
+    def get_answer(self, include_sources: bool = False) -> str:
+        answer = self.answer
+        if include_sources:
+            answer += self.get_sources_as_text()
+        return answer
+
+    def __str__(self):
+        return self.get_answer(include_sources=True)

questions.txt

@@ -0,0 +1,9 @@
+How to create audio dataset with Hugging Face?
+I want to check if 2 sentences are similar semantically. How can I do it?
+What are the benefits of Gradio?
+How to deploy a text-to-image model?
+Does Hugging Face offer any distributed training assistance? followup: Can you give me an example setup of it?
+I want to detect cars on video recording. How should I do it and what models do you recommend?
+Is there any tool for evaluating models in Hugging Face? followup: Can you give me an example setup of it?
+What are some advantages of the Hugging Face Hub?
+How would I use a model in 8 bit in transformers?

requirements.txt

@@ -0,0 +1,28 @@
+torch
+torchvision
+transformers
+accelerate
+einops
+huggingface_hub
+gradio
+beautifulsoup4==4.12.0
+discord.py==2.2.2
+evaluate==0.4.0
+fastapi==0.98.0
+langchain==0.0.154
+nltk==3.8.1
+nbconvert==7.6.0
+nbformat==5.9.0
+numpy==1.24.2
+markdown==3.4.4
+pandas==1.5.3
+python-dotenv==1.0.0
+Requests==2.29.0
+scikit_learn==1.2.2
+sentence-transformers==2.2.2
+InstructorEmbedding==1.0.0
+faiss_cpu==1.7.3
+tqdm==4.64.1
+uvicorn==0.22.0
+wandb==0.15.0
+pytest==7.3.1

run_docker.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker-compose down && docker-compose up --build

run_tests.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+pytest -o "testpaths=tests/" --noconftest -vv

tests/discord_bot/client/lorem_ipsum.txt

@@ -0,0 +1,14 @@
+Lorem ipsum dolor sit amet, ```consectetur adipiscing elit. Sed vitae arcu``` eros. Sed gravida tellus quis ante luctus, sed scelerisque mi auctor.
+Pellentesque consectetur fringilla turpis, in viverra odio posuere ut. In venenatis dui lectus, eget dignissim lectus lacinia et.
+Nulla commodo, nunc et vulputate vestibulum, mauris lorem semper nisl, a lacinia nulla urna ac lectus. Fusce pulvinar pulvinar augue vitae pulvinar. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Mauris ut elit nec erat tristique sagittis id in massa.
+Nunc ac libero in mauris consectetur bibendum non id nulla. Suspendisse feugiat ipsum metus, et auctor massa lobortis ac.
+Sed tincidunt dapibus sapien, ut tincidunt dui venenatis vitae. Praesent fermentum mi quis elit posuere, in consequat dui ultrices. Nullam tempus sapien non gravida faucibus.
+Etiam placerat magna risus, in dictum justo commodo eu. Fusce commodo viverra augue, non ullamcorper nunc efficitur et.
+Nam convallis dui vel nunc malesuada, at pharetra dui pulvinar. Cras commodo nibh et neque placerat eleifend. Mauris eget ante pharetra, mattis turpis ac, laoreet enim.
+Sed non sem sit amet dui venenatis posuere. Proin eget eros at nunc fringilla varius eu nec justo.
+Duis tristique consequat metus vitae euismod. Maecenas ullamcorper ullamcorper diam, nec tristique est congue nec. Suspendisse eu elit id massa fermentum pharetra in et mauris.
+In finibus metus nec odio aliquam, eu finibus ligula venenatis. Donec ullamcorper turpis eget sapien blandit volutpat. Sed tincidunt malesuada urna vitae cursus.
+Phasellus in est in purus lobortis dictum. Vivamus tristique turpis at lectus dapibus, ac volutpat arcu volutpat. Integer ac gravida felis.
+Donec laoreet neque non enim fermentum, vitae tincidunt est varius. Phasellus vehicula nisl nec metus dictum malesuada. Vestibulum sit amet erat et tellus euismod congue.
+Nulla facilisi. Mauris venenatis arcu vel elementum consectetur. Cras volutpat scelerisque sollicitudin.
+Fusce eu justo et magna gravida finibus id ut augue. Curabitur malesuada, justo ut semper dapibus, nulla urna maximus tellus, in vestibulum mauris justo id lectus. Vestibulum sit amet mauris feugiat, rhoncus ligula sed, finibus mi.

tests/discord_bot/client/test_utils.py

@@ -0,0 +1,69 @@
+import pytest
+import os
+from discord_bot.client.utils import ( \
+    find_max_split_index, \
+    find_max_split_index_from_sequence, \
+    split_text_into_chunks
+)
+
+
+@pytest.fixture(scope='module')
+def test_chunk() -> str:
+    return 't. , \n .'
+
+
+@pytest.fixture(scope='module')
+def test_text() -> str:
+    with open('tests/discord_bot/client/lorem_ipsum.txt', 'r') as f:
+        text = f.read()
+    assert text is not None, 'test text is empty'
+    return text
+
+
+def test_find_max_splitting_index(test_chunk: str):
+    index = find_max_split_index(test_chunk, char='\n')
+    assert index == 6, 'index should be 6'
+    index = find_max_split_index(test_chunk, char='. ')
+    assert index == 3, 'index should be 3'
+    index = find_max_split_index(test_chunk, char='.')
+    assert index == 8, 'index should be 8'
+
+
+def test_find_max_split_index_from_sequence(test_chunk: str):
+    index = find_max_split_index_from_sequence(
+        test_chunk,
+        split_characters=['\n']
+    )
+    assert index == 6, 'index should be 6'
+    index = find_max_split_index_from_sequence(
+        test_chunk,
+        split_characters=['.', ', ', '\n']
+    )
+    assert index == 8, 'index should be 8'
+
+
+def test_split_text_into_chunks_with_split_characters(test_text: str):
+    max_chunk_size = 250
+    chunks = split_text_into_chunks(
+        test_text,
+        split_characters=['. ', ', ', '\n'],
+        min_size=20,
+        max_size=max_chunk_size
+    )
+    for chunk in chunks:
+        assert len(chunk) > 0, 'Chunk length is zero'
+        assert len(chunk) <= max_chunk_size, 'Chunk length exceeds maximum limit'
+
+
+def test_split_text_into_chunks_without_split_characters():
+    test_text = 'a' * 1000
+    max_chunk_size = 250
+    chunks = split_text_into_chunks(
+        test_text,
+        split_characters=[],
+        min_size=20,
+        max_size=max_chunk_size
+    )
+    for chunk in chunks:
+        assert len(chunk) == max_chunk_size, \
+            'Chunk length is too small'

tests/index/test_index.py

@@ -0,0 +1,48 @@
+import pytest
+from typing import Any
+from huggingface_hub import snapshot_download
+from langchain.embeddings import HuggingFaceInstructEmbeddings
+from langchain.vectorstores import FAISS
+
+
+snapshot_download(
+    repo_id='KonradSzafer/index-large-notebooks',
+    allow_patterns=['*.faiss', '*.pkl'], 
+    repo_type='dataset',
+    local_dir='indexes/'
+)
+
+@pytest.fixture(scope='module')
+def embedding_model() -> HuggingFaceInstructEmbeddings:
+    model_name = 'hkunlp/instructor-large'
+    embed_instruction = 'Represent the Hugging Face library documentation'
+    query_instruction = 'Query the most relevant piece of information from the Hugging Face documentation'
+    return HuggingFaceInstructEmbeddings(
+        model_name=model_name,
+        embed_instruction=embed_instruction,
+        query_instruction=query_instruction,
+    )
+
+@pytest.fixture(scope='module')
+def index_path() -> str:
+    return 'indexes/'
+
+@pytest.fixture(scope='module')
+def index(embedding_model: HuggingFaceInstructEmbeddings, index_path: str):
+    return FAISS.load_local(index_path, embedding_model)
+
+@pytest.fixture(scope='module')
+def query() -> str:
+    return 'How to use the tokenizer?'
+
+def test_load_index(embedding_model: HuggingFaceInstructEmbeddings, index_path: str):
+    index = FAISS.load_local(index_path, embedding_model)
+    assert index is not None, 'Failed to load index'
+
+def test_index_page_content(index, query: str):
+    query_docs = index.similarity_search(query=query, k=3)
+    assert isinstance(query_docs[0].page_content, str)
+
+def test_index_metadata(index, query):
+    query_docs = index.similarity_search(query=query, k=3)
+    assert isinstance(query_docs[0].metadata['source'], str)

tests/qa_engine/test_response.py

@@ -0,0 +1,30 @@
+import os
+import pytest
+import importlib
+
+from qa_engine.response import Response
+
+
+def test_set_answer():
+    r = Response()
+    r.set_answer('Hello, World!')
+    assert r.get_answer() == 'Hello, World!'
+
+
+def test_set_sources():
+    r = Response()
+    r.set_sources(['source1', 'source1', 'source2'])
+    assert len(r.get_sources()) == 2
+
+
+def test_get_sources_as_text():
+    r = Response()
+    r.set_sources(['source1', 'source2'])
+    assert isinstance(r.get_sources_as_text(), str)
+
+
+def test_get_response_include_sources():
+    r = Response()
+    r.set_answer('Hello, World!')
+    r.set_sources(['source1', 'source2'])
+    assert len(r.get_answer(include_sources=True)) > len('Hello, World!')