Adding files from Github repository.

.envrc

@@ -0,0 +1,15 @@
+# -------------------- Defining default environment ---------------------------
+
+# --- Docker BuildKit
+export DOCKER_BUILDKIT_VALUE=1
+
+# --- Project variables
+export INPUT_APP_PORT=8501
+export OUTPUT_APP_PORT=8501
+export APP_SERVER_PORT=7860
+
+export HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN}
+export HUGGING_FACE_USERNAME=${HUGGING_FACE_USERNAME}
+
+export PATH="${PWD}:${PATH}"
+export PYTHONPATH="${PWD}:${PYTHONPATH}"

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/cicero_faiss_index.faiss filter=lfs diff=lfs merge=lfs -text
+data/clean_dataset.csv filter=lfs diff=lfs merge=lfs -text
+data/raw_dataset.csv filter=lfs diff=lfs merge=lfs -text

.github/workflows/code-linting.yml

@@ -0,0 +1,22 @@
+name: Project CICD
+run-name: ${{ github.actor }} - CICD
+on: [push]
+
+jobs:
+  #
+  # --- Code-linting
+  lint-code:
+    runs-on: ubuntu-latest
+    steps:
+      # Checkout repository
+      - uses: actions/checkout@v3
+      # Install Python
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      # Install python dependencies
+      - name: Install dependencies
+        run: |
+          make requirements
+          make pre-commit-install
+          make lint

.gitignore

@@ -0,0 +1,168 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+.envrc
+.python-version
+data/
+
+src/utils/gpt35_summaries/df_embed.csv
+src/utils/gpt35_summaries/df_embed_out2.csv
+src/utils/gpt35_summaries/words_alpha.txt

.isort.cfg

@@ -0,0 +1,5 @@
+[settings]
+line_length = 79
+multi_line_output = 3
+include_trailing_comma = True
+known_third_party =datasets,fastapi,gradio,huggingface_hub,numpy,pandas,pydantic,requests,sentence_transformers,spacy,tiktoken,torch,utils

.pre-commit-config.yaml

@@ -0,0 +1,57 @@
+# Created       : 2023-03-01
+# Last Modified : 2023-03-04
+#
+# Description
+#    This file summarizes the set of checks that pre-commit will perform
+#    prior to any commit.
+
+default_stages: [commit, manual]
+
+# Repositories to use
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml # Checks yaml files for parseable syntax.
+      - id: check-json # Checks json files for parseable syntax.
+      - id: check-added-large-files
+      - id: check-toml
+      - id: check-docstring-first
+      - id: check-case-conflict # Check for files that would conflict in case-insensitive filesystems
+      - id: check-merge-conflict # Check for files that contain merge conflict strings.
+      - id: debug-statements # Check for debugger imports and py37+ `breakpoint()` calls in python source.
+  - repo: https://github.com/pycqa/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8
+        language_version: python3.9
+        exclude: >
+          (?x)^(
+            src/focused_summary_example.py
+            )
+
+  - repo: https://github.com/ambv/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        language_version: python3.9
+
+  - repo: https://github.com/asottile/seed-isort-config
+    rev: v2.2.0
+    hooks:
+      - id: seed-isort-config
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+        name: isort (python)
+        exclude: hooks.py
+      - id: isort
+        name: isort (cython)
+        types: [cython]
+      - id: isort
+        name: isort (pyi)
+        types: [pyi]

.python-version

@@ -0,0 +1 @@
+Cicero_LLM_Synthesizer

.tmuxgo

@@ -0,0 +1,32 @@
+#!/bin/bash
+SESSION=`basename $PWD`
+ENVNAME='ml'
+
+# Creating new session
+tmux -2 new-session -d -s $SESSION `cd $PWD`
+
+# ------ Main window ------
+# Renaming window
+tmux rename-window -t $SESSION:0 main
+# Splitting panes and windows
+tmux split-window -v
+tmux select-pane -t 0
+tmux resize-pane -D 15
+tmux select-pane -t 1
+# Sending commands
+tmux send -t $SESSION:0.1 "cd $PWD; conda activate $ENVNAME; source $SHELL ;clear; htop" ENTER
+#
+# ------ Miscellaneous window ------
+tmux new-window -t $SESSION:1 -n 'misc'
+tmux send -t $SESSION:1.0 "cd $PWD; conda activate $ENVNAME; source $SHELL ; clear;" ENTER
+# ------ Extras window ------
+tmux new-window -t $SESSION:2 -n 'extras'
+tmux send -t $SESSION:2.0 "cd $PWD; conda activate $ENVNAME; source $SHELL ; clear;" ENTER
+# ------ Jupyter window ------
+tmux new-window -t $SESSION:3 -n 'jupyter'
+tmux send -t $SESSION:3.0 "cd $PWD; conda activate $ENVNAME; source $SHELL ; clear;" ENTER
+#
+# Selecting which window to start at
+tmux select-window -t $SESSION:0
+tmux select-pane -t 0
+tmux -2 attach -t $SESSION

Dockerfile

@@ -0,0 +1,104 @@
+ARG PYTHON_VERSION="3.9.13"
+ARG PLATFORM_NAME="linux/amd64"
+
+FROM --platform=${PLATFORM_NAME} python:${PYTHON_VERSION}
+
+# --- SYSTEM ARCHITECTURE
+ARG TARGETPLATFORM
+ARG TARGETARCH
+ARG TARGETVARIANT
+
+RUN printf "I'm building for TARGETPLATFORM=${TARGETPLATFORM}" \
+    && printf ", TARGETARCH=${TARGETARCH}" \
+    && printf ", TARGETVARIANT=${TARGETVARIANT} \n" \
+    && printf "With uname -s : " && uname -s \
+    && printf "and  uname -m : " && uname -mm
+
+# --- Environment variables
+ENV REQUIREMENTS_FILE="requirements.txt"
+ENV OUTDIR="/root"
+ENV PROJECT_DIR="/opt/ml"
+ENV PROGRAM_DIR="/opt/program"
+ENV HOME_DIR="/root/ml"
+ENV LOCAL_DEV_DIR="docker"
+ENV ALIASES_FILE="/root/aliases.sh"
+ENV DEBIAN_FRONTEND=noninteractive
+
+# --- Dockerfile Metadata
+LABEL Maintainer="Victor Calderon"
+
+# ------------------------- COPYING AND DIRECTORIES ---------------------------
+
+RUN mkdir -p ${HOME_DIR}
+
+COPY ./src ${PROJECT_DIR}/src
+COPY ${LOCAL_DEV_DIR}/aliases.sh ${ALIASES_FILE}
+
+COPY ${REQUIREMENTS_FILE} "${HOME_DIR}/${REQUIREMENTS_FILE}"
+
+# ---------------------- EXPOSING PORTS FOR APP -------------------------------
+
+EXPOSE 7860
+EXPOSE 8501
+
+# --------------------- INSTALLING EXTRA PACKAGES -----------------------------
+# --- Updating packages and installing packages at the system-level
+
+RUN apt-get -y update && \
+    apt-get upgrade -y && \
+    apt-get clean && \
+    # Instaling system-level packages
+    apt-get install -y \
+    git \
+    ssh \
+    tree \
+    git-flow \
+    tmux \
+    direnv \
+    bash-completion \
+    zsh \
+    htop \
+    vim \
+    && \
+    # Cleaning out
+    rm -rf /var/lib/apt/lists/* && \
+    # Cleaning installs
+    apt-get clean && \
+    # Installing ZSH and OhZSH
+    sh -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)" && \
+    echo "source /etc/profile.d/bash_completion.sh" >> /root/.bashrc && \
+    echo "source /etc/profile.d/bash_completion.sh" >> /root/.zshrc && \
+    echo "source /root/aliases.sh" >> "${OUTDIR}/.zshrc" && \
+    echo "source /root/aliases.sh" >> "${OUTDIR}/.bashrc" && \
+    # Install direnv
+    echo 'eval "$(direnv hook zsh)"' >> "${OUTDIR}/.zshrc" && \
+    echo 'eval "$(direnv hook bash)"' >> "${OUTDIR}/.bash"
+
+# -------------------------- DOCKER-SPECIFIC ----------------------------------
+
+RUN apt-get update -y && \
+    cd ${OUTDIR_DOCKER} && \
+    curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh
+
+# --------------------------- PYTHON-RELATED-LOCAL ----------------------------
+
+RUN pip install --upgrade pip && \
+    python -m pip install -r "${HOME_DIR}/${REQUIREMENTS_FILE}"
+
+# ----------------------------- PYTHON-SPECIFIC -------------------------------
+
+# Set some environment variables. PYTHONUNBUFFERED keeps Python from
+# buffering our standard output stream, which means that logs can be
+# delivered to the user quickly. PYTHONDONTWRITEBYTECODE keeps Python
+# from writing the .pyc files which are unnecessary in this case. We also
+# update PATH so that the train and serve programs are found when the
+# container is invoked.
+
+ENV PYTHONUNBUFFERED=TRUE
+ENV PYTHONDONTWRITEBYTECODE=TRUE
+ENV PATH="${PROGRAM_DIR}:${PATH}"
+ENV PYTHONPATH="${PROGRAM_DIR}:${PYTHONPATH}"
+
+WORKDIR ${PROJECT_DIR}
+
+CMD ["uvicorn", "src.api.index:app", "--host", "0.0.0.0","--port", "7860"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Maven-Building-LLMS-into-Production
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

LICENSE.rst

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Victor Calderon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,485 @@
+.PHONY:	show-params
+
+###############################################################################
+# GLOBALS                                                                     #
+###############################################################################
+
+PROJECT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+PROJECT_NAME := $(shell basename $(subst -,_,$(PROJECT_DIR)))
+PROJECT_NAME_LOWER := $(shell echo $(PROJECT_NAME) | tr  '[:upper:]' '[:lower:]')
+ENVIRONMENT_NAME = $(PROJECT_NAME)
+PYTHON_INTERPRETER = python3
+PIP_INTERPRETER = pip
+PYTHON_VERSION = 3.9
+PIP_VERSION = 22.3
+
+# --- REQUIREMENTS-RELATED
+REQUIREMENTS_FILE = $(PROJECT_DIR)/requirements.txt
+REQUIREMENTS_FILE_TEMP = $(PROJECT_DIR)/requirements.tmp
+REQUIREMENTS_DEV_FILE = $(PROJECT_DIR)/requirements-dev.txt
+REQUIREMENTS_DEV_FILE_TEMP = $(PROJECT_DIR)/requirements-dev.tmp
+REQUIREMENTS_DEPLOYMENT_FILE = $(PROJECT_DIR)/requirements-deploy.txt
+REQUIREMENTS_DEPLOYMENT_FILE_TEMP = $(PROJECT_DIR)/requirements-deploy.tmp
+
+# --- PATHS TO PROJECT DIRECTOIRES
+DATA_DIRECTORY = $(PROJECT_DIR)/data
+SRC_DIRECTORY = $(PROJECT_DIR)/src
+API_DIRECTORY = $(SRC_DIRECTORY)/api
+DATA_PROCESSING_DIRECTORY = $(SRC_DIRECTORY)/data_processing
+TRAINING_DIRECTORY = $(SRC_DIRECTORY)/training
+
+# -- Docker-related
+# Variable used for turning on/off Docker Buildkit
+DOCKER_BUILDKIT_VALUE=1
+LOCAL_DEVELOPMENT_DIR_PATH="$(PROJECT_DIR)/docker"
+
+# -- API-related
+INPUT_APP_PORT=8501
+OUTPUT_APP_PORT=8501
+API_WEBSERVER_URL="http://localhost:$(INPUT_APP_PORT)"
+
+# -- App-related
+APP_SERVER_PORT=7860
+APP_WEBSERVER_URL="http://localhost:$(APP_SERVER_PORT)"
+
+# ----------------------------- Python-specific -------------------------------
+# - Checking what type of python one is using
+# Anaconda
+ifeq (,$(shell which conda))
+HAS_CONDA=False
+else
+HAS_CONDA=True
+# We need to specify the following commands in order to properly activate the
+# Anaconda environment.
+SHELL=/bin/bash
+# Note that the extra activate is needed to ensure that the activate floats env to the front of PATH
+CONDA_ACTIVATE=source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate
+CONDA_DEACTIVATE=source $$(conda info --base)/etc/profile.d/conda.sh ; conda deactivate ; conda deactivate
+endif
+
+# - Pyenv
+ifeq (,$(shell which pyenv))
+HAS_PYENV=False
+else
+HAS_PYENV=True
+endif
+
+###############################################################################
+#  VARIABLES FOR COMMANDS                                                     #
+###############################################################################
+
+## Show the set of input parameters
+show-params:
+	@ printf "\n-------- GENERAL ---------------\n"
+	@ echo "PROJECT_DIR:                       $(PROJECT_DIR)"
+	@ echo "PROJECT_NAME:                      $(PROJECT_NAME)"
+	@ echo "LOCAL_DEVELOPMENT_DIR_PATH:        $(LOCAL_DEVELOPMENT_DIR_PATH)"
+	@ echo "ENVIRONMENT_NAME:                  $(ENVIRONMENT_NAME)"
+	@ echo "PYTHON_INTERPRETER:                $(PYTHON_INTERPRETER)"
+	@ echo "PYTHON_VERSION:                    $(PYTHON_VERSION)"
+	@ echo "PIP_VERSION:                       $(PIP_VERSION)"
+	@ echo "REQUIREMENTS_FILE:                 $(REQUIREMENTS_FILE)"
+	@ echo "REQUIREMENTS_FILE_TEMP:            $(REQUIREMENTS_FILE_TEMP)"
+	@ echo "REQUIREMENTS_DEV_FILE:             $(REQUIREMENTS_DEV_FILE)"
+	@ echo "REQUIREMENTS_DEV_FILE_TEMP:        $(REQUIREMENTS_DEV_FILE_TEMP)"
+	@ echo "REQUIREMENTS_DEPLOYMENT_FILE:      $(REQUIREMENTS_DEPLOYMENT_FILE)"
+	@ echo "REQUIREMENTS_DEPLOYMENT_FILE_TEMP: $(REQUIREMENTS_DEPLOYMENT_FILE_TEMP)"
+	@ printf "\n-------- DOCKER ---------------\n"
+	@ echo "DOCKER_BUILDKIT_VALUE:             $(DOCKER_BUILDKIT_VALUE)"
+	@ printf "\n-------- PYTHON ---------------\n"
+	@ echo "HAS_CONDA:                         $(HAS_CONDA)"
+	@ echo "HAS_PYENV:                         $(HAS_PYENV)"
+	@ printf "\n-------- LOCAL DEVELOPMENT ---------------\n"
+	@ echo "LOCAL_DEV_DOCKER_PROJECT_NAME:     $(LOCAL_DEV_DOCKER_PROJECT_NAME)"
+	@ echo "LOCAL_DEV_SERVICE_NAME:            $(LOCAL_DEV_SERVICE_NAME)"
+	@ printf "\n-------- API ---------------\n"
+	@ echo "APP_PORT:                          $(APP_PORT)"
+	@ echo "APP_WEBSERVER_URL:                 $(APP_WEBSERVER_URL)"
+	@ echo "API_SERVICE_NAME:                  $(API_SERVICE_NAME)"
+	@ echo "API_DOCKER_PROJECT_NAME:           $(API_DOCKER_PROJECT_NAME)"
+	@ printf "\n-----------------------\n"
+
+## Initialize the repository for code development
+init: clean create-envrc delete-environment create-environment
+ifeq (True,$(HAS_CONDA))
+	@ ($(CONDA_ACTIVATE) $(ENVIRONMENT_NAME) ; $(MAKE) requirements)
+	@ printf "\n\n>>> New Conda environment created. Activate with: \n\t: conda activate $(ENVIRONMENT_NAME)"
+	@ $(MAKE) show-params
+	@ printf "\n\n>>> Project initialized!"
+	@ ($(CONDA_ACTIVATE) $(ENVIRONMENT_NAME) ; $(MAKE) pre-commit-install )
+	@ ($(CONDA_ACTIVATE) $(ENVIRONMENT_NAME) ; $(MAKE) lint )
+else
+	@ direnv allow || echo ""
+	@ echo ">>> Continuing installation ..."
+	@ $(MAKE) requirements
+	@ $(MAKE) show-params
+	@ printf "\n\n>>> Project initialized!\n"
+	@ $(MAKE) pre-commit-install
+	@ $(MAKE) lint
+endif
+
+## Remove ALL of the artifacts + Python environments
+destroy: clean pre-commit-uninstall delete-environment
+	@ echo ">>> Deleted all artifacts and environments!"
+
+###############################################################################
+# MISCELLANEOUS COMMANDS                                                      #
+###############################################################################
+
+# -------------------- Functions for cleaning repository ----------------------
+
+## Removes artifacts from the build stage, and other common Python artifacts.
+clean: clean-build clean-pyc clean-test clean-secrets clean-model-files clean-images
+
+## Removes Python file artifacts
+clean-pyc:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+## Remove build artifacts
+clean-build:
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+
+## Remove test and coverage artifacts
+clean-test:
+	rm -fr .tox/
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -fr .pytest_cache
+
+## Remove files related to pre-trained models
+clean-model-files:
+	find . -name '*.pt' -exec rm -fr {} +
+	find . -name "runs" -type d -exec rm -rf {} + || echo ""
+
+## Clean left-over images
+clean-images:
+	find . -name '*.png' -exec rm -fr {} +
+	find . -name '*.jpg' -exec rm -fr {} +
+
+## Removes secret artifacts - Serverless
+clean-secrets:
+	find . -name "node_modules" -type d -exec rm -rf {} + || echo ""
+	find . -name ".serverless" -type d -exec rm -rf {} + || echo ""
+
+# ---------------------- Functions for local environment ----------------------
+
+## Set up the envrc file for the project.
+create-envrc:
+	@ echo "cat $(PROJECT_DIR)/template.envrc > $(PROJECT_DIR)/.envrc"
+	@ cat $(PROJECT_DIR)/template.envrc > $(PROJECT_DIR)/.envrc
+
+## Delete the local envrc file of the project
+delete-envrc:
+	@ rm -rf $(PROJECT_DIR)/.envrc || echo ""
+
+## Install git-flow
+git-flow-install:
+	@	(( if [[ ! -f "`which git-flow`" ]]; then \
+		echo "No Git-flow installed"! ; \
+			if [[ -f "`which brew`"  ]]; then \
+			echo "Homebrew installed"; \
+			HOMEBREW_NO_AUTO_UPDATE=1 brew install git-flow; \
+			elif [[ -f "`which apt-get`"  ]]; then \
+			echo "Apt-get installed"; \
+			apt-get install git-flow; \
+			else \
+			echo "Could not locate package manager! (brew or apt-get)"; \
+			fi; \
+		fi ) && git flow init -f -d) || echo "Git-Flow setup could not be completed"
+
+
+# ---------------------- Functions for Python environment ---------------------
+
+## Creates the Python environment
+create-environment:
+ifeq (True,$(HAS_CONDA))
+	@ echo ">>> Detected CONDA ... Creating new conda environment!"
+	@ echo ">>> \tCreating environment: \t $(ENVIRONMENT_NAME)"
+	@ conda create --name $(ENVIRONMENT_NAME) python=$(PYTHON_VERSION) -y  || echo ""
+	@ echo ">>> New conda environment created. Activate with: \n conda activate $(ENVIRONMENT_NAME)"
+else ifeq (True,$(HAS_PYENV))
+	@ echo ">>> Detected PYENV ... Creating new Pyenv environment!"
+	@ echo ">>> \tCreating environment: \t $(ENVIRONMENT_NAME)"
+	@ pyenv virtualenv $(PYTHON_VERSION) $(ENVIRONMENT_NAME) || echo ""
+	@ pyenv local $(ENVIRONMENT_NAME)
+	@ echo ">>> New Pyenv environment created: '$(ENVIRONMENT_NAME)'"
+	@ pyenv virtualenvs
+	@ echo
+endif
+
+## Deletes the Python environment
+delete-environment:
+ifeq (True,$(HAS_CONDA))
+	@ echo ">>> Detected CONDA ... Deleting Conda environment, if applicable!"
+	@ echo ">>> Deleting environment:    '$(ENVIRONMENT_NAME)'"
+	@ ($(CONDA_DEACTIVATE) ; conda env remove --name $(ENVIRONMENT_NAME) -y) || echo ""
+	@ echo ">>> Conda environment deleted: '$(ENVIRONMENT_NAME)'"
+else ifeq (True,$(HAS_PYENV))
+	@ echo ">>> Detected PYENV ... Deleting Pyenv environment!"
+	@ echo ">>> Deleting environment:    '$(ENVIRONMENT_NAME)'"
+	@ pyenv uninstall -f $(ENVIRONMENT_NAME) || echo ""
+	@ rm $(PROJECT_DIR)/.python-version || echo ""
+	@ echo ">>> Pyenv environment deleted: '$(ENVIRONMENT_NAME)'"
+	@ pyenv virtualenvs
+	@ echo
+endif
+
+## Upgrade the version of the 'pip' package
+pip-upgrade:
+	@ $(PYTHON_INTERPRETER) -m pip install --no-cache-dir -q --upgrade pip==$(PIP_VERSION)
+
+## Sort the project packages requirements file
+sort-requirements:
+	@ 	sort $(REQUIREMENTS_FILE) | grep "\S" > $(REQUIREMENTS_FILE_TEMP) && \
+		mv $(REQUIREMENTS_FILE_TEMP) $(REQUIREMENTS_FILE)
+	@ 	sort $(REQUIREMENTS_DEV_FILE) | grep "\S" > $(REQUIREMENTS_DEV_FILE_TEMP) && \
+		mv $(REQUIREMENTS_DEV_FILE_TEMP) $(REQUIREMENTS_DEV_FILE)
+	@ 	sort $(REQUIREMENTS_DEPLOYMENT_FILE) | grep "\S" > $(REQUIREMENTS_DEPLOYMENT_FILE_TEMP) && \
+		mv $(REQUIREMENTS_DEPLOYMENT_FILE_TEMP) $(REQUIREMENTS_DEPLOYMENT_FILE)
+
+
+## Install Python dependencies into the Python environment
+requirements: pip-upgrade sort-requirements
+	@ $(PYTHON_INTERPRETER) -m pip install --no-cache-dir -q -r $(REQUIREMENTS_DEV_FILE)
+
+# -------------------------- Functions for Code Linting -----------------------
+
+## Installing the pre-commit Git hook
+pre-commit-install:
+	@ pre-commit install
+
+## Uninstall the pre-commit Git hook
+pre-commit-uninstall:
+	@ pre-commit uninstall
+
+## Run the 'pre-commit' linting step manually
+lint:
+	@ pre-commit run -a --hook-stage manual
+
+
+###############################################################################
+# Docker Commands - Local development                                         #
+###############################################################################
+
+LOCAL_DEV_DOCKER_PROJECT_NAME="$(PROJECT_NAME_LOWER)_localdev_dind"
+LOCAL_DEV_SERVICE_NAME="local-dev"
+
+## Clean Docker images
+docker-prune:
+	@	docker system prune -f
+
+## Stops both the API service and the local development service
+all-stop: api-stop docker-local-dev-stop app-stop
+	@	echo "All services are down"
+
+## Starts both the API service and the local development service
+all-start: api-start docker-local-dev-start app-start
+	@	echo "All services are up!"
+
+## Build local development Docker image
+docker-local-dev-build: docker-prune
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(LOCAL_DEV_DOCKER_PROJECT_NAME) \
+		build $(LOCAL_DEV_SERVICE_NAME)
+
+## Start service for local development
+docker-local-dev-start: docker-local-dev-build docker-local-dev-stop
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(LOCAL_DEV_DOCKER_PROJECT_NAME) \
+		up -d $(LOCAL_DEV_SERVICE_NAME)
+
+## Stop service for local development
+docker-local-dev-stop:
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(LOCAL_DEV_DOCKER_PROJECT_NAME) \
+		stop $(LOCAL_DEV_SERVICE_NAME)
+	@	$(MAKE) docker-prune
+
+## Start a shell session into the docker container
+docker-local-dev-login:
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(LOCAL_DEV_DOCKER_PROJECT_NAME) \
+		exec \
+		$(LOCAL_DEV_SERVICE_NAME) /bin/zsh
+
+###############################################################################
+# Docker Commands - API-related                                               #
+###############################################################################
+
+API_DOCKER_PROJECT_NAME="$(PROJECT_NAME_LOWER)_api"
+API_SERVICE_NAME="api"
+
+## Build API Docker image
+api-build: docker-prune
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(API_DOCKER_PROJECT_NAME) \
+		build $(API_SERVICE_NAME)
+
+## Start API Docker image container
+api-start: api-stop api-build
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(API_DOCKER_PROJECT_NAME) \
+		up -d $(API_SERVICE_NAME)
+
+## Stop API Docker image container
+api-stop:
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(API_DOCKER_PROJECT_NAME) \
+		stop $(API_SERVICE_NAME)
+	@	$(MAKE) docker-prune
+
+## Open API in web browser
+api-web:
+	@	python -m webbrowser "$(API_WEBSERVER_URL)/docs"
+
+###############################################################################
+# Docker Commands - App-related                                               #
+###############################################################################
+
+APP_DOCKER_PROJECT_NAME="$(PROJECT_NAME_LOWER)_app"
+APP_SERVICE_NAME="app"
+
+## Build App Docker image
+app-app-build: docker-prune
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(APP_DOCKER_PROJECT_NAME) \
+		build $(APP_SERVICE_NAME)
+
+## Start App Docker image container
+app-app-start: app-app-stop app-app-build
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(APP_DOCKER_PROJECT_NAME) \
+		up -d $(APP_SERVICE_NAME)
+
+## Stop App Docker image container
+app-app-stop:
+	@	cd $(LOCAL_DEVELOPMENT_DIR_PATH) && \
+		docker compose \
+		--project-name $(APP_DOCKER_PROJECT_NAME) \
+		stop $(APP_SERVICE_NAME)
+	@	$(MAKE) docker-prune
+
+## Open App in web browser
+app-app-web:
+	@	python -m webbrowser "$(APP_WEBSERVER_URL)"
+
+###############################################################################
+# Unit Tests and Code checking                                                #
+###############################################################################
+
+# See: https://github.com/google/addlicense for more information
+## Add licenses to Python files
+add-licenses:
+	@	docker run -it \
+		-v ${PWD}:/src \
+		ghcr.io/google/addlicense \
+		-f ./LICENSE.rst  \
+		./src/**/*.py
+
+## Open up all web endpoints
+all-web: api-web app-app-web
+	@	echo "All web endpoints opened!"
+
+###############################################################################
+# PROJECT AND DATA FUNCTIONS                                                  #
+###############################################################################
+
+DATASET_PATH="https://raw.githubusercontent.com/hamzafarooq/maven-mlsystem-design-cohort-1/main/data/df_embed.csv"
+DATASET_WITH_SUMMARIES_NAME="cicero_dataset_with_summaries"
+DATASET_WITH_FAISS_AND_EMBEDDINGS_NAME="cicero_dataset_with_embeddings_and_faiss_index"
+HUGGING_FACE_REPOSITORY_NAME="cicero_synthesizer"
+FAISS_OUTPUT_FILENAME="cicero_faiss_index"
+
+## Run the data preparation on the input dataset
+prepare_data:
+	@	$(PYTHON_INTERPRETER) \
+		$(DATA_PROCESSING_DIRECTORY)/prepare_dataset.py \
+		--dataset-path $(DATASET_PATH)
+
+
+## Run the script for creating a FAISS index and text embeddings of the dataset
+run_faiss_and_embeddings:
+	@	$(PYTHON_INTERPRETER) \
+		$(TRAINING_DIRECTORY)/create_faiss_corpus_index.py \
+		--dataset-name $(DATASET_WITH_SUMMARIES_NAME) \
+		--output-dataset-name $(DATASET_WITH_FAISS_AND_EMBEDDINGS_NAME) \
+		--repository-name $(HUGGING_FACE_REPOSITORY_NAME) \
+		--faiss-index-name $(FAISS_OUTPUT_FILENAME)
+
+
+
+###############################################################################
+# Self Documenting Commands                                                   #
+###############################################################################
+
+.DEFAULT_GOAL := help
+
+# Inspired by <http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html>
+# sed script explained:
+# /^##/:
+# 	* save line in hold space
+# 	* purge line
+# 	* Loop:
+# 		* append newline + line to hold space
+# 		* go to next line
+# 		* if line starts with doc comment, strip comment character off and loop
+# 	* remove target prerequisites
+# 	* append hold space (+ newline) to line
+# 	* replace newline plus comments by `---`
+# 	* print line
+# Separate expressions are necessary because labels cannot be delimited by
+# semicolon; see <http://stackoverflow.com/a/11799865/1968>
+help:
+	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
+	@echo
+	@sed -n -e "/^## / { \
+		h; \
+		s/.*//; \
+		:doc" \
+		-e "H; \
+		n; \
+		s/^## //; \
+		t doc" \
+		-e "s/:.*//; \
+		G; \
+		s/\\n## /---/; \
+		s/\\n/ /g; \
+		p; \
+	}" ${MAKEFILE_LIST} \
+	| LC_ALL='C' sort --ignore-case \
+	| awk -F '---' \
+		-v ncol=$$(tput cols) \
+		-v indent=25 \
+		-v col_on="$$(tput setaf 6)" \
+		-v col_off="$$(tput sgr0)" \
+	'{ \
+		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
+		n = split($$2, words, " "); \
+		line_length = ncol - indent; \
+		for (i = 1; i <= n; i++) { \
+			line_length -= length(words[i]) + 1; \
+			if (line_length <= 0) { \
+				line_length = ncol - indent - length(words[i]) - 1; \
+				printf "\n%*s ", -indent, " "; \
+			} \
+			printf "%s ", words[i]; \
+		} \
+		printf "\n"; \
+	}' \
+	| more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')

README.md

@@ -1,11 +1,282 @@
----
-title: Cicero Synthesizer Space
-emoji: ⚡
-colorFrom: purple
-colorTo: pink
-sdk: docker
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Maven-Building-LLMS-into-Production/Cicero-LLM-Synthesizer/code-linting.yml)
+
+# Cicero LLM Synthesizer
+
+## Contents
+
+- [Setup](#setup)
+- [Setup for local code development](#setup-for-local-code-development)
+  - [Makefile](#makefile)
+  - [Starting up the Docker container and initializing the repository](#starting-up-the-docker-container-and-initializing-the-repository)
+  - [Starting the API service](#starting-the-api-service)
+  - [Starting up all the services](#starting-up-all-the-services)
+- [Tests](#tests)
+- [Helpful Commands](#helpful-commands)
+- [VS Code Extensions](#vs-code-extensions)
+- [GPT3.5 summaries](#gpt35-summaries)
+- [Resources](#resources)
+
+## Setup
+
+Ensure you have python and pip installed.
+
+```shell
+python --version
+pip --version
+```
+
+From the root directory run the following command to install the
+dependencies: `pip install -r requirements.txt`
+
+You can run the app using this command: `python -m uvicorn src.api.index:app --reload`
+
+Once running you can navigate to `http://127.0.0.1:8000/docs` to view the
+interactive API documentation.
+
+## Setup for local code development
+
+There are some steps that need to be done prior to being able to
+properly run and develop the code in this repository.
+
+The following is a list of steps that have to happen prior to starting to
+work / test the pipelines of this repository:
+
+### Makefile
+
+The project comes with a `Makefile` (**not supported in Windows!**)
+that can be used for executing commands that will make the interaction
+with this project much smoother. Keep in mind that folders with spaces in their names may cause issues.
+
+One can see all of the available options by:
+
+```bash
+    $: make
+
+    Available rules:
+
+    add-licenses              Add licenses to Python files
+    all-start                 Starts both the API service and the local development service
+    all-stop                  Stops both the API service and the local development service
+    all-web                   Open up all web endpoints
+    api-build                 Build API Docker image
+    api-start                 Start API Docker image container
+    api-stop                  Stop API Docker image container
+    api-web                   Open API in web browser
+    app-app-build             Build App Docker image
+    app-app-start             Start App Docker image container
+    app-app-stop              Stop App Docker image container
+    app-app-web               Open App in web browser
+    clean                     Removes artifacts from the build stage, and other common Python artifacts.
+    clean-build               Remove build artifacts
+    clean-images              Clean left-over images
+    clean-model-files         Remove files related to pre-trained models
+    clean-pyc                 Removes Python file artifacts
+    clean-secrets             Removes secret artifacts - Serverless
+    clean-test                Remove test and coverage artifacts
+    create-environment        Creates the Python environment
+    create-envrc              Set up the envrc file for the project.
+    delete-environment        Deletes the Python environment
+    delete-envrc              Delete the local envrc file of the project
+    destroy                   Remove ALL of the artifacts + Python environments
+    docker-local-dev-build    Build local development Docker image
+    docker-local-dev-login    Start a shell session into the docker container
+    docker-local-dev-start    Start service for local development
+    docker-local-dev-stop     Stop service for local development
+    docker-prune              Clean Docker images
+    git-flow-install          Install git-flow
+    init                      Initialize the repository for code development
+    lint                      Run the 'pre-commit' linting step manually
+    pip-upgrade               Upgrade the version of the 'pip' package
+    pre-commit-install        Installing the pre-commit Git hook
+    pre-commit-uninstall      Uninstall the pre-commit Git hook
+    prepare_data              Run the data preparation on the input dataset
+    requirements              Install Python dependencies into the Python environment
+    run_faiss_and_embeddings  Run the script for creating a FAISS index and text embeddings of the dataset
+    show-params               Show the set of input parameters
+    sort-requirements         Sort the project packages requirements file
+```
+
+> **NOTE**: If you're using `Windows`, you may have to copy and modify to some
+> extents the commands that are part of the `Makefile` for some tasks.
+
+### Starting up the Docker container and initializing the repository
+
+In order to work on current / new features, one can use *Docker* to
+start a new container and start the local development process.
+
+To build the Docker image, one must follow the following steps:
+
+1. Start the Docker daemon. If you're using Mac, one can use the
+Docker Desktop App.
+2. Go the project's directory and run the following command using the `Makefile`:
+```bash
+# Go the project's directory
+cd /path/to/directory
+
+# Build the Docker iamge and start a container
+make docker-local-dev-start
+```
+3. Log into the container
+```bash
+# Log into the container
+make docker-local-dev-login
+```
+
+4. Once you're inside the container, you'll see the following prompt:
+
+```bash
+# Log into the container
+???$: make docker-local-dev-login
+direnv: error /opt/program/.envrc is blocked. Run `direnv allow` to approve its content
+```
+> One will see the `direnv` error because `direnv` is installed and one must
+> *allow* the changes to take effect.
+
+5. Allow for the `direnv` changes
+```bash
+# Accept the changes
+$: direnv allow
+direnv: loading /opt/program/.envrc
+```
+
+6. The last thing is to initialize the repository. This can easily be done
+with the `init` command:
+
+```bash
+$: make init
+```
+This will do the following tasks:
+- Clean Python files
+- Initialize the `.envrc` file used by `direnv`.
+- Delete an existing python environment for the project, if it exists.
+- Creates a new environment, if applicable
+- Apply `direnv allow` to allow for `direnv` modifications.
+- Install package requirements via `pip`
+- Install `pre-commit` for code-linting and code-checking.
+- Install `git-flow`, whenever possible.
+
+These steps allow for the user to be able to develop new feature within
+Docker, which makes it easier for developers to have the exact same set of
+tools available.
+
+## Starting the API service
+
+The project comes with an out-of-the-box solution for starting and stopping
+the API endpoint via Docker.
+
+To start the container with the API endpoint, one must run the following
+command:
+
+```bash
+# Start API service
+make api-start
+```
+
+This service will start a Docker container that exposes the internal port
+`7860` to the local host's port `7860`. Once the image has been built and
+a container has started, one can go to the service's main page by using
+the following command:
+
+```bash
+# Go the URL of the API endpoint
+make api-web
+```
+
+> This will direct the user to the following URL:
+> [http://localhost:7860/docs](http://localhost:7860/docs)
+
+In order to *stop* the API service, one can run the following command:
+
+```bash
+# Stop the API service
+make api-stop
+```
+
+As one customizes the FastAPI with new features and more, these changes
+will be automatically displayed in the URL from above.
+
+### Starting up all the services
+
+Similar to the sections from above, one can spin up or spin down all the
+services at once with the help of 2 commands, i.e. `all-start` and `all-stop`.
+
+In order to spin up both the *api* service and that for *local development*,
+one can run:
+
+```bash
+make all-start
+```
+
+This command will execute both services and one will be able to log into the
+container for local development, as well to connect to the API via the
+browser.
+
+Similarly, in order to spin down all of the services, one can simply run:
+
+```bash
+make all-stop
+```
+
+This will stop both services and delete any unused Docker containers.
+
+## Tests
+
+Unit tests can be found under the `src` folder alongside source code.
+Test files end with `_test`. The following command will run all of the tests.
+
+```shell
+python -m pytest -v -s
+```
+
+The `-v` argument is for verbose output. The `-s` argument is for turning
+off the capture mode so that print statements are printed to the console.
+
+A Makefile command also exists to run these. See `make test`.
+
+## Helpful Commands
+
+Here is a list of commands that may be helpful when interacting with this project.
+
+### Docker
+
+List all Docker containers:
+
+```shell
+docker ps -a
+```
+
+## VS Code Extensions
+
+To help facilitate local development you can install
+the [Visual Studio Code Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)
+extension for VS Code. This will allow you to connect to the local development Docker container and more easily develop features.
+
+## GPT3.5 summaries
+
+To generate the GPT3.5 summaries for all articles, use the following commands:
+
+```
+cd src
+python3 -m utils.gpt35_summaries.cleanup_and_summarize
+```
+
+The output CSV file is placed in `src/utils/gpt35_summaries/df_embed_out.csv`
+The pre-generated summaries for all articles are in `df_embed_out2.csv` in the same directory.
+
+For an example of a focussed summary, please see `src/focused_summary_example.py`.
+
+## Resources
+
+- [direnv](https://github.com/direnv/direnv)
+- [Docker](https://docs.docker.com/reference/)
+- [Docker Compose](https://docs.docker.com/compose/)
+- [FastAPI](https://fastapi.tiangolo.com/)
+- [flake8](https://flake8.pycqa.org/en/latest/)
+- [git](https://git-scm.com/)
+- [GitHub Actions](https://docs.github.com/en/actions)
+- [isort](https://pycqa.github.io/isort/index.html)
+- [Makefile](https://www.gnu.org/software/make/manual/make.html)
+- [Markdown](https://www.markdownguide.org/)
+- [pre-commit](https://pre-commit.com)
+- [Python](https://www.python.org/)
+- [tmux](https://github.com/tmux/tmux/wiki/Getting-Started)

data/cicero_faiss_index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bf0b06752ccad29f09484b07bf65d429ec60a203e3697cc14661cee28447d37
+size 3511341

data/clean_dataset.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea0aafebd94acbbe0dbd7370d01a34b49284d5012ffadf3fe95130e83f2f13fd
+size 13751626

data/raw_dataset.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bb32f8b3a313d28aa704e84335ae9d8aefa039f405677a5054aeee67a75cbeb
+size 15756643

docker/aliases.sh

@@ -0,0 +1,20 @@
+# This is a compilation of useful 'USER-DEFINED' aliases to use
+
+alias tmux_create='tmux new -s'        # Creates new tmux session.
+alias tmux_attach='tmux a -t'          # Attaches to an existing tmux session.
+alias     tmux_ls='tmux ls'            # Lists all of the existing tmux sessions.
+alias   tmux_kill="tmux kill-session -t " # Kill a specific tmux session
+alias        gadd='git add'                # Adds a file / directory to repository
+alias        gcom='git commit -m'          # Commits any changes. Use as: gcom "Test"
+alias          gp='git push origin master' # Pushes changes to 'master'
+alias         gst='git status'             # Shows the status of the GIT repository.
+alias       sagent="eval $(ssh-agent -s)"  # Start SSH key agent
+alias          sa='conda activate'      # Activates an Anaconda environment
+alias          sd='conda deactivate'    # Deactivates an Anaconda environment
+alias      jl='jupyter lab --ip 0.0.0.0 --port 8890 --no-browser --allow-root'              # Opens 'Jupyter Lab'
+alias       jn='jupyter notebook --ip 0.0.0.0 --port 8890 --no-browser --allow-root'        # Opens 'Jupyter Notebook'
+alias      lll="ls -lah"
+# Docker-related
+alias       dps="docker ps -a"
+alias       dprune='docker system prune -f'
+alias       dallow="direnv allow"

docker/docker-compose.yaml

@@ -0,0 +1,71 @@
+version: '3'
+
+services:
+  # --- Service used for local development
+  local-dev:
+    # Building the local image
+    build:
+      context: ../
+      dockerfile: ./Dockerfile
+    # Running the local image
+    image: "cicero-synthesizer-local-dev"
+    container_name: "cicero-synthesizer-local-dev"
+    environment:
+      DOCKER_BUILDKIT_VALUE: ${DOCKER_BUILDKIT_VALUE}
+      HUGGING_FACE_HUB_TOKEN: ${HUGGING_FACE_HUB_TOKEN}
+    volumes:
+      - ${HOME}/.ssh:/root/.ssh
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ..:/opt/program
+    working_dir: /opt/program
+    command: [ "/bin/sleep", "365d" ]
+  #
+  # --- Service for running the API locally
+  api:
+    # Building the local image
+    build:
+      context: ../
+      dockerfile: ./Dockerfile
+    # Running the local image
+    image: "cicero-synthesizer-api"
+    container_name: "cicero-synthesizer-api"
+    environment:
+      HUGGING_FACE_HUB_TOKEN: ${HUGGING_FACE_HUB_TOKEN}
+      HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME}
+    volumes:
+      - ..:/opt/ml
+    ports:
+      - ${INPUT_APP_PORT:-8501}:${OUTPUT_APP_PORT:-8501}
+    working_dir: /opt/ml
+    command:
+      [
+        "uvicorn",
+        "src.api.index:app",
+        "--host",
+        "0.0.0.0",
+        "--port",
+        "8501",
+        "--reload",
+        "--reload-dir",
+        "/opt/ml"
+      ]
+  #
+  # --- Service for running the Gradio application locally
+  app:
+    # Building the local image
+    build:
+      context: ../
+      dockerfile: ./Dockerfile
+    # Running the local image
+    image: "cicero-synthesizer-app"
+    container_name: "cicero-synthesizer-app"
+    environment:
+      APP_SERVER_PORT: ${APP_SERVER_PORT}
+      HUGGING_FACE_HUB_TOKEN: ${HUGGING_FACE_HUB_TOKEN}
+      HUGGING_FACE_USERNAME: ${HUGGING_FACE_USERNAME}
+    volumes:
+      - ..:/opt/ml
+    ports:
+      - ${APP_SERVER_PORT:-7860}:${APP_SERVER_PORT:-7860}
+    working_dir: /opt/ml
+    command: [ "python", "src/app_service/app.py" ]

pyproject.toml

@@ -0,0 +1,34 @@
+[tool.isort]
+force_grid_wrap = 0
+include_trailing_comma = true
+line_length = 79
+multi_line_output = 3
+use_parentheses = true
+
+[tool.pytest.ini_options]
+addopts = """
+--cov-report term-missing \
+--cov src/ -ra"""
+
+[tool.black]
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+  # The following are specific to Black, you probably don't want those.
+  | blib2to3
+  | tests/data
+  | profiling
+)/
+'''
+include = '\.pyi?$'
+line-length = 79
+target-version = ['py36', 'py37', 'py38', 'py39']

requirements-deploy.txt

@@ -0,0 +1 @@
+docker==5.0.3

requirements-dev.txt

@@ -0,0 +1,12 @@
+-r ./requirements.txt
+black==22.3.0
+click>=8.0.2
+docker==5.0.3
+flake8==5.0.4
+ipython>=7.0.1
+isort>=4.3.21, <5.0
+jupyter_client>=5.1, <7.0
+jupyterlab>=0.31.1
+jupyter~=1.0
+pre-commit==2.10.1
+protobuf==3.20.1

requirements.txt

@@ -0,0 +1,14 @@
+datasets>=2.13.1
+faiss-cpu>=1.7.4
+fastapi==0.92.0
+gradio>=3.35.2
+huggingface_hub>=0.15.1
+openai>=0.27.8
+pandas>=2.0.2
+sentence-transformers
+setuptools==67.6.1
+spacy>=3.5.3
+tiktoken>=0.4.0
+torch==2.0.0
+tqdm==4.65.0
+uvicorn==0.20.0

src/api/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/api/index.py

@@ -0,0 +1,182 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import logging
+from typing import Dict, Optional
+
+from datasets import Dataset
+from fastapi import Depends, FastAPI
+from fastapi.responses import RedirectResponse
+from huggingface_hub import hf_hub_download
+from pydantic import BaseModel
+
+from src.classes import hugging_face_utils as hf
+from src.classes import semantic_search_engine as ss
+from src.utils import default_variables as dv
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s]: %(message)s",
+)
+logger.setLevel(logging.INFO)
+
+
+# ------------------------------- VARIABLES -----------------------------------
+
+APP_TITLE = "Cicero LLM Synthesizer"
+APP_DESCRIPTION = f"""
+The '{APP_TITLE}'is an app that will identify the top-N articles from the
+Cicero database that are most similar to the user's input query.
+"""
+APP_VERSION = "0.1"
+
+
+# ----------------------------- APP-SPECIFIC ----------------------------------
+
+# Defining the appliation value
+app = FastAPI(
+    title=APP_TITLE,
+    description=APP_DESCRIPTION,
+    version=APP_VERSION,
+)
+
+# -------------------------------- CLASSES ------------------------------------
+
+
+class QueryParams(BaseModel):
+    input_query: str
+    number_articles: Optional[int] = 5
+
+
+# ------------------------------- FUNCTIONS -----------------------------------
+
+
+def download_dataset_and_faiss_index() -> Dataset:
+    """
+    Function to download the corresponding dataset and the FAISS index
+    from HuggingFace.
+
+    Returns
+    -------------
+    dataset_with_faiss_index : datasets.Dataset
+        Dataset from HuggingFace with the FAISS index loaded.
+    """
+    # --- Initializing HuggingFace API
+    # Object for interacting with HuggingFace
+    hf_obj = hf.HuggingFaceHelper()
+
+    # Defining variable names for each of the objects
+    faiss_index_name = f"{dv.faiss_index_name}.faiss"
+    dataset_name = dv.dataset_faiss_embeddings_name
+    username = hf_obj.username
+    repository_name = dv.hugging_face_repository_name
+    repository_id = f"{username}/{repository_name}"
+    repository_type = "dataset"
+    split_type = "train"
+
+    # --- Downloading FAISS Index
+    faiss_index_local_path = hf_hub_download(
+        repo_id=repository_id,
+        filename=faiss_index_name,
+        repo_type=repository_type,
+        token=hf_obj.api.token,
+    )
+
+    # --- Downloading Dataset
+    dataset_obj = hf_obj.get_dataset_from_hub(
+        dataset_name=dataset_name,
+        username=username,
+        split=split_type,
+    )
+
+    # --- Adding FAISS index to the dataset
+    dataset_obj.load_faiss_index(
+        index_name=dv.embeddings_colname,
+        file=faiss_index_local_path,
+    )
+
+    return dataset_obj
+
+
+def run_semantic_search_task(query: str, number_articles: int) -> Dict:
+    """
+    Function to run semantic search on an input query. It will return a
+    set of 'Top-N' articles that are most similar to the input query.
+
+    Parameters
+    ------------
+    query : str
+        Input query to use when running the Semantic Search Engine.
+
+    number_articles : int
+        Number of articles to return from the Semantic Search.
+
+    Returns
+    ----------
+    ranked_results : dict
+        Dictionary containing the ranked results from the Semantic
+        Search Engine.
+    """
+    # --- Extracting dataset with FAISS index
+    corpus_dataset_with_faiss_index = download_dataset_and_faiss_index()
+
+    # --- Initializing Semantic Search Engine
+    semantic_search_obj = ss.SemanticSearchEngine(
+        corpus_dataset_with_faiss_index=corpus_dataset_with_faiss_index
+    )
+
+    # --- Running search on Top-N results
+
+    return semantic_search_obj.run_semantic_search(
+        query=query,
+        top_n=number_articles,
+    )
+
+
+# -------------------------------- ROUTES -------------------------------------
+
+
+@app.get("/", include_in_schema=False)
+async def docs_redirect():
+    return RedirectResponse(url="/docs")
+
+
+# ---- Semantic Search
+@app.post("/predict")
+async def run_semantic_search(query_params: QueryParams = Depends()):
+    """
+    Function to run semantic search on the an input query.
+
+    Parameters
+    --------------
+    query : str
+        Input query to use when running the Semantic Search Engine.
+
+    number_articles : int
+        Number of articles to return from the Semantic Search.
+    """
+
+    return run_semantic_search_task(
+        query=query_params.input_query,
+        number_articles=query_params.number_articles,
+    )

src/app_service/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/app_service/app.py

@@ -0,0 +1,167 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import logging
+from typing import Dict
+
+import gradio as gr
+from datasets import Dataset
+from huggingface_hub import hf_hub_download
+
+from src.classes import hugging_face_utils as hf
+from src.classes import semantic_search_engine as ss
+from src.utils import default_variables as dv
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s]: %(message)s",
+)
+logger.setLevel(logging.INFO)
+
+
+# ------------------------------ VARIABLES ------------------------------------
+
+APP_TITLE = "Cicero LLM Synthesizer"
+APP_DESCRIPTION = f"""
+The '{APP_TITLE}'is an app that will identify the top-N articles from the
+Cicero database that are most similar to the user's input query.
+"""
+APP_VERSION = "0.1"
+
+
+# ------------------------------ FUNCTIONS ------------------------------------
+
+
+def download_dataset_and_faiss_index() -> Dataset:
+    """
+    Function to download the corresponding dataset and the FAISS index
+    from HuggingFace.
+
+    Returns
+    -------------
+    dataset_with_faiss_index : datasets.Dataset
+        Dataset from HuggingFace with the FAISS index loaded.
+    """
+    # --- Initializing HuggingFace API
+    # Object for interacting with HuggingFace
+    hf_obj = hf.HuggingFaceHelper()
+
+    # Defining variable names for each of the objects
+    faiss_index_name = f"{dv.faiss_index_name}.faiss"
+    dataset_name = dv.dataset_faiss_embeddings_name
+    username = hf_obj.username
+    repository_name = dv.hugging_face_repository_name
+    repository_id = f"{username}/{repository_name}"
+    repository_type = "dataset"
+    split_type = "train"
+
+    # --- Downloading FAISS Index
+    faiss_index_local_path = hf_hub_download(
+        repo_id=repository_id,
+        filename=faiss_index_name,
+        repo_type=repository_type,
+        token=hf_obj.api.token,
+    )
+
+    # --- Downloading Dataset
+    dataset_obj = hf_obj.get_dataset_from_hub(
+        dataset_name=dataset_name,
+        username=username,
+        split=split_type,
+    )
+
+    # --- Adding FAISS index to the dataset
+    dataset_obj.load_faiss_index(
+        index_name=dv.embeddings_colname,
+        file=faiss_index_local_path,
+    )
+
+    return dataset_obj
+
+
+def run_semantic_search_task(query: str, number_articles: int) -> Dict:
+    # sourcery skip: remove-unnecessary-cast
+    """
+    Function to run semantic search on an input query. It will return a
+    set of 'Top-N' articles that are most similar to the input query.
+
+    Parameters
+    ------------
+    query : str
+        Input query to use when running the Semantic Search Engine.
+
+    number_articles : int
+        Number of articles to return from the Semantic Search.
+
+    Returns
+    ----------
+    ranked_results : dict
+        Dictionary containing the ranked results from the Semantic
+        Search Engine.
+    """
+    # --- Extracting dataset with FAISS index
+    corpus_dataset_with_faiss_index = download_dataset_and_faiss_index()
+
+    # --- Initializing Semantic Search Engine
+    semantic_search_obj = ss.SemanticSearchEngine(
+        corpus_dataset_with_faiss_index=corpus_dataset_with_faiss_index
+    )
+
+    # --- Running search on Top-N results
+    number_articles_mod = int(number_articles)
+
+    results = semantic_search_obj.run_semantic_search(
+        query=query,
+        top_n=number_articles_mod,
+    )
+
+    return list(results.values())
+
+
+# --------------------------------- APP ---------------------------------------
+
+# -- Semantic Search Engine
+semantic_search_engine = gr.Interface(
+    fn=run_semantic_search_task,
+    inputs=[
+        gr.components.Textbox(label="Input Query"),
+        gr.Slider(
+            minimum=1,
+            label="Choose number of documents to retrieve",
+            step=1,
+        ),
+    ],
+    outputs="json",
+    title=APP_TITLE,
+    description=APP_DESCRIPTION,
+)
+
+
+# ----------------------------- RUNNING APP -----------------------------------
+
+if __name__ == "__main__":
+    semantic_search_engine.launch(
+        debug=False,
+        share=False,
+        server_port=7860,
+    )

src/classes/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/classes/data_preparation.py

@@ -0,0 +1,403 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Module that contains the class definitions for the data preparation tasks.
+"""
+
+import logging
+import re
+from datetime import datetime
+from typing import List, Optional, Tuple, Union
+
+import pandas as pd
+from spacy.lang.en.stop_words import STOP_WORDS
+
+from src.classes import hugging_face_utils as hf
+from src.utils import default_variables as dv
+from src.utils import general_utilities as gu
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+# ---------------------------- CLASS DEFINITIONS ------------------------------
+
+
+# -- Defining functions that can be used for cleaning up and preparing text
+class NLPPrep(object):
+    """
+    Class object for handling the data processing of text.
+    """
+
+    def __init__(self):
+        # Defining the corresponding stop words
+        self.stop_words = list(STOP_WORDS)
+
+    def _lowercase_text(self, input_string: str) -> str:
+        """
+        Method for making the input text lowercase.
+
+        Parameters
+        ------------
+        input_string : str
+            Text variable to lowercase.
+
+        Returns
+        ----------
+        output_string : str
+            Lower-cased version of ``input_string``.
+        """
+
+        return input_string.lower()
+
+    def _only_keep_alphanumeric(self, input_string: str) -> str:
+        """
+        Method for only keeping alphanumerical characters in the text.
+
+        Parameters
+        ------------
+        input_string : str
+            Text variable to filter.
+
+        Returns
+        ----------
+        output_string : str
+            Filtered version of ``input_string`` that only contains
+            alphanumerical characters.
+        """
+        regex_pattern = r"[^a-zA-z0-9\s]"
+
+        return re.sub(regex_pattern, "", input_string)
+
+    def _remove_stopwords(self, input_string: str) -> str:
+        """
+        Method for removing stop words from the input text.
+
+        Parameters
+        ------------
+        input_string : str
+            Text variable to filter.
+
+        Returns
+        ----------
+        output_string : str
+            Filtered version of ``input_string`` without stop words in
+            the text.
+        """
+        # Splitting the text into 'tokens'
+        tokens = input_string.strip().split()
+
+        return " ".join(
+            [word for word in tokens if word not in self.stop_words]
+        )
+
+    def _remove_unicode(self, input_str: str) -> str:
+        """
+        Method for removing Unicode from the input text.
+
+        Parameters
+        ------------
+        input_str : str
+            Text variable, from which to remove Unicode characters.
+
+        Returns
+        ----------
+        string_decode : str
+            Filtered version of ``input_str`` without the Unicode characters.
+        """
+        string_encode = input_str.encode("ascii", "ignore")
+
+        return string_encode.decode()
+
+    def process_text(self, input_string: str) -> str:
+        """
+        Method for passing the input variable through NLP-based techniques
+        to process the text.
+
+        Parameters
+        ------------
+        input_string : str
+            Variable corresponding to the text that will be processed.
+
+        Returns
+        ------------
+        processed_string : str
+            Variable corresponding to the *processed* version of the input
+            string, after having gone through some NLP-based processing
+            techniques.
+
+        Notes
+        -----------
+        This function will perform the following NLP-based techniques:
+
+        1. Make the text lowercase.
+        2. Remove any non-alphanumeric character from the string.
+        3. Remove any stop words from the text.
+        """
+        # Remove Unicode characters
+        processed_string = self._remove_unicode(input_string)
+        # Lower case the text
+        processed_string = self._lowercase_text(processed_string)
+        # Removing non-alphanumeric characters
+        processed_string = self._only_keep_alphanumeric(processed_string)
+        # Removing stop words
+        processed_string = self._remove_stopwords(processed_string)
+
+        return processed_string
+
+
+class DatasetPrep(object):
+    """
+    Class object for the Data Processing of the input dataset.
+    """
+
+    def __init__(
+        self,
+        dataset_path: str,
+        **kwargs,
+    ):
+        """
+        Class object for the Data Processing of the input dataset.
+
+        Parameters
+        ------------
+        dataset_path : str
+            Path / URL to the input dataset.
+        """
+        # Path to the output dataset
+        self.datasets_dir = gu.get_project_paths()["data"]
+
+        # Other parameters
+        for colname in [
+            "save_to_disk",
+            "document_id_colname",
+            "title_colname",
+            "content_colname",
+            "clean_content_colname",
+        ]:
+            setattr(self, colname, kwargs.get(colname, getattr(dv, colname)))
+
+        # Initializing dataset
+        self.dataset_path = dataset_path
+        self.raw_dataset = self._get_dataset()
+
+        # Extracting the number of rows and columns, and column names
+        (
+            self.n_rows,
+            self.n_cols,
+            self.columns_names,
+        ) = self._get_columns_and_shape()
+
+        # Initializing NLP-Prep Object
+        self.nlp_obj = NLPPrep()
+
+    def show_params(self):
+        """
+        Method for displaying the set of input parameters of the class.
+        """
+
+        gu.show_params(
+            params_dict=self.__dict__,
+            logger=logger,
+            columns_to_omit=["raw_dataset"],
+        )
+
+    def _get_dataset(self) -> pd.DataFrame:
+        # sourcery skip: class-extract-method
+        """
+        Method for extracting the dataset from the input source.
+
+        Returns
+        ----------
+        raw_dataset : pandas.DataFrame
+            DataFrame containing the data from the input source.
+        """
+        logger.info(f">> Extracting dataset from `{self.dataset_path}`")
+
+        # Reading in dataset
+        raw_dataset = pd.read_csv(self.dataset_path)
+
+        # Saving to disk, if applicable
+        if self.save_to_disk:
+            dataset_filepath = self.datasets_dir.joinpath("raw_dataset.csv")
+            dataset_filepath.parent.mkdir(exist_ok=True, parents=True)
+            raw_dataset.to_csv(dataset_filepath, header=True, index=True)
+
+            logger.info(f">> Raw dataset saved to '{str(dataset_filepath)}'")
+
+        return raw_dataset
+
+    def _get_columns_and_shape(self) -> Tuple[int, int, List]:
+        # sourcery skip: use-fstring-for-formatting
+        """
+        Method for extracting the columns and information about the
+        raw dataset.
+
+        Returns
+        ----------
+        n_rows : int
+            Number of rows in the original dataset.
+
+        n_cols : int
+            Number of columns in the original dataset.
+
+        column_names_arr : list
+            List of columns from the original dataset.
+        """
+        # Number of rows and columns
+        n_rows, n_columns = self.raw_dataset.shape
+
+        logger.info(
+            ">> There are '{}' rows and '{}' columns in the dataset".format(
+                n_rows,
+                n_columns,
+            )
+        )
+
+        # Column names
+        column_names_arr = sorted(self.raw_dataset.columns)
+
+        logger.info(
+            ">> Columns in the dataset: \n\t{}".format(
+                "\n\t".join(column_names_arr)
+            )
+        )
+
+        return n_rows, n_columns, column_names_arr
+
+    def _process_text(self, input_text: str) -> str:
+        """
+        Method for applying NLP-based techniques on an input text in order
+        to prepare it to be used by the embedding algorithm.
+
+        Parameters
+        -----------
+        input_text : str
+            Variable corresponding to the input text.
+
+        Returns
+        -----------
+        processed_text : str
+            Processed version of the ``input_text``.
+
+        Notes
+        ----------
+        This function will perform the following NLP-based techniques:
+
+        1. Make the text lowercase.
+        2. Remove any non-alphanumeric character from the string.
+        3. Remove any stop words from the text.
+        """
+
+        return self.nlp_obj.process_text(input_string=input_text)
+
+    def clean_dataset(self) -> pd.DataFrame:
+        """
+        Method for cleaning the raw dataset and create a clean version
+        of the dataset.
+
+        Returns
+        ---------
+        dataset_clean : pandas.DataFrame
+            Clean version of the input dataset, after having gone through
+            data-cleaning techniques.
+        """
+        # --- Start time
+        logger.info(">> Data cleaning process ...")
+        start_time = datetime.now()
+        #
+
+        # --- Making a copy of the raw dataset
+        dataset_df = self.raw_dataset.copy()
+
+        # --- Data-cleaning techniques
+        # Removing duplicates
+        dataset_df.drop_duplicates(keep="first", inplace=True)
+
+        # Removing entries that have 'NaN' in the dataset
+        dataset_df.dropna(how="any", inplace=True)
+
+        # Casting proper data types
+        dataset_df = dataset_df.astype(str)
+
+        # Resetting the index of the dataset
+        dataset_df.reset_index(drop=True, inplace=True)
+
+        # Removing trailing whitespaces
+        for colname in [self.document_id_colname, self.title_colname]:
+            dataset_df.loc[:, colname] = dataset_df[colname].apply(
+                lambda x: x.strip()
+            )
+
+        # Processing content
+        dataset_df.loc[:, getattr(self, "clean_content_colname")] = dataset_df[
+            getattr(self, "content_colname")
+        ].apply(lambda text: self.nlp_obj.process_text(text))
+
+        # --- Saving to disk, if applicable
+        if self.save_to_disk:
+            dataset_filepath = self.datasets_dir.joinpath("clean_dataset.csv")
+            dataset_filepath.parent.mkdir(exist_ok=True, parents=True)
+            dataset_df.to_csv(dataset_filepath, header=True, index=True)
+
+            logger.info(f">> Clean dataset saved to '{str(dataset_filepath)}'")
+
+        # --- End time
+        end_time = datetime.now()
+        logger.info(f">>    Finished at: {end_time}")
+        logger.info(f">>    Took: {end_time - start_time}")
+        logger.info(">> Data cleaning process ... DONE")
+
+        return dataset_df
+
+    def push_dataset_to_hub(
+        self,
+        dataset: pd.DataFrame,
+        dataset_name: str,
+        username: Optional[Union[None, str]] = None,
+    ):
+        """
+        Method for pushing the ``dataset`` to the HuggingFace's Hub.
+
+        Parameters
+        -------------
+        dataset : pandas.DataFrame
+            Dataset that will be pushed to HuggingFace.
+
+        dataset_name : str
+            Name of the dataset to use.
+
+        username : str, NoneType, optional
+            Us
+        """
+        # Initializing class object
+        hf_obj = hf.HuggingFaceHelper()
+
+        # Transforming dataset type
+        hf_dataset = hf_obj.convert_dataframe_to_dataset(input_df=dataset)
+
+        # Push dataset to hub
+        hf_obj.push_dataset(
+            dataset=hf_dataset,
+            dataset_name=dataset_name,
+            username=username,
+        )

src/classes/hugging_face_utils.py

@@ -0,0 +1,223 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Module that includes utilities for interacting with HuggingFace
+"""
+
+import logging
+import os
+from typing import Dict, Optional, Union
+
+import pandas as pd
+from datasets import Dataset, load_dataset
+from huggingface_hub import HfApi
+
+from src.utils import default_variables as dv
+
+__all__ = ["HuggingFaceHelper"]
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+logger.setLevel(level=logging.INFO)
+
+
+class HuggingFaceHelper(object):
+    """
+    Class definition for creating, interacting, and sharing Datasets.
+    """
+
+    def __init__(self, **kwargs: Dict) -> None:
+        """
+        Class definition for creating, interacting, and sharing Datasets.
+        """
+        # Name of the HuggingFace token as stored in the user's environment
+        self.token_name = kwargs.get("token_name", dv.hugging_face_token_name)
+        self.username = kwargs.get(
+            "username",
+            os.environ.get(dv.hugging_face_username_name),
+        )
+
+        # HuggingFace endpoint
+        self.api_endpoint = "https://huggingface.co"
+        self.api = self._authenticate_api()
+
+    def _authenticate_api(self) -> HfApi:
+        """
+        Method for authenticating with HuggingFace using an authentication
+        token.
+
+        Returns
+        ---------
+        huggingface_api : huggingface_hub.hf_api.HfApi
+            Object corresponding to the HuggingFace API after authentication.
+        """
+        # Check that token is part of the user's environment
+        if not os.environ.get(self.token_name):
+            msg = f">>> HuggingFace API Token '{self.token_name}' not defined!"
+            logger.error(msg)
+            raise ValueError(msg)
+
+        # Initializing API object
+        return HfApi(
+            endpoint=self.api_endpoint,
+            token=os.environ.get(self.token_name),
+        )
+
+    def convert_dataframe_to_dataset(
+        self,
+        input_df: pd.DataFrame,
+    ) -> Dataset:
+        """
+        Function to convert an existing DataFrame into a ``Dataset`` object
+
+        Parameters
+        -------------
+        input_df : pandas.DataFrame
+            Variable corresponding to the DataFrame to convert.
+
+        Returns
+        -----------
+        dataset_obj : datasets.Dataset
+            Dataset object with the same data as ``input_df``.
+        """
+
+        return Dataset.from_pandas(df=input_df)
+
+    def get_dataset_from_hub(
+        self,
+        dataset_name: str,
+        username: Optional[Union[None, str]] = None,
+        split: Optional[Union[None, str]] = None,
+    ) -> Dataset:
+        # sourcery skip: extract-duplicate-method, use-fstring-for-formatting
+        """
+        Method for extracting the Dataset from HuggingFace.
+
+        Parameters
+        ------------
+        dataset_name : str
+            Name of the dataset to extract from HuggingFace's Hub.
+
+        username : str, NoneType, optional
+            Username to use when extracting the dataset from HuggingFace Hub.
+            This variable is set to ``None`` by default.
+
+        split : str, NoneType, optional
+            Type of ``split`` to load for the Dataset. If ``None``, the
+            method will extract all splits. This variable is set to
+            ``None`` by default.
+
+        Returns
+        --------
+        dataset_obj : datasets.Dataset
+            Variable corresponding to the dataset that was extracted
+            from the HuggingFace Hub.
+        """
+        # 'dataset_name' - Type
+        dataset_name_type_arr = (str,)
+        if not isinstance(dataset_name, dataset_name_type_arr):
+            msg = (
+                ">> 'dataset_name' ({}) is not a valid input type ({})".format(
+                    type(dataset_name),
+                    dataset_name_type_arr,
+                )
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+        # 'username' - Type
+        username_type_arr = (str, type(None))
+        if not isinstance(username, username_type_arr):
+            msg = ">> 'username' ({}) is not a valid input type ({})".format(
+                type(username),
+                username_type_arr,
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+        # 'split' - Type
+        split_type_arr = (str, type(None))
+        if not isinstance(split, split_type_arr):
+            msg = ">> 'split' ({}) is not a valid input type ({})".format(
+                type(split),
+                split_type_arr,
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+
+        # Defining the path to the dataset in HF.
+        dataset_path = (
+            f"{username}/{dataset_name}" if username else dataset_name
+        )
+
+        return load_dataset(dataset_path, split=split)
+
+    def push_dataset(
+        self,
+        dataset: Dataset,
+        dataset_name: str,
+        username: Optional[Union[None, str]] = None,
+    ):  # sourcery skip: extract-duplicate-method, use-fstring-for-formatting
+        """
+        Method for pushing an existing local Dataset to HuggingFace.
+        """
+        # --- Check input type
+        # 'dataset' - Type
+        dataset_type_arr = (Dataset,)
+        if not isinstance(dataset, dataset_type_arr):
+            msg = ">> 'dataset' ({}) is not a valid input type ({})".format(
+                type(dataset),
+                dataset_type_arr,
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+        # 'dataset_name' - Type
+        dataset_name_type_arr = (str,)
+        if not isinstance(dataset_name, dataset_name_type_arr):
+            msg = (
+                ">> 'dataset_name' ({}) is not a valid input type ({})".format(
+                    type(dataset_name),
+                    dataset_name_type_arr,
+                )
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+        # 'username' - Type
+        username_type_arr = (str, type(None))
+        if not isinstance(username, username_type_arr):
+            msg = ">> 'username' ({}) is not a valid input type ({})".format(
+                type(username),
+                username_type_arr,
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+
+        # Defining the path to the dataset in HF.
+        dataset_path = (
+            f"{username}/{dataset_name}" if username else dataset_name
+        )
+
+        # Pushing dataset to HuggingFace
+        dataset.push_to_hub(
+            repo_id=dataset_path,
+            token=os.environ.get(self.token_name),
+        )

src/classes/semantic_search_engine.py

@@ -0,0 +1,249 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import logging
+from typing import Dict, Optional
+
+import numpy as np
+import pandas as pd
+import torch
+from datasets import Dataset
+from sentence_transformers import SentenceTransformer
+
+from src.utils import default_variables as dv
+
+__author__ = ["Victor Calderon"]
+__copyright__ = ["Copyright 2023 Victor Calderon"]
+__all__ = ["SemanticSearchEngine"]
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s]: %(message)s",
+)
+logger.setLevel(logging.INFO)
+
+# --------------------------- CLASS DEFINITIONS -------------------------------
+
+
+class SemanticSearchEngine(object):
+    """
+    Class object for running Semantic Search on the input dataset.
+    """
+
+    def __init__(self, **kwargs):
+        """
+        Class object for running Semantic Search on the input dataset.
+        """
+        # --- Defining variables
+        # Device to use, i.e. CPU or GPU
+        self.device = self._get_device()
+        # Embedder model to use
+        self.model = "paraphrase-mpnet-base-v2"
+        # Defining the embedder
+        self.embedder = self._get_embedder()
+
+        # Corpus embeddings
+        self.source_colname = kwargs.get(
+            "source_colname",
+            "summary",
+        )
+        self.embeddings_colname = kwargs.get(
+            "embeddings_colname",
+            dv.embeddings_colname,
+        )
+
+        # Variables used for running semantic search
+        self.corpus_dataset_with_faiss_index = kwargs.get(
+            "corpus_dataset_with_faiss_index"
+        )
+
+    def _get_device(self) -> str:
+        """
+        Method for determining the device to use.
+
+        Returns
+        ----------
+        device_type : str
+            Type of device to use (e.g. 'cpu' or 'cuda').
+
+            Options:
+                - ``cpu``  : Uses a CPU.
+                - ``cuda`` : Uses a GPU.
+        """
+        # Determining the type of device to use
+        device_type = "cuda" if torch.cuda.is_available() else "cpu"
+
+        logger.info(f">> Running on a '{device_type.upper()}' device")
+
+        return device_type
+
+    def _get_embedder(self):
+        """
+        Method for extracting the Embedder model.
+
+        Returns
+        ---------
+        embedder : model
+            Variable corresponding to the Embeddings models.
+        """
+        embedder = SentenceTransformer(self.model)
+        embedder.to(self.device)
+
+        return embedder
+
+    def generate_corpus_index_and_embeddings(
+        self,
+        corpus_dataset: Dataset,
+    ) -> Dataset:
+        """
+        Method for generating the Text Embeddings and FAISS indices from
+        the input dataset.
+
+        Parameters
+        ------------
+        corpus_dataset : datasets.Dataset
+            Dataset containing the text to use to create the text
+            embeddings and FAISS indices.
+
+        Returns
+        ----------
+        corpus_dataset_with_embeddings : datasets.Dataset
+            Dataset containing the original data rom ``corpus_dataset``
+            plus the corresponding text embeddings of the ``source_colname``
+            column.
+        """
+        torch.set_grad_enabled(False)
+
+        # --- Generate text embeddings for the source column
+        corpus_dataset_with_embeddings = corpus_dataset.map(
+            lambda corpus: {
+                self.embeddings_colname: self.embedder.encode(
+                    corpus[self.source_colname]
+                )
+            },
+            batched=True,
+            desc="Computing Semantic Search Embeddings",
+        )
+
+        # --- Adding FAISS index
+        corpus_dataset_with_embeddings.add_faiss_index(
+            column=self.embeddings_colname,
+            faiss_verbose=True,
+            device=None if self.device == "cpu" else 1,
+        )
+
+        return corpus_dataset_with_embeddings
+
+    def run_semantic_search(
+        self,
+        query: str,
+        top_n: Optional[int] = 5,
+    ) -> Dict:  # sourcery skip: extract-duplicate-method
+        """
+        Method for running a semantic search on a query after having
+        created the corpus of the text embeddings.
+
+        Parameters
+        --------------
+        query : str
+            Text query to use for searching the database.
+
+        top_n : int, optional
+            Variable corresponding to the 'Top N' values to return based on the
+            similarity score between the input query and the corpus. This
+            variable is set to ``10`` by default.
+
+        Returns
+        ---------
+        match_results : dict
+            Dictionary containing the metadata of each of the articles
+            that were in the Top-N in terms of being most similar to the
+            input query ``query``.
+        """
+        # --- Checking input parameters
+        # 'query' - Type
+        query_type_arr = (str,)
+        if not isinstance(query, query_type_arr):
+            msg = ">> 'query' ({}) is not a valid input type ({})".format(
+                type(query), query_type_arr
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+        # 'top_n' - Type
+        top_n_type_arr = (int,)
+        if not isinstance(top_n, top_n_type_arr):
+            msg = ">> 'top_n' ({}) is not a valid input type ({})".format(
+                type(top_n), top_n_type_arr
+            )
+            logger.error(msg)
+            raise TypeError(msg)
+
+        # 'top_n' - Value
+        if top_n <= 0:
+            msg = f">> 'top_n' ({top_n}) must be larger than '0'!"
+            logger.error(msg)
+            raise ValueError(msg)
+
+        # --- Checking that the encoder has been indexed correctly
+        if self.corpus_dataset_with_faiss_index is None:
+            msg = ">>> The FAISS index was not properly set!"
+            logger.error(msg)
+            raise ValueError(msg)
+
+        # --- Encode the input query and extract the embedding
+        query_embedding = self.embedder.encode(query)
+
+        # --- Extracting the top-N results
+        (
+            scores,
+            results,
+        ) = self.corpus_dataset_with_faiss_index.get_nearest_examples(
+            self.embeddings_colname,
+            query_embedding,
+            k=top_n,
+        )
+
+        # --- Sorting from highest to lowest
+        # NOTE: We need to deconstruct the 'results' to be able to organize
+        # the results
+        parsed_results = pd.DataFrame.from_dict(
+            data=results,
+            orient="columns",
+        )
+        parsed_results.loc[:, "relevance"] = scores
+
+        # Sorting in descending order
+        parsed_results = parsed_results.sort_values(
+            by=["relevance"],
+            ascending=False,
+        ).reset_index(drop=True)
+
+        # Casting data type for the 'relevance'
+        parsed_results.loc[:, "relevance"] = parsed_results["relevance"].apply(
+            lambda x: str(np.round(x, 5))
+        )
+
+        # Only keeping certain columns
+        columns_to_keep = ["_id", "title", "relevance", "content"]
+
+        return parsed_results[columns_to_keep].to_dict(orient="index")

src/data_processing/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/data_processing/prepare_dataset.py

@@ -0,0 +1,196 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Module for preparing the input dataset.
+"""
+
+import logging
+from pathlib import Path
+from typing import Dict
+
+from src.classes import data_preparation as dp
+from src.utils import default_variables as dv
+from src.utils import general_utilities as gu
+
+__author__ = ["Victor Calderon"]
+__copyright__ = ["Copyright 2023 Victor Calderon"]
+__all__ = []
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s]: %(message)s",
+)
+logger.setLevel(logging.INFO)
+
+
+# ---------------------------- PROJECT VARIABLES ------------------------------
+
+MODULE_DESCRIPTION = "Module for data preparation"
+MODULE_VERSION = "1.0"
+
+
+# ----------------------------- INPUT PARAMETERS ------------------------------
+
+
+def get_parser():
+    """
+    Function to get the input parameters to the script.
+    """
+    # Defining the 'parser' object to use
+    parser = gu._get_parser_obj(description=MODULE_DESCRIPTION)
+
+    # Path to the input dataset
+    parser.add_argument(
+        "--dataset-path",
+        dest="dataset_path",
+        default=dv.cicero_dataset_url,
+        type=str,
+        help="""
+        Path / URL to the input dataset.
+        [Default: '%(default)s']
+        """,
+    )
+
+    return parser.parse_args()
+
+
+# -------------------------------  FUNCTIONS ----------------------------------
+
+
+def _resolve_input_object_path(object_path: str) -> str:
+    """
+    Check whether or not the path corresponds to a local file or a URL.
+
+    Parameters
+    -------------
+    object_path : str
+        Path of the input object.
+
+    Returns
+    ----------
+    parsed_object_path : str
+        Modified / parsed version of the input object ``object_path``.
+
+    Raises
+    ------------
+    TypeError ; Error
+        This error gets raised whenever the input object is neither
+        a 'file' nor a valid 'url'.
+    """
+    object_type = gu.check_url_or_file_type(object_path=object_path)
+
+    if object_type == "unspecified":
+        msg = (
+            f">>> Unspecified data type for '{object_path}' or does not exist"
+        )
+        logger.error(msg)
+        raise TypeError(msg)
+
+    return (
+        object_path
+        if object_type == "url"
+        else str(Path(object_path).resolve())
+    )
+
+
+def _temp_create_dataset_with_summaries():
+    """
+    Function to **temporarily** create the Dataset object in HuggingFace
+    using the dataset with summaries for each of the articles.
+
+    Notes
+    --------
+    This is a temporary solution UNTIL the ``Summarizer`` is put in place.
+    """
+    # Path to the dataset
+    dataset_filepath = str(
+        (
+            gu.get_project_paths()
+            .get("src")
+            .joinpath(
+                "utils",
+                "gpt35_summaries",
+                "df_embed_out2.csv",
+            )
+        ).resolve()
+    )
+
+    # Reading in dataset
+    data_prep_obj = dp.DatasetPrep(dataset_path=dataset_filepath)
+
+    # Uploading it to HuggingFace Hub
+    data_prep_obj.push_dataset_to_hub(
+        dataset=data_prep_obj.raw_dataset,
+        dataset_name=dv.summaries_dataset_name,
+    )
+
+    return
+
+
+# ------------------------------ MAIN FUNCTIONS -------------------------------
+
+
+def main(params_dict: Dict):
+    """
+    Main function to process the data.
+    """
+    # Determine if the path corresponds to a file or a URL
+    params_dict["object_path"] = _resolve_input_object_path(
+        params_dict["dataset_path"]
+    )
+
+    # Showing set of input parameters
+    gu.show_params(params_dict=params_dict, logger=logger)
+
+    # Initializing input parameters
+    data_prep_obj = dp.DatasetPrep(dataset_path=params_dict["object_path"])
+    data_prep_obj.show_params()
+    clean_dataset = data_prep_obj.clean_dataset()
+
+    logger.info(f"\n>>> Raw dataset: \n{data_prep_obj.raw_dataset}\n")
+    logger.info(f"\n>>> Clean dataset: \n{clean_dataset}\n")
+
+    # --- Pushing datasets to HuggingFace Hub
+    # 'Raw' dataset
+    data_prep_obj.push_dataset_to_hub(
+        dataset=data_prep_obj.raw_dataset,
+        dataset_name=dv.raw_dataset_name,
+    )
+    # 'Clean' dataset
+    data_prep_obj.push_dataset_to_hub(
+        dataset=clean_dataset,
+        dataset_name=dv.clean_dataset_name,
+    )
+
+    # Dataset with summaries
+    _temp_create_dataset_with_summaries()
+
+    return
+
+
+if __name__ == "__main__":
+    # Getting input parameters
+    params_dict = vars(get_parser())
+    # Running main function
+    main(params_dict=params_dict)

src/focused_summary_example.py

@@ -0,0 +1,20 @@
+from utils.gpt35_summaries.summarizer import Summarizer
+
+s = Summarizer()
+article_text = """
+div g alt rchttpswwwaeiorgwpcontentuploads202010thatcherpngx91208 iv pa relnoreferrer noopener efhttpsenwikipediaorgwikimargaret_thatcher margaret hilda pictured above was born on this day october 13 in 1925 and today would have been her birthday unfortunately she died on april 8 2013 at the age of 87 to honor baroness thatcher on her birthday here is my annual tribute to the iron lady in recognition of her significant contributions during her political career including serving as the prime minister of the k from 1979 to 1990 below are some videos quotations and related articles to celebrate prime minister margaret thatchers birthday and her rich legacy defending liberty and freedom and fighting socialism p llowfullscreen width640 height350iframecenter 1 the video above features margaret thatchers address to the conservative party conference in 1983 when she classwpblockquoteplet us never forget this fundamental truth the state has no source of money other than the money people earn themselves if the state wishes to spend more it can do so only by borrowing your savings or by taxing you more ere is no such thing as public money there is only taxpayers te m a very fundamental truth that is frequently forgotten any time you see or hear the terms public funding public funds government funding or government funds be sure to substitute taxpayer funding and taxpayer p 2 here are the five reasons margaret thatcher is still an inspiration to women trong via the a relnoreferrer noopener efhttpiwforgblog2804951topfivereasonsmargaretthatcherisstillaninspirationtowomentoday ndependent on margaret thatchers birthday in 2017 summarized he didnt use her sex to influence her he was he challenged the status he had to work for her he was a modern pp llowfullscreen rchttpswwwyoutubecomembedrv5t6rc6yvg width640 height350iframecenter p 3 the video above is margaret thatchers last house of commons speech on november 22 1990 which is known as thatchers last stand against socialism heres the a relnoreferrer noopener efhttpswwwmargaretthatcherorgdocument108256 full and heres a think that the hon gentleman knows that i have the same contempt for his socialist policies as the people of east europe who have experienced them have for theirs i think that i must have hit the right nail on the head when i pointed out that the logic of those policies is that they would rather the poor were poorer once they start to talk about the gap they would rather that the gap were tem[indicating[emdown here not [indicating[embutem[indicatingem] so long as the gap is smaller they would rather have the poor poorer one does not create wealth and opportunity that way one does not create a ropertyowning democracy that way 4 here are 10 great margaret thatcher quotes which are just as relevant and timely for america today if not more so than they were for the k more than a ercentury ago listen up marx oc thanks to larry reed for a relnoreferrer noopener efhttpswwwfeeorgarticlesmargaretthatcheronsocialism20ofherbestquotesfbclidiwar0ypr1qt8cco_rft4xqrr33_ebujx0aymff6mfhh149d_1uwtiyagjoblk ome of these classwpblockquotep1 the problem with socialism is that you eventually run out of other peoples moneypp2 do you know that one of the great problems of our age is that we are governed by people who care more about feelings than they do about thoughts and deaspp3 i think weve been through a period where too many people have been given to understand that if they have a problem its the governments job to cope with it i have a problem ill get a grant im homeless the government must house me theyre casting their problem on society and you know there is no such thing as society there are individual men and women and there are families and no government can do anything except through people and people must look to themselves first its our duty to look after ourselves and then also to look after our neighbor people have got the too much in mind without the obligations theres no such thing as entitlement unless someone has first met an obligationpp4 no one would remember the good samaritan if hed only had good intentions he had money as wellpp5 the philosophical reason for which we are against nationalization and for private enterprise is because we believe that economic progress comes from the inventiveness ability determination and the pioneering spirit of extraordinary men and women if they cannot exercise that spirit here they will go away to another free enterprise country which will then make more economic progress than we do we ought in fact to be encouraging small firms and small companies because the extent to which innovation comes through these companies is endouspp6 our challenge is to create the kind of economic background which enables private initiative and private enterprise to flourish for the benefit of the consumer employee the pensioner and society as a believe we should judge people on merit and not on background i believe the person who is prepared to work hardest should get the greatest rewards and keep them after tax that we should back the workers and not the shirkers that it is not only permissible but praiseworthy to want to benefit your own family by your own effortspp7 i place a profound eliefindeed a fervent faithin the virtues of elfreliance and personal independence on these is founded the whole case for the free society for the assertion that human progress is best achieved by offering the freest possible scope for the development of individual talents qualified only by a respect for the qualities and the freedom of othersfor many years there has been a subtle erosion of the essential virtues of the free society elfreliance has been sneered at as if it were an absurd suburban pretention thrift has been denigrated as if it were greed the desire of parents to choose and to struggle for what they themselves regarded as the best possible education for their children has been cornedpp8 what are the lessons then that weve learned from the last thirty years first that the pursuit of equality itself is a mirage whats more desirable and more practicable than the pursuit of equality is the pursuit of equality of opportunity and opportunity means nothing unless it includes the right to be unequal and the freedom to be different one of the reasons that we value individuals is not because theyre all the same but because theyre all different i believe you have a saying in the middle west dont cut down the tall poppies let them rather grow tall i would say let our children grow tall and some taller than others if they have the ability in them to do so because we must build a society in which each citizen can develop his full potential both for his own benefit and for the community as a whole a society in which originality skill energy and thrift are rewarded in which we encourage rather than restrict the variety and richness of human naturepp9 some socialists seem to believe that people should be numbers in a state computer we believe they should be individuals we are all unequal no one thank heavens is like anyone else however much the socialists may pretend otherwise we believe that everyone has the right to be unequal but to us every human being is equally mportantpp10 there is no such thing as safe socialism if its safe its not socialism and if its socialism its not safe the signposts of socialism point downhill to less freedom less prosperity downhill to more muddle more failure if we follow them to their destination they will lead this nation into lockquote happy birthday margaret thatcher p e post a efhttpswwwaeiorgcarpediemhappy96thbirthdaymargaretthatcherhappy birthday margaret appeared first on a efhttpswwwaeiorgamerican enterprise institute
+"""
+search_string = "most significant contribution of thatcher"
+
+print("Focussed summary for search string", f'"{search_string}":')
+print(s._run_model(article_text, search_string))
+
+print("Vanilla summary:")
+print(s._run_model(article_text))
+
+# Example output:s
+# Focussed summary for search string "most significant contribution of thatcher":
+# In commemoration of former British Prime Minister Margaret Thatcher's birthday, which falls on October 13, American Enterprise Institute pays tribute to her political career and significant contributions, including serving as the Prime Minister of the UK from 1979 to 1990. The article features various videos, quotations, and articles that celebrate Thatcher's rich legacy of defending freedom, liberty, and fighting socialism. The article lists several of Thatcher's speeches, quotes, and her last stand against socialism.
+# Overall, the article does not explicitly state Thatcher's most significant contribution, but it heavily implies that Thatcher's significant contributions were her defense of liberty and freedom and her fight against socialism.
+# Vanilla summary:
+# To commemorate Margaret Thatcher's birthday, born on October 13 in 1925, AEI presents an annual tribute to the Iron Lady. Thatcher's contribution during her political career and her rich legacy defending liberty and freedom and fighting socialism are celebrated. The article provides several videos, quotations, and related articles to commemorate the Prime Minister's birthday and her lasting influence as a female leader in the UK. Among Thatcher's inspirational quotes, she advocated for private enterprise as a means of economic progress, and emphasized the importance of individual responsibility, merit, and personal independence in building a free society.

src/training/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/training/create_faiss_corpus_index.py

@@ -0,0 +1,209 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import logging
+from pathlib import Path
+from typing import Dict
+
+from src.classes import hugging_face_utils as hf
+from src.classes import semantic_search_engine as ss
+from src.utils import default_variables as dv
+from src.utils import general_utilities as gu
+
+__author__ = ["Victor Calderon"]
+__copyright__ = ["Copyright 2023 Victor Calderon"]
+__all__ = []
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s]: %(message)s",
+)
+logger.setLevel(logging.INFO)
+
+# ---------------------------- PROJECT VARIABLES ------------------------------
+
+MODULE_DESCRIPTION = "Module for data preparation"
+MODULE_VERSION = "1.0"
+
+
+# ----------------------------- INPUT PARAMETERS ------------------------------
+
+
+def get_parser():
+    """
+    Function to get the input parameters to the script.
+    """
+    # Defining the 'parser' object to use
+    parser = gu._get_parser_obj(description=MODULE_DESCRIPTION)
+
+    # Path to the input dataset
+    parser.add_argument(
+        "--dataset-name",
+        dest="dataset_name",
+        default=dv.summaries_dataset_name,
+        type=str,
+        help="""
+        Name of the HuggingFace dataset
+        [Default: '%(default)s']
+        """,
+    )
+    # Name of the output Dataset with FAISS index and embeddings
+    parser.add_argument(
+        "--output-dataset-name",
+        dest="output_dataset_name",
+        default=dv.dataset_faiss_embeddings_name,
+        type=str,
+        help="""
+        Name of the output dataset that will contain a FAISS index the
+        text embeddings of the summaries.
+        [Default: '%(default)s']
+        """,
+    )
+    # Name of the HuggingFace repository
+    parser.add_argument(
+        "--repository-name",
+        dest="repository_name",
+        default=dv.hugging_face_repository_name,
+        type=str,
+        help="""
+        Name of the HuggingFace repository to use for storing artifacts.
+        [Default: '%(default)s']
+        """,
+    )
+    # Name of the FAISS Index
+    parser.add_argument(
+        "--faiss-index-name",
+        dest="faiss_index_name",
+        default=dv.faiss_index_name,
+        type=str,
+        help="""
+        Name of the FAISS Index of the output dataset.
+        [Default: '%(default)s']
+        """,
+    )
+
+    return parser.parse_args()
+
+
+# -------------------------------  FUNCTIONS ----------------------------------
+
+
+def create_faiss_index_and_embeddings_from_dataset(params_dict: Dict):
+    """
+    Function to create a Dataset object with a FAISS index and the
+    corresponding text embeddings.
+
+    Parameters
+    -----------
+    params_dict : dict
+        Dictionary with set of parameters that are used throughout the project.
+    """
+    # --- Initializing object for interacting with Datasets
+    hf_obj = hf.HuggingFaceHelper()
+
+    # --- Download dataset from HuggingFace Hub
+    dataset_obj = hf_obj.get_dataset_from_hub(
+        dataset_name=params_dict["dataset_name"],
+        username=hf_obj.username,
+        split="train",
+    )
+
+    # --- Generate the FAISS index and Text embeddings
+    # Initialize Semantic Search engine
+    semantic_search_obj = ss.SemanticSearchEngine()
+
+    # Create FAISS index and the dataset with text embeddings
+    dataset_with_embeddings_obj = (
+        semantic_search_obj.generate_corpus_index_and_embeddings(
+            corpus_dataset=dataset_obj
+        )
+    )
+
+    # --- Extract FAISS index and upload it to HuggingsFace Hub
+    # Path to the output file that will contain the FAISS index
+    faiss_index_local_path = str(
+        gu.get_project_paths()["data"].joinpath(
+            f'{params_dict["faiss_index_name"]}.faiss'
+        )
+    )
+
+    dataset_with_embeddings_obj.save_faiss_index(
+        index_name=semantic_search_obj.embeddings_colname,
+        file=faiss_index_local_path,
+    )
+
+    # Creating repository in HuggingFace
+    repo_name = f'{hf_obj.username}/{params_dict["repository_name"]}'
+    repo_type = "dataset"
+
+    _ = hf_obj.api.create_repo(
+        repo_id=repo_name,
+        repo_type=repo_type,
+        exist_ok=True,
+    )
+
+    # Uploading FAISS
+    hf_obj.api.upload_file(
+        path_or_fileobj=faiss_index_local_path,
+        path_in_repo=Path(faiss_index_local_path).name,
+        repo_id=repo_name,
+        repo_type=repo_type,
+    )
+
+    # --- Upload new Dataset to HuggingFace
+    # Dropping FAISS index
+    dataset_with_embeddings_obj.drop_index(
+        index_name=semantic_search_obj.embeddings_colname
+    )
+
+    # Pushing dataset to HuggingFace
+    hf_obj.push_dataset(
+        dataset=dataset_with_embeddings_obj,
+        dataset_name=params_dict["output_dataset_name"],
+        username=hf_obj.username,
+    )
+
+    return
+
+
+# ------------------------------ MAIN FUNCTIONS -------------------------------
+
+
+def main(params_dict: Dict):
+    """
+    Main function for creating a dataset with FAISS index.
+    """
+    # Showing set of input parameters
+    gu.show_params(params_dict=params_dict, logger=logger)
+
+    # Create FAISS index and Text embeddings for the dataset.
+    create_faiss_index_and_embeddings_from_dataset(params_dict=params_dict)
+
+    return
+
+
+if __name__ == "__main__":
+    # Getting input parameters
+    params_dict = vars(get_parser())
+    # Running main function
+    main(params_dict=params_dict)

src/utils/__init__.py

@@ -0,0 +1,21 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.

src/utils/default_variables.py

@@ -0,0 +1,76 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Module containing the set of default variables of the project.
+"""
+
+# Option for saving the output data to disk
+save_to_disk = True
+
+# URL to the CICERO dataset
+cicero_dataset_url = "https://raw.githubusercontent.com/hamzafarooq/maven-mlsystem-design-cohort-1/main/data/df_embed.csv"  # noqa: E501
+
+# Option for saving to disk
+save_to_disk = True
+
+# Name of the column that corresponds to the Document ID
+document_id_colname = "_id"
+
+# Name of the column that corresponds to the title of the document.
+title_colname = "title"
+
+# Name of the column that contains the content of the document.
+content_colname = "content"
+
+# Name of teh target column name that will contain the parsed / clean version
+# of the document's content.
+clean_content_colname = "clean_content"
+
+# Name of the 'raw' dataset
+raw_dataset_name = "cicero_raw_dataset"
+
+# Name of the 'clean' dataset
+clean_dataset_name = "cicero_clean_dataset"
+
+# Name of the dataset with summaries
+summaries_dataset_name = "cicero_dataset_with_summaries"
+
+# Name of the dataaset with embeddings and FAISS index
+dataset_faiss_embeddings_name = (
+    "cicero_dataset_with_embeddings_and_faiss_index"
+)
+
+# Name of the environment variable with the HuggingFace Token
+hugging_face_token_name = "HUGGING_FACE_HUB_TOKEN"
+
+# Name of the environment variable with the HuggingFace Username
+hugging_face_username_name = "HUGGING_FACE_USERNAME"
+
+# Name of the HuggingFace repository
+hugging_face_repository_name = "cicero_synthesizer"
+
+# Name of the FAISS Index
+faiss_index_name = "cicero_faiss_index"
+
+# Name of the column that contains the embedding in the dataset
+embeddings_colname = "embeddings"

src/utils/general_utilities.py

@@ -0,0 +1,181 @@
+# MIT License
+#
+# Copyright (c) 2023 Victor Calderon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Module that includes general utitlity functions.
+"""
+
+import argparse as argparse
+import logging as logging
+import re
+from argparse import ArgumentParser as _ArgumentParser
+from argparse import HelpFormatter as _HelpFormatter
+from operator import attrgetter as _attrgetter
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+logger.setLevel(level=logging.INFO)
+
+
+__all__ = ["get_project_paths"]
+
+
+def _get_root_dir():
+    """
+    Function for determining the path to the root directory of the project.
+
+    Returns
+    ----------
+    root_dir : str
+        Path to the root directory of the project.
+    """
+
+    return str(list(Path(__file__).resolve().parents)[2].resolve())
+
+
+def get_project_paths() -> Dict[str, Path]:
+    """
+    Function to extract the set of directories of the project.
+
+    Returns
+    ----------
+    proj_dict : dict
+        Dictionary containing the path to the project's directories.
+    """
+    # --- Defining set of directories
+    # Base directory
+    base_dir = Path(_get_root_dir())
+    # Data directory
+    data_dir = base_dir.joinpath("data").resolve()
+    # Source directory / Codebase
+    src_dir = base_dir.joinpath("src").resolve()
+
+    # --- Creating project dictionary with the project directories
+    proj_dict = {
+        "base": base_dir,
+        "data": data_dir,
+        "src": src_dir,
+    }
+
+    # --- Making sure the directories exist
+    for directory in proj_dict.values():
+        directory.mkdir(
+            exist_ok=True,
+            parents=True,
+        )
+
+    return proj_dict
+
+
+def is_float(s: str):
+    """
+    Function that checks whether or not ``s` is a string.
+    """
+    return s.count(".") == 1 and s.replace(".", "").isdigit()
+
+
+def _str2bool(v):
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
+class SortingHelpFormatter(_HelpFormatter):
+    def add_arguments(self, actions):
+        """
+        Modifier for `argparse` help parameters, that sorts them alphabetically
+        """
+        actions = sorted(actions, key=_attrgetter("option_strings"))
+        super(SortingHelpFormatter, self).add_arguments(actions)
+
+
+def _get_parser_obj(description: str):
+    """
+    Function to create an 'argparse' ``parser`` object.
+    """
+
+    return _ArgumentParser(
+        description=description,
+        formatter_class=SortingHelpFormatter,
+    )
+
+
+def show_params(
+    params_dict: Dict,
+    logger: logging.Logger,
+    columns_to_omit: Optional[Union[List, None]] = None,
+):
+    """
+    Function to show the defined of the class.
+    """
+    # Checking input parameters
+    columns_to_omit = columns_to_omit or []
+    #
+    msg = "\n" + "-" * 50 + "\n"
+    msg += "\t---- INPUT PARAMETERS ----" + "\n"
+    msg += "" + "\n"
+    # Sorting keys of dictionary
+    keys_sorted = np.sort(list(params_dict.keys()))
+    for key_ii in keys_sorted:
+        if key_ii not in columns_to_omit:
+            msg += f"\t>>> {key_ii} : {params_dict[key_ii]}\n"
+    #
+    msg += "\n" + "-" * 50 + "\n"
+    logger.info(msg)
+
+    return
+
+
+def check_url_or_file_type(object_path: str) -> str:
+    """
+    Function to determine whether the input variable is a file or a URL.
+
+    Parameters
+    ------------
+    object_path : str
+        Path to the object.
+
+    Returns
+    ------------
+    object_type : str
+        Type of the object.
+
+        Options :
+            - `url` : The object is a valid URL
+            - `file` : The object corresponds to a local file.
+            - `unspecified` : This object is neither a file nor a URL.
+    """
+    # Set of regular expressions for each type
+    url_pattern = r"^https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+/?\S*$"
+
+    if re.match(url_pattern, object_path):
+        return "url"
+
+    # Checking if 'object_path' is a file or directory
+    return "file" if Path(object_path).is_file() else "unspecified"

src/utils/gpt35_summaries/cleanup_and_summarize.py

@@ -0,0 +1,107 @@
+# sourcery skip: use-named-expression
+import csv
+import sys
+from pathlib import Path
+
+from .summarizer import Summarizer
+
+P_SCRIPT_DIR = Path(__file__).parent
+
+csv.field_size_limit(sys.maxsize)
+
+
+HTML_STRINGS = []
+with open(P_SCRIPT_DIR / "html_tags.txt") as f:
+    for line in f:
+        tag = line.strip()
+        if tag:
+            HTML_STRINGS.append(tag)
+HTML_STRINGS.extend(("target_blank", "relnoopen", "relnofollow"))
+HTML_STRINGS = tuple(HTML_STRINGS)
+
+WORDS = set()
+with open(P_SCRIPT_DIR / "words_alpha.txt") as f:
+    for line in f:
+        word = line.strip()
+        if word:
+            WORDS.add(word)
+
+
+def filter_content(content):
+    c_words = content.split()
+    c_filt_words = []
+    for w in c_words:
+        if w not in WORDS:
+            while w.startswith(HTML_STRINGS):
+                smax = ""
+                for s in HTML_STRINGS:
+                    if w.startswith(s) and len(s) > len(smax):
+                        smax = s
+                w = w[len(smax) :]  # noqa: E203
+            while w.endswith(HTML_STRINGS):
+                smax = ""
+                for s in HTML_STRINGS:
+                    if w.endswith(s) and len(s) > len(smax):
+                        smax = s
+                w = w[len(smax) :]  # noqa: E203
+        if w:
+            c_filt_words.append(w)
+    return " ".join(c_filt_words)
+
+
+def main():
+    DF_EMBED_OUT_DICT = {}
+    if (P_SCRIPT_DIR / "df_embed_out.csv").exists():
+        with open(
+            P_SCRIPT_DIR / "df_embed_out.csv",
+            encoding="ascii",
+            errors="ignore",
+        ) as fin:
+            for csv_row in csv.DictReader(fin):
+                DF_EMBED_OUT_DICT[csv_row["_id"]] = csv_row
+
+    SUMMARIZER = Summarizer()
+
+    with open(
+        P_SCRIPT_DIR / "df_embed.csv", encoding="ascii", errors="ignore"
+    ) as fin, open(
+        P_SCRIPT_DIR / "df_embed_out.csv",
+        "w",
+        encoding="ascii",
+        errors="ignore",
+    ) as fout:
+        csv_reader = csv.DictReader(fin)
+        fieldnames = csv_reader.fieldnames[:]
+        fieldnames.append("summary")
+        fieldnames.append("content_filtered")
+        csv_writer = csv.DictWriter(fout, fieldnames)
+        csv_writer.writeheader()
+        for csv_row in csv_reader:
+            if csv_row["_id"] in DF_EMBED_OUT_DICT:
+                print("Re-using existing data for", csv_row["_id"])
+                csv_row = DF_EMBED_OUT_DICT[csv_row["_id"]]
+            if not csv_row["title"] and not csv_row["content"]:
+                csv_row["content"] = csv_row["_id"]
+                csv_row["_id"] = ""
+            content_filtered = filter_content(csv_row["content"])
+            print(content_filtered)
+            csv_row["content_filtered"] = content_filtered
+            # input()
+            if not csv_row.get("summary") and (
+                csv_row["title"] or csv_row["content_filtered"]
+            ):
+                print("Running GPT...\n")
+                while True:
+                    summary = SUMMARIZER.summarize(
+                        csv_row["title"], content_filtered
+                    )
+                    if summary:
+                        break
+                # input()
+                csv_row["summary"] = summary
+            csv_writer.writerow(csv_row)
+            fout.flush()
+
+
+if __name__ == "__main__":
+    main()

src/utils/gpt35_summaries/html_tags.txt

@@ -0,0 +1,109 @@
+a
+abbr
+address
+area
+article
+aside
+audio
+b
+base
+bdi
+bdo
+blockquote
+body
+br
+button
+canvas
+caption
+cite
+code
+col
+colgroup
+data
+datalist
+dd
+del
+details
+dfn
+dialog
+div
+dl
+dt
+em
+embed
+fieldset
+figcaption
+figure
+footer
+form
+h1
+h2
+h3
+h4
+h5
+h6
+head
+header
+hr
+html
+i
+iframe
+img
+input
+ins
+kbd
+label
+legend
+li
+link
+main
+map
+mark
+meta
+meter
+nav
+noscript
+object
+ol
+optgroup
+option
+output
+p
+param
+picture
+pre
+progress
+q
+rp
+rt
+ruby
+s
+samp
+script
+section
+select
+small
+source
+span
+strong
+style
+sub
+summary
+sup
+table
+tbody
+td
+template
+textarea
+tfoot
+th
+thead
+time
+title
+tr
+track
+u
+ul
+var
+video
+wbr

src/utils/gpt35_summaries/summarizer.py

@@ -0,0 +1,157 @@
+import json
+import os
+import sys
+from typing import Optional
+
+import requests
+import tiktoken
+
+
+class Summarizer:
+    def __init__(self, **kwargs):
+        self.openai_endpoint = "https://api.openai.com/v1/chat/completions"
+
+        # Prompt template
+        self.prompt_template = self._get_prompt_template()
+
+        # Type of model to use
+        self.model = kwargs.get("model", "gpt-3.5-turbo")
+
+        # Model hyperparameters
+        self.max_tokens = kwargs.get("max_tokens", 4096)
+        self.result_tokens = kwargs.get("result_tokens", 300)
+
+        # Model encoding
+        self.model_encoding = self._get_model_encoding()
+
+        # Token length of the prompt template
+        self.prompt_token_length = self._get_number_of_tokens(
+            self.prompt_template
+        )
+
+    def _get_prompt_template(self, search_string=None) -> str:
+        # Defining the template to use
+        template_text = """
+    Create a concise, clear, and in-depth summary of the following online
+    article. Adhere to the following guidelines:
+
+    1. Sound professional, detached and avoid emotionally charged language.
+    2. Make sure to describe who is discussed in the article, what are
+the events or concepts, when things happened, and, if this information is
+available, why.
+    3. The summary should be between one and three paragraphs.
+"""
+        if search_string:
+            template_text += f"""
+    4. Make sure to include and emphasize any information in the article that
+relates to the following search string:
+"{search_string}"
+    """
+
+        return template_text
+
+    def _get_model_encoding(self):
+        return tiktoken.encoding_for_model(self.model)
+
+    def _get_number_of_tokens(self, input_text: str) -> int:
+        """
+        Method for determining the number of tokens of the input text.
+
+        Parameters
+        -----------
+        input_text : str
+            Text to use for calculating its token length.
+
+        Returns
+        ---------
+        text_token_length : int
+            Lenght of the tokens of the input text.
+        """
+
+        return len(self.model_encoding.encode(input_text))
+
+    def _run_model(
+        self,
+        user_content: str,
+        search_string: Optional[str] = None,
+        temperature: Optional[float] = 1,
+    ):
+        """
+        Method for running the model that will create the summary for a given
+        observation.
+
+        Parameters
+        ------------
+        user_content : str
+            Content by the user that will be sent to the model via its API.
+
+        temperature : float, optional
+            Amount of ``temperature`` to give to the model. This parameter
+            handles the amount of creativity that the model can have when
+            creating the output response. This variable is set to ``1`` by
+            default.
+
+        Returns
+        ----------
+        """
+        # Creating the headers
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f'Bearer {os.environ["OPENAI_API_KEY"]}',
+        }
+        # Composing the input messages
+        messages = [
+            {
+                "role": "system",
+                "content": self._get_prompt_template(search_string),
+            },
+            {"role": "user", "content": user_content},
+        ]
+        # Parsing the request data
+        request_data = {
+            "model": self.model,
+            "messages": messages,
+            "temperature": temperature,
+        }
+        # Extracting the response from the model's API
+        response = requests.post(
+            self.openai_endpoint,
+            headers=headers,
+            data=json.dumps(request_data),
+            timeout=60,
+        )
+
+        # Checkig if the response was OK
+        if response.status_code == 200:
+            return response.json()["choices"][0]["message"]["content"]
+        else:
+            raise RuntimeError(
+                f"HTTP request failed {response.status_code}, {response.text}"
+            )
+
+    def summarize(self, title, content, search_string=None):
+        content_for_summary = f"{title}\n\n{content}"
+        prompt_token_length = (
+            self.prompt_token_length
+            if search_string
+            else self._get_number_of_tokens(
+                self._get_prompt_template(search_string)
+            )
+        )
+        data_token_length = self._get_number_of_tokens(content_for_summary)
+        while data_token_length + prompt_token_length > self.max_tokens - 10:
+            print("Decimating the content.")
+            content = content.split()
+            del content[::10]
+            content = " ".join(content)
+            content_for_summary = f"{title}\n\n{content}"
+            data_token_length = self._get_number_of_tokens(content_for_summary)
+
+        while True:
+            try:
+                return self._run_model(
+                    user_content=content_for_summary,
+                    search_string=search_string,
+                )
+            except Exception as e:
+                print(e, file=sys.stderr)

template.envrc

@@ -0,0 +1,15 @@
+# -------------------- Defining default environment ---------------------------
+
+# --- Docker BuildKit
+export DOCKER_BUILDKIT_VALUE=1
+
+# --- Project variables
+export INPUT_APP_PORT=8501
+export OUTPUT_APP_PORT=8501
+export APP_SERVER_PORT=7860
+
+export HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN}
+export HUGGING_FACE_USERNAME=${HUGGING_FACE_USERNAME}
+
+export PATH="${PWD}:${PATH}"
+export PYTHONPATH="${PWD}:${PYTHONPATH}"