fixed dependency generation

pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "mpi-data-store"
+version = "0.1.0"
+description = ""
+authors = ["ricklon <rianders@docs.rutgers.edu>"]
+readme = "README.md"
+packages = [{include = "mpi_data_store"}]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+langchain = "^0.1.13"
+pypdf = "^4.1.0"
+langchain-community = "^0.0.29"
+sentence-transformers = "^2.6.0"
+faiss-cpu = "^1.8.0"
+pandas = "^2.2.1"
+tqdm = "^4.66.2"
+streamlit = "^1.32.2"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,101 @@
+aiohttp==3.9.3 ; python_version >= "3.10" and python_version < "4.0"
+aiosignal==1.3.1 ; python_version >= "3.10" and python_version < "4.0"
+altair==5.2.0 ; python_version >= "3.10" and python_version < "4.0"
+annotated-types==0.6.0 ; python_version >= "3.10" and python_version < "4.0"
+anyio==4.3.0 ; python_version >= "3.10" and python_version < "4.0"
+async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11"
+attrs==23.2.0 ; python_version >= "3.10" and python_version < "4.0"
+blinker==1.7.0 ; python_version >= "3.10" and python_version < "4.0"
+cachetools==5.3.3 ; python_version >= "3.10" and python_version < "4.0"
+certifi==2024.2.2 ; python_version >= "3.10" and python_version < "4.0"
+charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "4.0"
+click==8.1.7 ; python_version >= "3.10" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and platform_system == "Windows"
+dataclasses-json==0.6.4 ; python_version >= "3.10" and python_version < "4.0"
+exceptiongroup==1.2.0 ; python_version >= "3.10" and python_version < "3.11"
+faiss-cpu==1.8.0 ; python_version >= "3.10" and python_version < "4.0"
+filelock==3.13.1 ; python_version >= "3.10" and python_version < "4.0"
+frozenlist==1.4.1 ; python_version >= "3.10" and python_version < "4.0"
+fsspec==2024.3.1 ; python_version >= "3.10" and python_version < "4.0"
+gitdb==4.0.11 ; python_version >= "3.10" and python_version < "4.0"
+gitpython==3.1.42 ; python_version >= "3.10" and python_version < "4.0"
+greenlet==3.0.3 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "aarch64" or python_version >= "3.10" and python_version < "4.0" and platform_machine == "ppc64le" or python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" or python_version >= "3.10" and python_version < "4.0" and platform_machine == "amd64" or python_version >= "3.10" and python_version < "4.0" and platform_machine == "AMD64" or python_version >= "3.10" and python_version < "4.0" and platform_machine == "win32" or python_version >= "3.10" and python_version < "4.0" and platform_machine == "WIN32"
+huggingface-hub==0.21.4 ; python_version >= "3.10" and python_version < "4.0"
+idna==3.6 ; python_version >= "3.10" and python_version < "4.0"
+jinja2==3.1.3 ; python_version >= "3.10" and python_version < "4.0"
+joblib==1.3.2 ; python_version >= "3.10" and python_version < "4.0"
+jsonpatch==1.33 ; python_version >= "3.10" and python_version < "4.0"
+jsonpointer==2.4 ; python_version >= "3.10" and python_version < "4.0"
+jsonschema-specifications==2023.12.1 ; python_version >= "3.10" and python_version < "4.0"
+jsonschema==4.21.1 ; python_version >= "3.10" and python_version < "4.0"
+langchain-community==0.0.29 ; python_version >= "3.10" and python_version < "4.0"
+langchain-core==0.1.33 ; python_version >= "3.10" and python_version < "4.0"
+langchain-text-splitters==0.0.1 ; python_version >= "3.10" and python_version < "4.0"
+langchain==0.1.13 ; python_version >= "3.10" and python_version < "4.0"
+langsmith==0.1.31 ; python_version >= "3.10" and python_version < "4.0"
+markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "4.0"
+markupsafe==2.1.5 ; python_version >= "3.10" and python_version < "4.0"
+marshmallow==3.21.1 ; python_version >= "3.10" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.10" and python_version < "4.0"
+mpmath==1.3.0 ; python_version >= "3.10" and python_version < "4.0"
+multidict==6.0.5 ; python_version >= "3.10" and python_version < "4.0"
+mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "4.0"
+networkx==3.2.1 ; python_version >= "3.10" and python_version < "4.0"
+numpy==1.26.4 ; python_version < "4.0" and python_version >= "3.10"
+nvidia-cublas-cu12==12.1.3.1 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cuda-cupti-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cuda-nvrtc-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cuda-runtime-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cudnn-cu12==8.9.2.26 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cufft-cu12==11.0.2.54 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-curand-cu12==10.3.2.106 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cusolver-cu12==11.4.5.107 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-cusparse-cu12==12.1.0.106 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-nccl-cu12==2.19.3 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-nvjitlink-cu12==12.4.99 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+nvidia-nvtx-cu12==12.1.105 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version >= "3.10" and python_version < "4.0"
+orjson==3.9.15 ; python_version >= "3.10" and python_version < "4.0"
+packaging==23.2 ; python_version >= "3.10" and python_version < "4.0"
+pandas==2.2.1 ; python_version >= "3.10" and python_version < "4.0"
+pillow==10.2.0 ; python_version >= "3.10" and python_version < "4.0"
+protobuf==4.25.3 ; python_version >= "3.10" and python_version < "4.0"
+pyarrow==15.0.2 ; python_version >= "3.10" and python_version < "4.0"
+pydantic-core==2.16.3 ; python_version >= "3.10" and python_version < "4.0"
+pydantic==2.6.4 ; python_version >= "3.10" and python_version < "4.0"
+pydeck==0.8.1b0 ; python_version >= "3.10" and python_version < "4.0"
+pygments==2.17.2 ; python_version >= "3.10" and python_version < "4.0"
+pypdf==4.1.0 ; python_version >= "3.10" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "4.0"
+pytz==2024.1 ; python_version >= "3.10" and python_version < "4.0"
+pyyaml==6.0.1 ; python_version >= "3.10" and python_version < "4.0"
+referencing==0.34.0 ; python_version >= "3.10" and python_version < "4.0"
+regex==2023.12.25 ; python_version >= "3.10" and python_version < "4.0"
+requests==2.31.0 ; python_version >= "3.10" and python_version < "4.0"
+rich==13.7.1 ; python_version >= "3.10" and python_version < "4.0"
+rpds-py==0.18.0 ; python_version >= "3.10" and python_version < "4.0"
+safetensors==0.4.2 ; python_version >= "3.10" and python_version < "4.0"
+scikit-learn==1.4.1.post1 ; python_version >= "3.10" and python_version < "4.0"
+scipy==1.12.0 ; python_version >= "3.10" and python_version < "4.0"
+sentence-transformers==2.6.0 ; python_version >= "3.10" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.10" and python_version < "4.0"
+smmap==5.0.1 ; python_version >= "3.10" and python_version < "4.0"
+sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0"
+sqlalchemy==2.0.29 ; python_version >= "3.10" and python_version < "4.0"
+streamlit==1.32.2 ; python_version >= "3.10" and python_version < "4.0"
+sympy==1.12 ; python_version >= "3.10" and python_version < "4.0"
+tenacity==8.2.3 ; python_version >= "3.10" and python_version < "4.0"
+threadpoolctl==3.4.0 ; python_version >= "3.10" and python_version < "4.0"
+tokenizers==0.15.2 ; python_version >= "3.10" and python_version < "4.0"
+toml==0.10.2 ; python_version >= "3.10" and python_version < "4.0"
+toolz==0.12.1 ; python_version >= "3.10" and python_version < "4.0"
+torch==2.2.1 ; python_version >= "3.10" and python_version < "4.0"
+tornado==6.4 ; python_version >= "3.10" and python_version < "4.0"
+tqdm==4.66.2 ; python_version >= "3.10" and python_version < "4.0"
+transformers==4.39.1 ; python_version >= "3.10" and python_version < "4.0"
+triton==2.2.0 ; platform_system == "Linux" and platform_machine == "x86_64" and python_version < "3.12" and python_version >= "3.10"
+typing-extensions==4.10.0 ; python_version >= "3.10" and python_version < "4.0"
+typing-inspect==0.9.0 ; python_version >= "3.10" and python_version < "4.0"
+tzdata==2024.1 ; python_version >= "3.10" and python_version < "4.0"
+urllib3==2.2.1 ; python_version >= "3.10" and python_version < "4.0"
+watchdog==4.0.0 ; python_version >= "3.10" and python_version < "4.0" and platform_system != "Darwin"
+yarl==1.9.4 ; python_version >= "3.10" and python_version < "4.0"

save_mpi_mining_data_as_text_backup.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+"""Save  MPI Mining Data as Text Backup
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1bpGk38t5or9A4MCj8QVPLdTRH0D9qIN-
+"""
+
+!git clone https://github.com/ricklon/mpi_data.git
+# todo: pages store the enture colletion to embedding as well.
+
+!pip install -q langchain pypdf langchain_community sentence_transformers faiss-cpu pandas tqdm
+
+import os
+import  pandas as pd
+from io import StringIO
+from tqdm import tqdm
+import time
+import datetime
+
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnablePassthrough
+from langchain_core.output_parsers import StrOutputParser
+
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.llms import HuggingFaceEndpoint
+
+from langchain.document_loaders import PyPDFDirectoryLoader
+
+EMBEDDING_MODEL_NAME = "thenlper/gte-small"
+HG_MODEL = "mistralai/Mistral-7B-Instruct-v0.2"
+# hg_model = "HuggingFaceH4/zephyr-7b-beta"
+#hg_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" ## Tiny LLAMA not supported
+
+llm = HuggingFaceEndpoint(repo_id=HG_MODEL,
+  max_new_tokens=250,
+  top_k=10,
+  top_p=0.95,
+  typical_p=0.95,
+  temperature=0.01,
+  repetition_penalty=1.035)
+
+VECTOR_SOURCE="miningdata"
+
+embedding_model = HuggingFaceEmbeddings(
+    model_name=EMBEDDING_MODEL_NAME,
+    multi_process=True,
+    model_kwargs={"device": "cpu"},
+    encode_kwargs={"normalize_embeddings": True},  # set True for cosine similarity
+)

utils_cli.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Function to export requirements using Poetry
+export_requirements() {
+    poetry export --without-hashes --format=requirements.txt --output=requirements.txt
+}
+
+# Main function
+main() {
+    echo "Choose an option:"
+    echo "1. Export requirements using Poetry"
+    echo "2. Option 2"
+    echo "3. Option 3"
+    echo "4. Option 4"
+    echo "5. Option 5"
+    read -p "Enter your choice [1-5]: " choice
+
+    case $choice in
+        1)
+            export_requirements
+            ;;
+        2)
+            echo "You chose option 2"
+            # Add your code for option 2 here
+            ;;
+        3)
+            echo "You chose option 3"
+            # Add your code for option 3 here
+            ;;
+        4)
+            echo "You chose option 4"
+            # Add your code for option 4 here
+            ;;
+        5)
+            echo "You chose option 5"
+            # Add your code for option 5 here
+            ;;
+        *)
+            echo "Invalid choice. Please enter a number from 1 to 5."
+            ;;
+    esac
+}
+
+# Call the main function
+main