experiment with new ai-tutor-db

data/scraping/huggingface_docs/parse_hf_html.py

@@ -147,10 +147,13 @@ def save_to_jsonl(data, output_file):
 
 
 def main():
-    # html_dir = "huggingface_docs"  # Directory where HTML files are saved
-    html_dir = "transformers_docs_v4.42.0"  # Directory where HTML files are saved
-    base_url = "https://huggingface.co/docs/transformers/"
-    output_file = "hf_transformers_v4_42_0.jsonl"
+    # html_dir = "transformers_docs_v4.42.0"  # Directory where HTML files are saved
+    # base_url = "https://huggingface.co/docs/transformers/"
+
+    html_dir = "peft_docs_v0.11.0"  # Directory where HTML files are saved
+    base_url = "https://huggingface.co/docs/peft/"
+
+    output_file = "hf_peft_v0_11_0.jsonl"
 
     all_sections = parse_saved_html_files(html_dir, base_url)
     save_to_jsonl(all_sections, output_file)

data/scraping/huggingface_docs/scrape_hf_docs_from_web.py

@@ -70,6 +70,7 @@ class DocsSpider(scrapy.Spider):
             f.write(response.body)
 
         self.pages.append({"url": response.url, "html": response.body})
+        # if self.progress_bar:
         self.progress_bar.update(1)
 
         for href in response.css("a::attr(href)").getall():

scripts/create_db.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Create AI-Tutor vector database"
+    "# Create AI-Tutor vector database\n"
    ]
   },
   {
@@ -13,7 +13,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
     "from dotenv import load_dotenv\n",
     "\n",
     "load_dotenv(\"../.env\")"
@@ -30,18 +29,76 @@
     "nest_asyncio.apply()"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Clean data\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "import chromadb\n",
+    "import json\n",
+    "import tiktoken\n",
+    "from collections import OrderedDict\n",
+    "\n",
+    "\n",
+    "def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
+    "    \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
+    "    encoding = tiktoken.get_encoding(encoding_name)\n",
+    "    num_tokens = len(\n",
+    "        encoding.encode(\n",
+    "            string, disallowed_special=(encoding.special_tokens_set - {\"<|endoftext|>\"})\n",
+    "        )\n",
+    "    )\n",
+    "    return num_tokens\n",
+    "\n",
+    "\n",
+    "def clean_jsonl_file(input_filepath, output_filepath):\n",
+    "    cleaned_data = []\n",
+    "\n",
+    "    with open(input_filepath, \"r\") as file:\n",
+    "        for line in file:\n",
+    "            json_obj = json.loads(line)\n",
+    "            content = json_obj.get(\"content\", \"\")\n",
+    "            token_count = num_tokens_from_string(content, \"cl100k_base\")\n",
+    "\n",
+    "            # Check conditions for keeping the line\n",
+    "            if token_count > 7 and not (\n",
+    "                token_count == 92 and json_obj.get(\"name\") == \"Transformers\"\n",
+    "            ):\n",
+    "                # Create a new OrderedDict with 'tokens' as the first key\n",
+    "                new_obj = OrderedDict([(\"tokens\", token_count)])\n",
+    "                # Add the rest of the key-value pairs from the original object\n",
+    "                new_obj.update(json_obj)\n",
+    "                cleaned_data.append(new_obj)\n",
+    "\n",
+    "    with open(output_filepath, \"w\") as file:\n",
+    "        for item in cleaned_data:\n",
+    "            json.dump(item, file)\n",
+    "            file.write(\"\\n\")\n",
+    "\n",
+    "    print(f\"Original number of lines: {sum(1 for _ in open(input_filepath))}\")\n",
+    "    print(f\"Cleaned number of lines: {len(cleaned_data)}\")\n",
+    "\n",
+    "\n",
+    "# Usage\n",
+    "input_filepath = \"../hf_transformers_v4_42_0.jsonl\"\n",
+    "output_filepath = \"../hf_transformers_v4_42_0_cleaned.jsonl\"\n",
+    "clean_jsonl_file(input_filepath, output_filepath)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Merges lines by 'URL' and creates a new file with the merged data.\n",
     "\n",
-    "# create client and a new collection\n",
-    "# chromadb.EphemeralClient saves data in-memory.\n",
-    "chroma_client = chromadb.PersistentClient(path=\"./ai-tutor-db\")\n",
-    "chroma_collection = chroma_client.create_collection(\"ai-tutor-db\")"
+    "Fixes the 'name'\n"
    ]
   },
   {
@@ -50,13 +107,124 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
-    "from llama_index.core import StorageContext\n",
+    "import json\n",
+    "from collections import defaultdict\n",
+    "import tiktoken\n",
+    "\n",
+    "\n",
+    "def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
+    "    \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
+    "    encoding = tiktoken.get_encoding(encoding_name)\n",
+    "    num_tokens = len(\n",
+    "        encoding.encode(\n",
+    "            string, disallowed_special=(encoding.special_tokens_set - {\"<|endoftext|>\"})\n",
+    "        )\n",
+    "    )\n",
+    "    return num_tokens\n",
+    "\n",
+    "\n",
+    "def should_not_merge(url):\n",
+    "    \"\"\"Check if the URL contains any of the exclusion patterns.\"\"\"\n",
+    "    exclusion_patterns = [\"model_doc\", \"internal\", \"main_classes\"]\n",
+    "    return any(pattern in url for pattern in exclusion_patterns)\n",
+    "\n",
+    "\n",
+    "def merge_jsonl(input_file, output_file):\n",
+    "    # Dictionary to store merged data\n",
+    "    merged_data = defaultdict(list)\n",
+    "\n",
+    "    # Read and process the input file\n",
+    "    with open(input_file, \"r\") as f:\n",
+    "        for line in f:\n",
+    "            data = json.loads(line)\n",
+    "            url = data[\"url\"]\n",
+    "            merged_data[url].append(data)\n",
+    "\n",
+    "    # Write the merged data to the output file\n",
+    "    with open(output_file, \"w\") as f:\n",
+    "        for url, entries in merged_data.items():\n",
+    "            if len(entries) == 1 or should_not_merge(url):\n",
+    "                # If there's only one entry or it shouldn't be merged, write all entries as is\n",
+    "                for entry in entries:\n",
+    "                    entry[\"retrieve_doc\"] = False\n",
+    "                    json.dump(entry, f)\n",
+    "                    f.write(\"\\n\")\n",
+    "            else:\n",
+    "                # Merge the entries\n",
+    "                merged_entry = entries[0].copy()\n",
+    "                merged_entry[\"content\"] = \"\\n\\n\".join(\n",
+    "                    entry[\"content\"] for entry in entries\n",
+    "                )\n",
+    "                merged_entry[\"tokens\"] = num_tokens_from_string(\n",
+    "                    merged_entry[\"content\"], \"cl100k_base\"\n",
+    "                )\n",
+    "                merged_entry[\"retrieve_doc\"] = True\n",
+    "                json.dump(merged_entry, f)\n",
+    "                f.write(\"\\n\")\n",
+    "\n",
+    "\n",
+    "# Usage\n",
+    "input_file = \"../hf_transformers_v4_42_0_cleaned.jsonl\"\n",
+    "output_file = \"../hf_transformers_v4_42_0_merged.jsonl\"\n",
+    "merge_jsonl(input_file, output_file)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Count tokens of lines in merged file\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import json\n",
+    "# import tiktoken\n",
+    "\n",
     "\n",
-    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "# def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
+    "#     \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
+    "#     encoding = tiktoken.get_encoding(encoding_name)\n",
+    "#     num_tokens = len(\n",
+    "#         encoding.encode(\n",
+    "#             string, disallowed_special=(encoding.special_tokens_set - {\"<|endoftext|>\"})\n",
+    "#         )\n",
+    "#     )\n",
+    "#     return num_tokens\n",
     "\n",
-    "# Define a storage context object using the created vector store.\n",
-    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+    "\n",
+    "# def count_tokens(input_file):\n",
+    "\n",
+    "#     # Read and process the input file\n",
+    "#     with open(input_file, \"r\") as f:\n",
+    "#         for i, line in enumerate(f):\n",
+    "#             data = json.loads(line)\n",
+    "#             content = data[\"content\"]\n",
+    "#             nb_tokens = num_tokens_from_string(content, \"cl100k_base\")\n",
+    "#             # print(i + 1, data[\"url\"], nb_tokens)\n",
+    "#             if nb_tokens > 2000:\n",
+    "#                 print(i + 1, data[\"url\"], data[\"name\"], nb_tokens)\n",
+    "#             # if nb_tokens < 8:\n",
+    "#             # print(nb_tokens)\n",
+    "#             # print(data[\"url\"])\n",
+    "#             # print(data[\"content\"])\n",
+    "\n",
+    "\n",
+    "# # Usage\n",
+    "# input_file = \"../hf_transformers_v4_42_0_merged.jsonl\"\n",
+    "# # input_file = \"../hf_transformers_v4_42_0.jsonl\"\n",
+    "# count_tokens(input_file)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Create a set of llama-index Documents\n"
    ]
   },
   {
@@ -65,29 +233,47 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from llama_index.core import Document\n",
+    "from llama_index.core.schema import MetadataMode\n",
     "import json\n",
-    "from llama_index.core.schema import TextNode\n",
     "\n",
-    "def load_jsonl_create_nodes(filepath):\n",
-    "    nodes = []  # List to hold the created node objects\n",
-    "    with open(filepath, \"r\") as file:\n",
-    "        for line in file:\n",
-    "            # Load each line as a JSON object\n",
-    "            json_obj = json.loads(line)\n",
-    "            # Extract required information\n",
-    "            title = json_obj.get(\"title\")\n",
-    "            url = json_obj.get(\"url\")\n",
-    "            content = json_obj.get(\"content\")\n",
-    "            source = json_obj.get(\"source\")\n",
-    "            # Create a TextNode object and append to the list\n",
-    "            node = TextNode(\n",
-    "                text=content,\n",
-    "                metadata={\"title\": title, \"url\": url, \"source\": source},\n",
-    "                excluded_embed_metadata_keys=[\"title\", \"url\", \"source\"],\n",
-    "                excluded_llm_metadata_keys=[\"title\", \"url\", \"source\"],\n",
+    "\n",
+    "def create_docs(input_file):\n",
+    "    with open(input_file, \"r\") as f:\n",
+    "        documents = []\n",
+    "        for i, line in enumerate(f):\n",
+    "            data = json.loads(line)\n",
+    "            documents.append(\n",
+    "                Document(\n",
+    "                    text=data[\"content\"],\n",
+    "                    metadata={\n",
+    "                        \"url\": data[\"url\"],\n",
+    "                        \"title\": data[\"name\"],\n",
+    "                        \"tokens\": data[\"tokens\"],\n",
+    "                        \"retrieve_doc\": data[\"retrieve_doc\"],\n",
+    "                    },\n",
+    "                    excluded_llm_metadata_keys=[\n",
+    "                        \"url\",\n",
+    "                        \"title\",\n",
+    "                        \"tokens\",\n",
+    "                        \"retrieve_doc\",\n",
+    "                    ],\n",
+    "                    excluded_embed_metadata_keys=[\n",
+    "                        \"url\",\n",
+    "                        \"title\",\n",
+    "                        \"tokens\",\n",
+    "                        \"retrieve_doc\",\n",
+    "                    ],\n",
+    "                )\n",
     "            )\n",
-    "            nodes.append(node)\n",
-    "    return nodes"
+    "        return documents\n",
+    "\n",
+    "\n",
+    "documents = create_docs(\"../hf_transformers_v4_42_0_merged.jsonl\")\n",
+    "print(documents[0])\n",
+    "print(documents[0].metadata)\n",
+    "\n",
+    "document_dict = {doc.doc_id: doc for doc in documents}"
    ]
   },
   {
@@ -96,18 +282,265 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "filepath = \"../combined_data.jsonl\"\n",
-    "nodes = load_jsonl_create_nodes(filepath)\n",
+    "# import chromadb\n",
     "\n",
-    "print(f\"Loaded {len(nodes)} nodes/chunks from the JSONL file\\n \")\n",
+    "# # create client and a new collection\n",
+    "# # chromadb.EphemeralClient saves data in-memory.\n",
+    "# chroma_client = chromadb.PersistentClient(path=\"./ai-tutor-dataset\")\n",
+    "# chroma_collection = chroma_client.create_collection(\"ai-tutor-dataset\")\n",
     "\n",
-    "node = nodes[0]\n",
-    "print(f\"ID: {node.id_} \\nText: {node.text}, \\nMetadata: {node.metadata}\")\n",
+    "# from llama_index.vector_stores.chroma import ChromaVectorStore\n",
+    "# from llama_index.core import StorageContext\n",
     "\n",
-    "print(\"\\n\")\n",
+    "# # Define a storage context object using the created vector database.\n",
+    "# vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "# storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core import VectorStoreIndex\n",
+    "from llama_index.core.node_parser import SentenceSplitter\n",
+    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
     "\n",
-    "node = nodes[-10000]\n",
-    "print(f\"ID: {node.id_} \\nText: {node.text}, \\nMetadata: {node.metadata}\")"
+    "# Build index / generate embeddings using OpenAI embedding model\n",
+    "index = VectorStoreIndex.from_documents(\n",
+    "    documents,\n",
+    "    # embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\"),\n",
+    "    embed_model=OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"similarity\"),\n",
+    "    transformations=[SentenceSplitter(chunk_size=800, chunk_overlap=400)],\n",
+    "    show_progress=True,\n",
+    "    use_async=True,\n",
+    "    # storage_context=storage_context,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from llama_index.llms.openai import OpenAI\n",
+    "\n",
+    "# llm = OpenAI(temperature=1, model=\"gpt-3.5-turbo\", max_tokens=None)\n",
+    "# query_engine = index.as_query_engine(\n",
+    "#     llm=llm,\n",
+    "#     similarity_top_k=5,\n",
+    "#     embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\"),\n",
+    "#     use_async=True,\n",
+    "# )\n",
+    "retriever = index.as_retriever(\n",
+    "    similarity_top_k=10,\n",
+    "    use_async=True,\n",
+    "    embed_model=OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"similarity\"),\n",
+    "    # embed_model=OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"text_search\"),\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.data_structs import Node\n",
+    "from llama_index.core.schema import NodeWithScore\n",
+    "\n",
+    "# res = query_engine.query(\"What is the LLaMa model?\")\n",
+    "# res.response\n",
+    "\n",
+    "# query = \"fine-tune a pretrained model\"\n",
+    "# query = \"fine-tune an llm\"\n",
+    "query = \"how to fine-tune an llm?\"\n",
+    "\n",
+    "nodes_context = []\n",
+    "nodes = retriever.retrieve(query)\n",
+    "\n",
+    "\n",
+    "# # Filter nodes with the same ref_doc_id\n",
+    "# def filter_nodes_by_unique_doc_id(nodes):\n",
+    "#     unique_nodes = {}\n",
+    "#     for node in nodes:\n",
+    "#         doc_id = node.node.ref_doc_id\n",
+    "#         if doc_id is not None and doc_id not in unique_nodes:\n",
+    "#             unique_nodes[doc_id] = node\n",
+    "#     return list(unique_nodes.values())\n",
+    "\n",
+    "\n",
+    "# nodes = filter_nodes_by_unique_doc_id(nodes)\n",
+    "\n",
+    "for node in nodes:\n",
+    "    print(\"Node ID\\t\", node.node_id)\n",
+    "    print(\"Title\\t\", node.metadata[\"title\"])\n",
+    "    print(\"Text\\t\", node.text)\n",
+    "    print(\"Score\\t\", node.score)\n",
+    "    print(\"Metadata\\t\", node.metadata)\n",
+    "    print(\"-_\" * 20)\n",
+    "    if node.metadata[\"retrieve_doc\"] == True:\n",
+    "        print(\"This node will be replaced by the document\")\n",
+    "        doc = document_dict[node.node.ref_doc_id]\n",
+    "        # print(doc.text)\n",
+    "        new_node = (\n",
+    "            NodeWithScore(\n",
+    "                node=Node(text=doc.text, metadata=node.metadata), score=node.score\n",
+    "            ),\n",
+    "        )\n",
+    "        nodes_context.append(new_node)\n",
+    "    else:\n",
+    "        nodes_context.append(node)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from llama_index.core.schema import TextNode\n",
+    "\n",
+    "# for src in res.source_nodes:\n",
+    "#     print(src.node.ref_doc_id)\n",
+    "#     # print(src.node.get_metadata_str())\n",
+    "#     print(\"Node ID\\t\", src.node_id)\n",
+    "#     print(\"Title\\t\", src.metadata[\"title\"])\n",
+    "#     print(\"Text\\t\", src.text)\n",
+    "#     print(\"Score\\t\", src.score)\n",
+    "#     print(\"Metadata\\t\", src.metadata)\n",
+    "#     print(\"-_\" * 20)\n",
+    "#     break"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llama_index.core.data_structs import Node\n",
+    "from llama_index.core.schema import NodeWithScore\n",
+    "from llama_index.core import get_response_synthesizer\n",
+    "from llama_index.llms.gemini import Gemini\n",
+    "from llama_index.llms.openai import OpenAI\n",
+    "\n",
+    "from tutor_prompts import (\n",
+    "    TEXT_QA_TEMPLATE,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "# llm = Gemini(model=\"models/gemini-1.5-flash\", temperature=1, max_tokens=None)\n",
+    "# llm = Gemini(model=\"models/gemini-1.5-pro\", temperature=1, max_tokens=None)\n",
+    "# llm = OpenAI(temperature=1, model=\"gpt-3.5-turbo\", max_tokens=None)\n",
+    "llm = OpenAI(temperature=1, model=\"gpt-4o\", max_tokens=None)\n",
+    "\n",
+    "response_synthesizer = get_response_synthesizer(\n",
+    "    llm=llm, response_mode=\"simple_summarize\", text_qa_template=TEXT_QA_TEMPLATE\n",
+    ")\n",
+    "\n",
+    "response = response_synthesizer.synthesize(\n",
+    "    query,\n",
+    "    nodes=nodes,\n",
+    "    # nodes=[\n",
+    "    #     NodeWithScore(\n",
+    "    #         node=Node(text=\"LLama2 model has a total of 2B parameters.\"), score=1.0\n",
+    "    #     ),\n",
+    "    # ],\n",
+    "    # text_chunks=[\"text1\", \"text2\", \"text3\"],\n",
+    ")\n",
+    "print(response.response)\n",
+    "# for src in response.source_nodes:\n",
+    "#     print(src.node.ref_doc_id)\n",
+    "#     print(\"Node ID\\t\", src.node_id)\n",
+    "#     print(\"Title\\t\", src.metadata[\"title\"])\n",
+    "#     print(\"Text\\t\", src.text)\n",
+    "#     print(\"Score\\t\", src.score)\n",
+    "#     print(\"Metadata\\t\", src.metadata)\n",
+    "#     print(\"-_\" * 20)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import chromadb\n",
+    "\n",
+    "# # create client and a new collection\n",
+    "# # chromadb.EphemeralClient saves data in-memory.\n",
+    "# chroma_client = chromadb.PersistentClient(path=\"./ai-tutor-db\")\n",
+    "# chroma_collection = chroma_client.create_collection(\"ai-tutor-db\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# from llama_index.vector_stores.chroma import ChromaVectorStore\n",
+    "# from llama_index.core import StorageContext\n",
+    "\n",
+    "# vector_store = ChromaVectorStore(chroma_collection=chroma_collection)\n",
+    "\n",
+    "# # Define a storage context object using the created vector store.\n",
+    "# storage_context = StorageContext.from_defaults(vector_store=vector_store)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import json\n",
+    "# from llama_index.core.schema import TextNode\n",
+    "\n",
+    "\n",
+    "# def load_jsonl_create_nodes(filepath):\n",
+    "#     nodes = []  # List to hold the created node objects\n",
+    "#     with open(filepath, \"r\") as file:\n",
+    "#         for line in file:\n",
+    "#             # Load each line as a JSON object\n",
+    "#             json_obj = json.loads(line)\n",
+    "#             # Extract required information\n",
+    "#             title = json_obj.get(\"title\")\n",
+    "#             url = json_obj.get(\"url\")\n",
+    "#             content = json_obj.get(\"content\")\n",
+    "#             source = json_obj.get(\"source\")\n",
+    "#             # Create a TextNode object and append to the list\n",
+    "#             node = TextNode(\n",
+    "#                 text=content,\n",
+    "#                 metadata={\"title\": title, \"url\": url, \"source\": source},\n",
+    "#                 excluded_embed_metadata_keys=[\"title\", \"url\", \"source\"],\n",
+    "#                 excluded_llm_metadata_keys=[\"title\", \"url\", \"source\"],\n",
+    "#             )\n",
+    "#             nodes.append(node)\n",
+    "#     return nodes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# filepath = \"../combined_data.jsonl\"\n",
+    "# nodes = load_jsonl_create_nodes(filepath)\n",
+    "\n",
+    "# print(f\"Loaded {len(nodes)} nodes/chunks from the JSONL file\\n \")\n",
+    "\n",
+    "# node = nodes[0]\n",
+    "# print(f\"ID: {node.id_} \\nText: {node.text}, \\nMetadata: {node.metadata}\")\n",
+    "\n",
+    "# print(\"\\n\")\n",
+    "\n",
+    "# node = nodes[-10000]\n",
+    "# print(f\"ID: {node.id_} \\nText: {node.text}, \\nMetadata: {node.metadata}\")"
    ]
   },
   {
@@ -138,16 +571,23 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
-    "from llama_index.core import VectorStoreIndex\n",
+    "# from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "# from llama_index.core import VectorStoreIndex\n",
     "\n",
-    "# embeds = OpenAIEmbedding(model=\"text-embedding-3-small\", mode=\"similarity\")\n",
-    "# embeds = OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"similarity\")\n",
-    "embeds = OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"text_search\")\n",
-    "# embeds = OpenAIEmbedding(model=\"text-embedding-ada-002\", mode=\"similarity\")\n",
+    "# # embeds = OpenAIEmbedding(model=\"text-embedding-3-small\", mode=\"similarity\")\n",
+    "# # embeds = OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"similarity\")\n",
+    "# embeds = OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"text_search\")\n",
+    "# # embeds = OpenAIEmbedding(model=\"text-embedding-ada-002\", mode=\"similarity\")\n",
     "\n",
-    "# Build index / generate embeddings using OpenAI.\n",
-    "index = VectorStoreIndex(nodes=nodes, show_progress=True, use_async=True, storage_context=storage_context, embed_model=embeds, insert_batch_size=3000,)"
+    "# # Build index / generate embeddings using OpenAI.\n",
+    "# index = VectorStoreIndex(\n",
+    "#     nodes=nodes,\n",
+    "#     show_progress=True,\n",
+    "#     use_async=True,\n",
+    "#     storage_context=storage_context,\n",
+    "#     embed_model=embeds,\n",
+    "#     insert_batch_size=3000,\n",
+    "# )"
    ]
   },
   {
@@ -156,10 +596,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from llama_index.llms.openai import OpenAI\n",
+    "# from llama_index.llms.openai import OpenAI\n",
     "\n",
-    "llm = OpenAI(temperature=0, model=\"gpt-3.5-turbo\", max_tokens=None)\n",
-    "query_engine = index.as_query_engine(llm=llm, similarity_top_k=5, embed_model=embeds)"
+    "# llm = OpenAI(temperature=0, model=\"gpt-3.5-turbo\", max_tokens=None)\n",
+    "# query_engine = index.as_query_engine(llm=llm, similarity_top_k=5, embed_model=embeds)"
    ]
   },
   {
@@ -168,7 +608,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res = query_engine.query(\"What is the LLaMa model?\")"
+    "# res = query_engine.query(\"What is the LLaMa model?\")"
    ]
   },
   {
@@ -177,7 +617,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "res.response"
+    "# res.response"
    ]
   },
   {
@@ -186,196 +626,128 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "for src in res.source_nodes:\n",
-    "  print(\"Node ID\\t\", src.node_id)\n",
-    "  print(\"Title\\t\", src.metadata['title'])\n",
-    "  print(\"Text\\t\", src.text)\n",
-    "  print(\"Score\\t\", src.score)\n",
-    "  print(\"Metadata\\t\", src.metadata) \n",
-    "  print(\"-_\"*20)"
+    "# for src in res.source_nodes:\n",
+    "#     print(\"Node ID\\t\", src.node_id)\n",
+    "#     print(\"Title\\t\", src.metadata[\"title\"])\n",
+    "#     print(\"Text\\t\", src.text)\n",
+    "#     print(\"Score\\t\", src.score)\n",
+    "#     print(\"Metadata\\t\", src.metadata)\n",
+    "#     print(\"-_\" * 20)"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Load DB from disk"
+    "# Load DB from disk\n"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:chromadb.telemetry.product.posthog:Anonymized telemetry enabled. See                     https://docs.trychroma.com/telemetry for more information.\n",
-      "INFO:chromadb.api.segment:Collection ai-tutor-db is not created.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "import logging\n",
+    "# import logging\n",
     "\n",
-    "logger = logging.getLogger(__name__)\n",
-    "logging.basicConfig(level=logging.INFO)\n",
+    "# logger = logging.getLogger(__name__)\n",
+    "# logging.basicConfig(level=logging.INFO)\n",
     "\n",
     "\n",
-    "import chromadb\n",
-    "from llama_index.vector_stores.chroma import ChromaVectorStore\n",
-    "# Create your index\n",
-    "db2 = chromadb.PersistentClient(path=\"./ai-tutor-db\")\n",
-    "chroma_collection = db2.get_or_create_collection(\"ai-tutor-db\")\n",
-    "vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
+    "# import chromadb\n",
+    "# from llama_index.vector_stores.chroma import ChromaVectorStore\n",
+    "\n",
+    "# # Create your index\n",
+    "# db2 = chromadb.PersistentClient(path=\"./ai-tutor-db\")\n",
+    "# chroma_collection = db2.get_or_create_collection(\"ai-tutor-db\")\n",
+    "# vector_store = ChromaVectorStore(chroma_collection=chroma_collection)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Create your index\n",
-    "from llama_index.core import VectorStoreIndex\n",
-    "index = VectorStoreIndex.from_vector_store(vector_store=vector_store)"
+    "# # Create your index\n",
+    "# from llama_index.core import VectorStoreIndex\n",
+    "\n",
+    "# index = VectorStoreIndex.from_vector_store(vector_store=vector_store)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
-    "from llama_index.llms.openai import OpenAI\n",
-    "from llama_index.core.vector_stores import (\n",
-    "    ExactMatchFilter,\n",
-    "    MetadataFilters,\n",
-    "    MetadataFilter,\n",
-    "    FilterOperator,\n",
-    "    FilterCondition,\n",
-    ")\n",
+    "# from llama_index.embeddings.openai import OpenAIEmbedding\n",
+    "# from llama_index.llms.openai import OpenAI\n",
+    "# from llama_index.core.vector_stores import (\n",
+    "#     ExactMatchFilter,\n",
+    "#     MetadataFilters,\n",
+    "#     MetadataFilter,\n",
+    "#     FilterOperator,\n",
+    "#     FilterCondition,\n",
+    "# )\n",
     "\n",
-    "filters = MetadataFilters(\n",
-    "    filters=[\n",
-    "        MetadataFilter(key=\"source\", value=\"lanchain_course\"),\n",
-    "        MetadataFilter(key=\"source\", value=\"langchain_docs\"),\n",
-    "    ],\n",
-    "    condition=FilterCondition.OR,\n",
-    ")\n",
+    "# filters = MetadataFilters(\n",
+    "#     filters=[\n",
+    "#         MetadataFilter(key=\"source\", value=\"lanchain_course\"),\n",
+    "#         MetadataFilter(key=\"source\", value=\"langchain_docs\"),\n",
+    "#     ],\n",
+    "#     condition=FilterCondition.OR,\n",
+    "# )\n",
     "\n",
-    "llm = OpenAI(temperature=0, model=\"gpt-3.5-turbo\", max_tokens=None)\n",
-    "embeds = OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"text_search\")\n",
+    "# llm = OpenAI(temperature=0, model=\"gpt-3.5-turbo\", max_tokens=None)\n",
+    "# embeds = OpenAIEmbedding(model=\"text-embedding-3-large\", mode=\"text_search\")\n",
+    "# # query_engine = index.as_query_engine(\n",
+    "# #     llm=llm, similarity_top_k=5, embed_model=embeds, verbose=True, streaming=True, filters=filters\n",
+    "# # )\n",
     "# query_engine = index.as_query_engine(\n",
-    "#     llm=llm, similarity_top_k=5, embed_model=embeds, verbose=True, streaming=True, filters=filters\n",
-    "# )\n",
-    "query_engine = index.as_query_engine(\n",
-    "    llm=llm, similarity_top_k=5, embed_model=embeds, verbose=True,\n",
-    ")"
+    "#     llm=llm,\n",
+    "#     similarity_top_k=5,\n",
+    "#     embed_model=embeds,\n",
+    "#     verbose=True,\n",
+    "# )"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
-      "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
-     ]
-    }
-   ],
-   "source": [
-    "res = query_engine.query(\"What is the LLama model?\")\n",
-    "\n",
-    "# history = \"\"   \n",
-    "# for token in res.response_gen:\n",
-    "#     history += token\n",
-    "#     print(history)"
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# res = query_engine.query(\"What is the LLama model?\")\n",
+    "\n",
+    "# # history = \"\"\n",
+    "# # for token in res.response_gen:\n",
+    "# #     history += token\n",
+    "# #     print(history)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "'The LLama model is a family of large language models (LLMs) released by Meta AI, with different model sizes ranging from 7 billion to 65 billion parameters in the first version. The developers reported that the 13 billion parameter model outperformed larger models like GPT-3 and was competitive with state-of-the-art models like PaLM and Chinchilla. The model weights were released to the research community under a noncommercial license, but they were leaked to the public shortly after.'"
-      ]
-     },
-     "execution_count": 5,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
-    "res.response"
+    "# res.response"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Node ID\t 6f8c13d8-1458-444c-857b-c1dc11d7f134\n",
-      "Source\t wikipedia\n",
-      "Title\t LLaMA\n",
-      "Text\t LLaMA LLaMA (Large Language Model Meta AI) is a family of large language models (LLMs), released by Meta AI starting in February 2023. For the first version of LLaMa, four model sizes were trained: 7, 13, 33 and 65 billion parameters. LLaMA's developers reported that the 13B parameter model's performance on most NLP benchmarks exceeded that of the much larger GPT-3 (with 175B parameters) and that the largest model was competitive with state of the art models such as PaLM and Chinchilla. Whereas the most powerful LLMs have generally been accessible only through limited APIs (if at all), Meta released LLaMA's model weights to the research community under a noncommercial license. Within a week of LLaMA's release, its weights were leaked to the public on 4chan via BitTorrent. In July 2023, Meta released several models as Llama 2, using 7, 13 and 70 billion parameters.\n",
-      "Score\t 0.5133216393307418\n",
-      "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
-      "Node ID\t 7226ba39-fd90-4790-879f-9f6af7b4ac6d\n",
-      "Source\t towards_ai\n",
-      "Title\t Fine-Tuning a Llama-2 7B Model for Python Code Generation\n",
-      "Text\t New Llama-2 model In mid-July, Meta released its new family of pre-trained and finetuned models called Llama-2, with an open source and commercial character to facilitate its use and expansion. The base model was released with a chat version and sizes 7B, 13B, and 70B. Together with the models, the corresponding papers were published describing their characteristics and relevant points of the learning process, which provide very interesting information on the subject. For pre-training, 40% more tokens were used, reaching 2T, the context length was doubled and the grouped-query attention (GQA) technique was applied to speed up inference on the heavier 70B model. On the standard transformer architecture, RMSNorm normalization, SwiGLU activation, and rotatory positional embedding are used, the context length reaches 4096 tokens, and an Adam optimizer is applied with a cosine learning rate schedule, a weight decay of 0.1 and gradient clipping. \n",
-      "Score\t 0.49851718223076963\n",
-      "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
-      "Node ID\t a654d9c1-fa81-4ad8-b16a-06a49f145636\n",
-      "Source\t hf_transformers\n",
-      "Title\t Overview\n",
-      "Text\t The Open-Llama model was proposed in Open-Llama project by community developer s-JoL.\n",
-      "The model is mainly based on LLaMA with some modifications, incorporating memory-efficient attention from Xformers, stable embedding from Bloom, and shared input-output embedding from PaLM.\n",
-      "And the model is pre-trained on both Chinese and English, which gives it better performance on Chinese language tasks.\n",
-      "This model was contributed by s-JoL.\n",
-      "The original code can be found Open-Llama.\n",
-      "Checkpoint and usage can be found at s-JoL/Open-Llama-V1.\n",
-      "Score\t 0.4888355341806359\n",
-      "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
-      "Node ID\t 846101f6-d9c5-4356-b569-c4383fe9b481\n",
-      "Source\t towards_ai\n",
-      "Title\t Meta's Llama 2: Revolutionizing Open Source Language Models for Commercial Use\n",
-      "Text\t II. Llama 2 Model Flavors Llama 2 is available in four different model sizes: 7 billion, 13 billion, 34 billion, and 70 billion parameters. While 7B, 13B, and 70B have already been released, the 34B model is still awaited. The pretrained variant, trained on a whopping 2 trillion tokens, boasts a context window of 4096 tokens, twice the size of its predecessor Llama 1. Meta also released a Llama 2 fine-tuned model for chat applications that was trained on over 1 million human annotations. Such extensive training comes at a cost, with the 70B model taking a staggering 1720320 GPU hours to train. The context window's length determines the amount of content the model can process at once, making Llama 2 a powerful language model in terms of scale and efficiency. \n",
-      "Score\t 0.4818213806198265\n",
-      "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n",
-      "Node ID\t bfffeff1-76d5-4ad4-9988-38be59ad9562\n",
-      "Source\t hf_transformers\n",
-      "Title\t Overview\n",
-      "Text\t d 34B parameters each. All models are trained on sequences of 16k tokens and show improvements on inputs with up to 100k tokens. 7B and 13B Code Llama and Code Llama - Instruct variants support infilling based on surrounding content. Code Llama reaches state-of-the-art performance among open models on several code benchmarks, with scores of up to 53% and 55% on HumanEval and MBPP, respectively. Notably, Code Llama - Python 7B outperforms Llama 2 70B on HumanEval and MBPP, and all our models outperform every other publicly available model on MultiPL-E. We release Code Llama under a permissive license that allows for both research and commercial use.\n",
-      "Check out all Code Llama models here and the officially released ones in the codellama org.\n",
-      "The Llama2 family models, on which Code Llama is based, were trained using bfloat16, but the original inference uses float16. Let’s look at the different precisions: float32: PyTorch convention on model initialization is to load models in float32, no\n",
-      "Score\t 0.4799444818466049\n",
-      "-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "for src in res.source_nodes:\n",
-    "  print(\"Node ID\\t\", src.node_id)\n",
-    "  print(\"Source\\t\", src.metadata['source'])\n",
-    "  print(\"Title\\t\", src.metadata['title'])\n",
-    "  print(\"Text\\t\", src.text)\n",
-    "  print(\"Score\\t\", src.score)\n",
-    "  print(\"-_\"*20)"
+    "# for src in res.source_nodes:\n",
+    "#     print(\"Node ID\\t\", src.node_id)\n",
+    "#     print(\"Source\\t\", src.metadata[\"source\"])\n",
+    "#     print(\"Title\\t\", src.metadata[\"title\"])\n",
+    "#     print(\"Text\\t\", src.text)\n",
+    "#     print(\"Score\\t\", src.score)\n",
+    "#     print(\"-_\" * 20)"
    ]
   },
   {
@@ -384,14 +756,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from IPython.display import Markdown, display\n",
-    "# define prompt viewing function\n",
-    "def display_prompt_dict(prompts_dict):\n",
-    "    for k, p in prompts_dict.items():\n",
-    "        text_md = f\"**Prompt Key**: {k}<br>\" f\"**Text:** <br>\"\n",
-    "        display(Markdown(text_md))\n",
-    "        print(p.get_template())\n",
-    "        display(Markdown(\"<br><br>\"))"
+    "# from IPython.display import Markdown, display\n",
+    "\n",
+    "\n",
+    "# # define prompt viewing function\n",
+    "# def display_prompt_dict(prompts_dict):\n",
+    "#     for k, p in prompts_dict.items():\n",
+    "#         text_md = f\"**Prompt Key**: {k}<br>\" f\"**Text:** <br>\"\n",
+    "#         display(Markdown(text_md))\n",
+    "#         print(p.get_template())\n",
+    "#         display(Markdown(\"<br><br>\"))"
    ]
   },
   {
@@ -400,7 +774,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "prompts_dict = query_engine.get_prompts()"
+    "# prompts_dict = query_engine.get_prompts()"
    ]
   },
   {
@@ -409,7 +783,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "display_prompt_dict(prompts_dict)"
+    "# display_prompt_dict(prompts_dict)"
    ]
   },
   {
@@ -450,7 +824,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.8"
+   "version": "3.12.3"
   }
  },
  "nbformat": 4,