Upload 7 files

faiss_store_openai.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50fc100d2bd725fcc1b8855388f7050c2b3011f0651f84adeb4beabc11e6e7f3
+size 408881

faiss_tutorial.ipynb

@@ -0,0 +1,574 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c9493399",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "#Install Packages\n",
+    "!pip install faiss-cpu\n",
+    "!pip install sentence-transformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "c49be142",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# import necessary libraries\n",
+    "import pandas as pd\n",
+    "pd.set_option('display.max_colwidth', 100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "f5a30989",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(8, 2)"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = pd.read_csv(\"sample_text.csv\")\n",
+    "df.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "id": "b72e2ecb",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "      <th>category</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Meditation and yoga can improve mental health</td>\n",
+       "      <td>Health</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>Fruits, whole grains and vegetables helps control blood pressure</td>\n",
+       "      <td>Health</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>These are the latest fashion trends for this week</td>\n",
+       "      <td>Fashion</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Vibrant color jeans for male are becoming a trend</td>\n",
+       "      <td>Fashion</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>The concert starts at 7 PM tonight</td>\n",
+       "      <td>Event</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>Navaratri dandiya program at Expo center in Mumbai this october</td>\n",
+       "      <td>Event</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>Exciting vacation destinations for your next trip</td>\n",
+       "      <td>Travel</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>Maldives and Srilanka are gaining popularity in terms of low budget vacation places</td>\n",
+       "      <td>Travel</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                                                  text  \\\n",
+       "0                                        Meditation and yoga can improve mental health   \n",
+       "1                     Fruits, whole grains and vegetables helps control blood pressure   \n",
+       "2                                    These are the latest fashion trends for this week   \n",
+       "3                                    Vibrant color jeans for male are becoming a trend   \n",
+       "4                                                   The concert starts at 7 PM tonight   \n",
+       "5                      Navaratri dandiya program at Expo center in Mumbai this october   \n",
+       "6                                    Exciting vacation destinations for your next trip   \n",
+       "7  Maldives and Srilanka are gaining popularity in terms of low budget vacation places   \n",
+       "\n",
+       "  category  \n",
+       "0   Health  \n",
+       "1   Health  \n",
+       "2  Fashion  \n",
+       "3  Fashion  \n",
+       "4    Event  \n",
+       "5    Event  \n",
+       "6   Travel  \n",
+       "7   Travel  "
+      ]
+     },
+     "execution_count": 49,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2d935944",
+   "metadata": {},
+   "source": [
+    "### Step 1 : Create source embeddings for the text column"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "cd04834b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\dhava\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sentence_transformers import SentenceTransformer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "03ed4874",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "encoder = SentenceTransformer(\"all-mpnet-base-v2\")\n",
+    "vectors = encoder.encode(df.text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "b8b8c1ce",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(8, 768)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "vectors.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "8e5c7da8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "768"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dim = vectors.shape[1]\n",
+    "dim"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "149e6b32",
+   "metadata": {},
+   "source": [
+    "### Step 2 : Build a FAISS Index for vectors"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "1033b6bd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import faiss\n",
+    "\n",
+    "index = faiss.IndexFlatL2(dim)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "76ad509d",
+   "metadata": {},
+   "source": [
+    "### Step 3 : Normalize the source vectors (as we are using L2 distance to measure similarity) and add to the index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "90b527fc",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "index.add(vectors)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "7ac0b8ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<faiss.swigfaiss.IndexFlatL2; proxy of <Swig Object of type 'faiss::IndexFlatL2 *' at 0x00000252CF6123D0> >"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "index"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6c42234c",
+   "metadata": {},
+   "source": [
+    "### Step 4 : Encode search text using same encorder and normalize the output vector"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 64,
+   "id": "018faf33",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(768,)"
+      ]
+     },
+     "execution_count": 64,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "search_query = \"I want to buy a polo t-shirt\"\n",
+    "# search_query = \"looking for places to visit during the holidays\"\n",
+    "# search_query = \"An apple a day keeps the doctor away\"\n",
+    "vec = encoder.encode(search_query)\n",
+    "vec.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "id": "af05bce3",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(1, 768)"
+      ]
+     },
+     "execution_count": 66,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "svec = np.array(vec).reshape(1,-1)\n",
+    "svec.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 67,
+   "id": "84275adf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# faiss.normalize_L2(svec)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "90c0cdd8",
+   "metadata": {},
+   "source": [
+    "### Step 5: Search for similar vector in the FAISS index created"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 68,
+   "id": "3d5a0e69",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[1.3844836, 1.4039096]], dtype=float32)"
+      ]
+     },
+     "execution_count": 68,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "distances, I = index.search(new_vec, k=2)\n",
+    "distances"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 69,
+   "id": "7ef978ca",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "array([[3, 2]], dtype=int64)"
+      ]
+     },
+     "execution_count": 69,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "I"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 70,
+   "id": "e2fceefd",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[[3, 2]]"
+      ]
+     },
+     "execution_count": 70,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "I.tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "id": "68f88083",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[3, 2]"
+      ]
+     },
+     "execution_count": 71,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "row_indices = I.tolist()[0]\n",
+    "row_indices"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 72,
+   "id": "d856895d",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>text</th>\n",
+       "      <th>category</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>Vibrant color jeans for male are becoming a trend</td>\n",
+       "      <td>Fashion</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>These are the latest fashion trends for this week</td>\n",
+       "      <td>Fashion</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                text category\n",
+       "3  Vibrant color jeans for male are becoming a trend  Fashion\n",
+       "2  These are the latest fashion trends for this week  Fashion"
+      ]
+     },
+     "execution_count": 72,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.loc[row_indices]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 73,
+   "id": "b65050a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'I want to buy a polo t-shirt'"
+      ]
+     },
+     "execution_count": 73,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "search_query"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e066c78d",
+   "metadata": {},
+   "source": [
+    "You can see that the two results from the dataframe are similar to a search_query"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

nvda_news_1.txt

@@ -0,0 +1,48 @@
+The stock of NVIDIA Corp (NASDAQ:NVDA) experienced a daily loss of -3.56% and a 3-month gain of 32.35%. With an Earnings Per Share (EPS) (EPS) of $1.92, the question arises: is the stock significantly overvalued? This article aims to provide a detailed valuation analysis of NVIDIA, offering insights into its financial strength, profitability, growth, and more. We invite you to delve into this comprehensive analysis.
+
+Company Overview
+Warning! GuruFocus has detected 10 Warning Signs with NVDA. Click here to check it out.
+
+NVDA 30-Year Financial Data
+
+The intrinsic value of NVDA
+
+
+NVIDIA Corp (NASDAQ:NVDA) is a leading designer of discrete graphics processing units that enhance the experience on computing platforms. The firm's chips are widely used in various end markets, including PC gaming and data centers. In recent years, NVIDIA has broadened its focus from traditional PC graphics applications such as gaming to more complex and favorable opportunities, including artificial intelligence and autonomous driving, leveraging the high-performance capabilities of its products.
+
+Currently, NVIDIA's stock price stands at $418.01, significantly higher than the GF Value of $310.28, indicating the stock might be overvalued. With a market cap of $1 trillion, the valuation seems steep. The following analysis aims to delve deeper into the company's value.
+
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Understanding the GF Value
+The GF Value is a unique measure of the intrinsic value of a stock, calculated based on historical trading multiples, a GuruFocus adjustment factor, and future business performance estimates. If the stock price is significantly above the GF Value Line, it is overvalued, and its future return is likely to be poor. Conversely, if it is significantly below the GF Value Line, its future return will likely be higher.
+
+According to GuruFocus Value calculation, NVIDIA (NASDAQ:NVDA) appears to be significantly overvalued. The stock's current price of $418.01 per share and the market cap of $1 trillion further strengthen this assumption.
+
+Given that NVIDIA is significantly overvalued, the long-term return of its stock is likely to be much lower than its future business growth.
+
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Link: These companies may deliver higher future returns at reduced risk.
+
+Financial Strength of NVIDIA
+Examining the financial strength of a company is crucial before investing in its stock. Companies with poor financial strength pose a higher risk of permanent loss. NVIDIA's cash-to-debt ratio of 1.27 is worse than 58.04% of companies in the Semiconductors industry. However, NVIDIA's overall financial strength is 8 out of 10, indicating a strong financial position.
+
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Profitability and Growth
+Consistent profitability over the long term reduces the risk for investors. NVIDIA, with its profitability ranking of 10 out of 10, has been profitable for the past 10 years. The company's operating margin of 17.37% ranks better than 76.5% of companies in the Semiconductors industry.
+
+However, growth is a crucial factor in a company's valuation. NVIDIA's growth ranks worse than 52.99% of companies in the Semiconductors industry, with its 3-year average revenue growth rate better than 87.88% of companies in the industry.
+
+ROIC vs WACC
+Comparing a company's return on invested capital (ROIC) to its weighted average cost of capital (WACC) is an effective way to evaluate its profitability. Over the past 12 months, NVIDIA's ROIC was 20.32 while its WACC was 16.74, suggesting that the company is creating value for its shareholders.
+
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Is NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis
+Conclusion
+In conclusion, NVIDIA (NASDAQ:NVDA) appears to be significantly overvalued. Despite its strong financial condition and profitability, its growth ranks lower than 52.99% of companies in the Semiconductors industry. To learn more about NVIDIA stock, you can check out its 30-Year Financials here.
+
+To find out the high quality companies that may deliver above-average returns, please check out GuruFocus High Quality Low Capex Screener.
+
+This article first appeared on GuruFocus.

requirements.txt

@@ -0,0 +1,10 @@
+langchain==0.0.284
+python-dotenv==1.0.0
+streamlit==1.22.0
+unstructured==0.9.2
+tiktoken==0.4.0
+faiss-cpu==1.7.4
+libmagic==1.0
+python-magic==0.4.27
+python-magic-bin==0.4.14
+OpenAI == 0.28.0

retrieval.ipynb

@@ -0,0 +1,489 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "0c2aff87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import streamlit as st\n",
+    "import pickle\n",
+    "import time\n",
+    "import langchain\n",
+    "from langchain import OpenAI\n",
+    "from langchain.chains import RetrievalQAWithSourcesChain\n",
+    "from langchain.chains.qa_with_sources.loading import load_qa_with_sources_chain\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "from langchain.document_loaders import UnstructuredURLLoader\n",
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "from langchain.vectorstores import FAISS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "80fc5e57",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#load openAI api key\n",
+    "os.environ['OPENAI_API_KEY'] = 'sk-proj-YXQCFNWqx1cRLsNczXtST3BlbkFJvG1UVBNpEJTUvgb6zSrV'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "39e721c4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Initialise LLM with required params\n",
+    "llm = OpenAI(temperature=0.9, max_tokens=500) "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bd0c3ff7",
+   "metadata": {},
+   "source": [
+    "### (1) Load data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "55fa0ef5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "2"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "loaders = UnstructuredURLLoader(urls=[\n",
+    "    \"https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.html\",\n",
+    "    \"https://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\"\n",
+    "])\n",
+    "data = loaders.load() \n",
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9f51a5bd",
+   "metadata": {},
+   "source": [
+    "### (2) Split data to create chunks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "054a6361",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "text_splitter = RecursiveCharacterTextSplitter(\n",
+    "    chunk_size=1000,\n",
+    "    chunk_overlap=200\n",
+    ")\n",
+    "\n",
+    "# As data is of type documents we can directly use split_documents over split_text in order to get the chunks.\n",
+    "docs = text_splitter.split_documents(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "379e3d94",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "41"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "637ee7ae",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Document(page_content='English\\n\\nHindi\\n\\nGujarati\\n\\nSpecials\\n\\nTrending Stocks\\n\\nIRFC\\xa0INE053F01010, IRFC, 543257\\n\\nTata Power\\xa0INE245A01021, TATAPOWER, 500400\\n\\nRail Vikas\\xa0INE415G01027, RVNL, 542649\\n\\nJio Financial\\xa0INE758E01017, JIOFIN, 543940\\n\\nSuzlon Energy\\xa0INE040H01021, SUZLON, 532667\\n\\nCheck your Credit Score here!\\n\\nQuotes\\n\\nMutual Funds\\n\\nCommodities\\n\\nFutures & Options\\n\\nCurrency\\n\\nNews\\n\\nCryptocurrency\\n\\nForum\\n\\nNotices\\n\\nVideos\\n\\nGlossary\\n\\nAll\\n\\nHello, LoginHello, LoginLog-inor Sign-UpMy AccountMy Profile My PortfolioMy WatchlistMy Credit ScoreMy MessagesMy AlertsMy Profile My PROMy PortfolioMy WatchlistMy Credit ScoreMy MessagesMy AlertsLogoutChat with UsDownload AppFollow us on:\\n\\nUpgrade', metadata={'source': 'https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.html'})"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "docs[0]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9e35a876",
+   "metadata": {},
+   "source": [
+    "### (3) Create embeddings for these chunks and save them to FAISS index"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "c3d0a6dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Create the embeddings of the chunks using openAIEmbeddings\n",
+    "embeddings = OpenAIEmbeddings()\n",
+    "\n",
+    "# Pass the documents and embeddings inorder to create FAISS vector index\n",
+    "vectorindex_openai = FAISS.from_documents(docs, embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "a9686c13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Storing vector index create in local\n",
+    "file_path=\"vector_index.pkl\"\n",
+    "with open(file_path, \"wb\") as f:\n",
+    "    pickle.dump(vectorindex_openai, f)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "688dc29b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if os.path.exists(file_path):\n",
+    "    with open(file_path, \"rb\") as f:\n",
+    "        vectorIndex = pickle.load(f)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fbd96296",
+   "metadata": {},
+   "source": [
+    "### (4) Retrieve similar embeddings for a given question and call LLM to retrieve final answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "01f5e1e8",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "RetrievalQAWithSourcesChain(memory=None, callbacks=None, callback_manager=None, verbose=False, tags=None, metadata=None, combine_documents_chain=MapReduceDocumentsChain(memory=None, callbacks=None, callback_manager=None, verbose=False, tags=None, metadata=None, input_key='input_documents', output_key='output_text', llm_chain=LLMChain(memory=None, callbacks=None, callback_manager=None, verbose=False, tags=None, metadata=None, prompt=PromptTemplate(input_variables=['context', 'question'], output_parser=None, partial_variables={}, template='Use the following portion of a long document to see if any of the text is relevant to answer the question. \\nReturn any relevant text verbatim.\\n{context}\\nQuestion: {question}\\nRelevant text, if any:', template_format='f-string', validate_template=True), llm=OpenAI(cache=None, verbose=False, callbacks=None, callback_manager=None, tags=None, metadata=None, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.9, max_tokens=500, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key='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', openai_api_base='', openai_organization='', openai_proxy='', batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False, allowed_special=set(), disallowed_special='all', tiktoken_model_name=None), output_key='text', output_parser=StrOutputParser(), return_final_only=True, llm_kwargs={}), reduce_documents_chain=ReduceDocumentsChain(memory=None, callbacks=None, callback_manager=None, verbose=False, tags=None, metadata=None, input_key='input_documents', output_key='output_text', combine_documents_chain=StuffDocumentsChain(memory=None, callbacks=None, callback_manager=None, verbose=False, tags=None, metadata=None, input_key='input_documents', output_key='output_text', llm_chain=LLMChain(memory=None, callbacks=None, callback_manager=None, verbose=False, tags=None, metadata=None, prompt=PromptTemplate(input_variables=['summaries', 'question'], output_parser=None, partial_variables={}, template='Given the following extracted parts of a long document and a question, create a final answer with references (\"SOURCES\"). \\nIf you don\\'t know the answer, just say that you don\\'t know. Don\\'t try to make up an answer.\\nALWAYS return a \"SOURCES\" part in your answer.\\n\\nQUESTION: Which state/country\\'s law governs the interpretation of the contract?\\n=========\\nContent: This Agreement is governed by English law and the parties submit to the exclusive jurisdiction of the English courts in  relation to any dispute (contractual or non-contractual) concerning this Agreement save that either party may apply to any court for an  injunction or other relief to protect its Intellectual Property Rights.\\nSource: 28-pl\\nContent: No Waiver. Failure or delay in exercising any right or remedy under this Agreement shall not constitute a waiver of such (or any other)  right or remedy.\\n\\n11.7 Severability. The invalidity, illegality or unenforceability of any term (or part of a term) of this Agreement shall not affect the continuation  in force of the remainder of the term (if any) and this Agreement.\\n\\n11.8 No Agency. Except as expressly stated otherwise, nothing in this Agreement shall create an agency, partnership or joint venture of any  kind between the parties.\\n\\n11.9 No Third-Party Beneficiaries.\\nSource: 30-pl\\nContent: (b) if Google believes, in good faith, that the Distributor has violated or caused Google to violate any Anti-Bribery Laws (as  defined in Clause 8.5) or that such a violation is reasonably likely to occur,\\nSource: 4-pl\\n=========\\nFINAL ANSWER: This Agreement is governed by English law.\\nSOURCES: 28-pl\\n\\nQUESTION: What did the president say about Michael Jackson?\\n=========\\nContent: Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \\n\\nLast year COVID-19 kept us apart. This year we are finally together again. \\n\\nTonight, we meet as Democrats Republicans and Independents. But most importantly as Americans. \\n\\nWith a duty to one another to the American people to the Constitution. \\n\\nAnd with an unwavering resolve that freedom will always triumph over tyranny. \\n\\nSix days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to his menacing ways. But he badly miscalculated. \\n\\nHe thought he could roll into Ukraine and the world would roll over. Instead he met a wall of strength he never imagined. \\n\\nHe met the Ukrainian people. \\n\\nFrom President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world. \\n\\nGroups of citizens blocking tanks with their bodies. Everyone from students to retirees teachers turned soldiers defending their homeland.\\nSource: 0-pl\\nContent: And we won’t stop. \\n\\nWe have lost so much to COVID-19. Time with one another. And worst of all, so much loss of life. \\n\\nLet’s use this moment to reset. Let’s stop looking at COVID-19 as a partisan dividing line and see it for what it is: A God-awful disease.  \\n\\nLet’s stop seeing each other as enemies, and start seeing each other for who we really are: Fellow Americans.  \\n\\nWe can’t change how divided we’ve been. But we can change how we move forward—on COVID-19 and other issues we must face together. \\n\\nI recently visited the New York City Police Department days after the funerals of Officer Wilbert Mora and his partner, Officer Jason Rivera. \\n\\nThey were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \\n\\nOfficer Mora was 27 years old. \\n\\nOfficer Rivera was 22. \\n\\nBoth Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \\n\\nI spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves.\\nSource: 24-pl\\nContent: And a proud Ukrainian people, who have known 30 years  of independence, have repeatedly shown that they will not tolerate anyone who tries to take their country backwards.  \\n\\nTo all Americans, I will be honest with you, as I’ve always promised. A Russian dictator, invading a foreign country, has costs around the world. \\n\\nAnd I’m taking robust action to make sure the pain of our sanctions  is targeted at Russia’s economy. And I will use every tool at our disposal to protect American businesses and consumers. \\n\\nTonight, I can announce that the United States has worked with 30 other countries to release 60 Million barrels of oil from reserves around the world.  \\n\\nAmerica will lead that effort, releasing 30 Million barrels from our own Strategic Petroleum Reserve. And we stand ready to do more if necessary, unified with our allies.  \\n\\nThese steps will help blunt gas prices here at home. And I know the news about what’s happening can seem alarming. \\n\\nBut I want you to know that we are going to be okay.\\nSource: 5-pl\\nContent: More support for patients and families. \\n\\nTo get there, I call on Congress to fund ARPA-H, the Advanced Research Projects Agency for Health. \\n\\nIt’s based on DARPA—the Defense Department project that led to the Internet, GPS, and so much more.  \\n\\nARPA-H will have a singular purpose—to drive breakthroughs in cancer, Alzheimer’s, diabetes, and more. \\n\\nA unity agenda for the nation. \\n\\nWe can do this. \\n\\nMy fellow Americans—tonight , we have gathered in a sacred space—the citadel of our democracy. \\n\\nIn this Capitol, generation after generation, Americans have debated great questions amid great strife, and have done great things. \\n\\nWe have fought for freedom, expanded liberty, defeated totalitarianism and terror. \\n\\nAnd built the strongest, freest, and most prosperous nation the world has ever known. \\n\\nNow is the hour. \\n\\nOur moment of responsibility. \\n\\nOur test of resolve and conscience, of history itself. \\n\\nIt is in this moment that our character is formed. Our purpose is found. Our future is forged. \\n\\nWell I know this nation.\\nSource: 34-pl\\n=========\\nFINAL ANSWER: The president did not mention Michael Jackson.\\nSOURCES:\\n\\nQUESTION: {question}\\n=========\\n{summaries}\\n=========\\nFINAL ANSWER:', template_format='f-string', validate_template=True), llm=OpenAI(cache=None, verbose=False, callbacks=None, callback_manager=None, tags=None, metadata=None, client=<class 'openai.api_resources.completion.Completion'>, model_name='text-davinci-003', temperature=0.9, max_tokens=500, top_p=1, frequency_penalty=0, presence_penalty=0, n=1, best_of=1, model_kwargs={}, openai_api_key='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', openai_api_base='', openai_organization='', openai_proxy='', batch_size=20, request_timeout=None, logit_bias={}, max_retries=6, streaming=False, allowed_special=set(), disallowed_special='all', tiktoken_model_name=None), output_key='text', output_parser=StrOutputParser(), return_final_only=True, llm_kwargs={}), document_prompt=PromptTemplate(input_variables=['page_content', 'source'], output_parser=None, partial_variables={}, template='Content: {page_content}\\nSource: {source}', template_format='f-string', validate_template=True), document_variable_name='summaries', document_separator='\\n\\n'), collapse_documents_chain=None, token_max=3000), document_variable_name='context', return_intermediate_steps=False), question_key='question', input_docs_key='docs', answer_key='answer', sources_answer_key='sources', return_source_documents=False, retriever=VectorStoreRetriever(tags=['FAISS'], metadata=None, vectorstore=<langchain.vectorstores.faiss.FAISS object at 0x000001E36D3E27D0>, search_type='similarity', search_kwargs={}), reduce_k_below_max_tokens=False, max_tokens_limit=3375)"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chain = RetrievalQAWithSourcesChain.from_llm(llm=llm, retriever=vectorIndex.as_retriever())\n",
+    "chain"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "8c2e228b",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain] Entering Chain run with input:\n",
+      "\u001b[0m{\n",
+      "  \"question\": \"what is the price of Tiago iCNG?\"\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain] Entering Chain run with input:\n",
+      "\u001b[0m[inputs]\n",
+      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain] Entering Chain run with input:\n",
+      "\u001b[0m{\n",
+      "  \"input_list\": [\n",
+      "    {\n",
+      "      \"context\": \"The company also said it has also introduced the twin-cylinder technology on its Tiago and Tigor models.\\n\\nThe Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh, while the Tigor iCNG comes at a price range of Rs 7.8 lakh to Rs 8.95 lakh.\\n\\nTata Motors Passenger Vehicles Ltd Head-Marketing, Vinay Pant said these introductions put together will make the company's CNG line up \\\"appealing, holistic, and stronger than ever\\\".\\n\\nPTI\\n\\nTags:\\n\\n#Business\\n\\n#Companies\\n\\nfirst published: Aug 4, 2023 02:17 pm\\n\\nbusiness news,\\n\\nSensex, and\\n\\nNifty updates. Obtain\\n\\nPersonal Finance insights, tax queries, and expert opinions on\\n\\nMoneycontrol or download the\\n\\nMoneycontrol App to stay updated!\\n\\nForum\\n\\nFacebook\\n\\nTwitter\\n\\nInstagram\\n\\nLinkedin\\n\\nRSS\\n\\nPortfolio\\n\\nMarkets\\n\\nWatchlist\\n\\nLive TV Show\\n\\nCurrencies\\n\\nCredit Score\\n\\nCommodities\\n\\nFixed Income\\n\\nPersonal Finance\\n\\nMutual Fund\\n\\nPre-Market\\n\\nIPO\\n\\nGlobal Market\\n\\nBudget 2023\\n\\nGold Rate\\n\\nBSE Sensex\\n\\nForum\\n\\nMC 30\\n\\nNews\\n\\nBusiness\\n\\nMarkets\\n\\nStocks\\n\\nEconomy\",\n",
+      "      \"question\": \"what is the price of Tiago iCNG?\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"context\": \"Tata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\n\\nWatchlist\\n\\nPortfolio\\n\\nMessage\\n\\nSet Alert\\n\\nlive\\n\\nbselive\\n\\nnselive\\n\\nVolume \\n\\nTodays L/H \\n\\nMore\\n\\nTata Motors on Friday launched the CNG variant of its micro SUV Punch priced between Rs 7.1 lakh and Rs 9.68 lakh (ex-showroom, Delhi).\\n\\nThe Punch iCNG is equipped with the company's proprietary twin-cylinder technology with enhanced safety features like a micro-switch to keep the car switched off at the time of refuelling and thermal incident protection that cuts off CNG supply to the engine and releases gas into the atmosphere, Tata Motors said in a statement.\\n\\nIt is also equipped with other features such as voice assisted electric sunroof, automatic projector headlamps, LED DRLs, 16-inch diamond cut alloy wheels, 7-inch infotainment system by Harman that supports Android Auto and Apple Carplay connectivity, rain sensing wipers and height adjustable driver seat.\",\n",
+      "      \"question\": \"what is the price of Tiago iCNG?\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"context\": \"Be a PRO\\n\\nBusiness\\n\\nMarkets\\n\\nStocks\\n\\nEconomy\\n\\nCompanies\\n\\nTrends\\n\\nIPO\\n\\nOpinion\\n\\nEV Special\\n\\nVisa Expert:\\n\\nGet instant updates on the latest news in the immigration world right at your fingertips with Visa Expert. Click Here!\\n\\nyou are here:\\n\\nHome\\n\\nNews\\n\\nBusiness\\n\\nTata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\n\\nThe Punch iCNG is equipped with the company's proprietary twin-cylinder technology with enhanced safety features like a micro-switch to keep the car switched off at the time of refuelling and thermal incident protection that cuts off CNG supply to the engine and releases gas into the atmosphere, Tata Motors said in a statement.\\n\\nPTI\\n\\nAugust 04, 2023 / 02:17 PM IST\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nTata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\n\\nWatchlist\\n\\nPortfolio\\n\\nMessage\\n\\nSet Alert\\n\\nlive\\n\\nbselive\\n\\nnselive\\n\\nVolume \\n\\nTodays L/H \\n\\nMore\",\n",
+      "      \"question\": \"what is the price of Tiago iCNG?\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"context\": \"English\\n\\nHindi\\n\\nGujarati\\n\\nSpecials\\n\\nTrending Stocks\\n\\nIRFC INE053F01010, IRFC, 543257\\n\\nTata Power INE245A01021, TATAPOWER, 500400\\n\\nRail Vikas INE415G01027, RVNL, 542649\\n\\nJio Financial INE758E01017, JIOFIN, 543940\\n\\nSuzlon Energy INE040H01021, SUZLON, 532667\\n\\nCheck your Credit Score here!\\n\\nQuotes\\n\\nMutual Funds\\n\\nCommodities\\n\\nFutures & Options\\n\\nCurrency\\n\\nNews\\n\\nCryptocurrency\\n\\nForum\\n\\nNotices\\n\\nVideos\\n\\nGlossary\\n\\nAll\\n\\nHello, LoginHello, LoginLog-inor Sign-UpMy AccountMy Profile My PortfolioMy WatchlistMy Credit ScoreMy MessagesMy AlertsMy Profile My PROMy PortfolioMy WatchlistMy Credit ScoreMy MessagesMy AlertsLogoutChat with UsDownload AppFollow us on:\\n\\nUpgrade\",\n",
+      "      \"question\": \"what is the price of Tiago iCNG?\"\n",
+      "    }\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 5:llm:OpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"Use the following portion of a long document to see if any of the text is relevant to answer the question. \\nReturn any relevant text verbatim.\\nThe company also said it has also introduced the twin-cylinder technology on its Tiago and Tigor models.\\n\\nThe Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh, while the Tigor iCNG comes at a price range of Rs 7.8 lakh to Rs 8.95 lakh.\\n\\nTata Motors Passenger Vehicles Ltd Head-Marketing, Vinay Pant said these introductions put together will make the company's CNG line up \\\"appealing, holistic, and stronger than ever\\\".\\n\\nPTI\\n\\nTags:\\n\\n#Business\\n\\n#Companies\\n\\nfirst published: Aug 4, 2023 02:17 pm\\n\\nbusiness news,\\n\\nSensex, and\\n\\nNifty updates. Obtain\\n\\nPersonal Finance insights, tax queries, and expert opinions on\\n\\nMoneycontrol or download the\\n\\nMoneycontrol App to stay updated!\\n\\nForum\\n\\nFacebook\\n\\nTwitter\\n\\nInstagram\\n\\nLinkedin\\n\\nRSS\\n\\nPortfolio\\n\\nMarkets\\n\\nWatchlist\\n\\nLive TV Show\\n\\nCurrencies\\n\\nCredit Score\\n\\nCommodities\\n\\nFixed Income\\n\\nPersonal Finance\\n\\nMutual Fund\\n\\nPre-Market\\n\\nIPO\\n\\nGlobal Market\\n\\nBudget 2023\\n\\nGold Rate\\n\\nBSE Sensex\\n\\nForum\\n\\nMC 30\\n\\nNews\\n\\nBusiness\\n\\nMarkets\\n\\nStocks\\n\\nEconomy\\nQuestion: what is the price of Tiago iCNG?\\nRelevant text, if any:\"\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 6:llm:OpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"Use the following portion of a long document to see if any of the text is relevant to answer the question. \\nReturn any relevant text verbatim.\\nTata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\n\\nWatchlist\\n\\nPortfolio\\n\\nMessage\\n\\nSet Alert\\n\\nlive\\n\\nbselive\\n\\nnselive\\n\\nVolume \\n\\nTodays L/H \\n\\nMore\\n\\nTata Motors on Friday launched the CNG variant of its micro SUV Punch priced between Rs 7.1 lakh and Rs 9.68 lakh (ex-showroom, Delhi).\\n\\nThe Punch iCNG is equipped with the company's proprietary twin-cylinder technology with enhanced safety features like a micro-switch to keep the car switched off at the time of refuelling and thermal incident protection that cuts off CNG supply to the engine and releases gas into the atmosphere, Tata Motors said in a statement.\\n\\nIt is also equipped with other features such as voice assisted electric sunroof, automatic projector headlamps, LED DRLs, 16-inch diamond cut alloy wheels, 7-inch infotainment system by Harman that supports Android Auto and Apple Carplay connectivity, rain sensing wipers and height adjustable driver seat.\\nQuestion: what is the price of Tiago iCNG?\\nRelevant text, if any:\"\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 7:llm:OpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"Use the following portion of a long document to see if any of the text is relevant to answer the question. \\nReturn any relevant text verbatim.\\nBe a PRO\\n\\nBusiness\\n\\nMarkets\\n\\nStocks\\n\\nEconomy\\n\\nCompanies\\n\\nTrends\\n\\nIPO\\n\\nOpinion\\n\\nEV Special\\n\\nVisa Expert:\\n\\nGet instant updates on the latest news in the immigration world right at your fingertips with Visa Expert. Click Here!\\n\\nyou are here:\\n\\nHome\\n\\nNews\\n\\nBusiness\\n\\nTata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\n\\nThe Punch iCNG is equipped with the company's proprietary twin-cylinder technology with enhanced safety features like a micro-switch to keep the car switched off at the time of refuelling and thermal incident protection that cuts off CNG supply to the engine and releases gas into the atmosphere, Tata Motors said in a statement.\\n\\nPTI\\n\\nAugust 04, 2023 / 02:17 PM IST\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nTata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\n\\nWatchlist\\n\\nPortfolio\\n\\nMessage\\n\\nSet Alert\\n\\nlive\\n\\nbselive\\n\\nnselive\\n\\nVolume \\n\\nTodays L/H \\n\\nMore\\nQuestion: what is the price of Tiago iCNG?\\nRelevant text, if any:\"\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 8:llm:OpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"Use the following portion of a long document to see if any of the text is relevant to answer the question. \\nReturn any relevant text verbatim.\\nEnglish\\n\\nHindi\\n\\nGujarati\\n\\nSpecials\\n\\nTrending Stocks\\n\\nIRFC INE053F01010, IRFC, 543257\\n\\nTata Power INE245A01021, TATAPOWER, 500400\\n\\nRail Vikas INE415G01027, RVNL, 542649\\n\\nJio Financial INE758E01017, JIOFIN, 543940\\n\\nSuzlon Energy INE040H01021, SUZLON, 532667\\n\\nCheck your Credit Score here!\\n\\nQuotes\\n\\nMutual Funds\\n\\nCommodities\\n\\nFutures & Options\\n\\nCurrency\\n\\nNews\\n\\nCryptocurrency\\n\\nForum\\n\\nNotices\\n\\nVideos\\n\\nGlossary\\n\\nAll\\n\\nHello, LoginHello, LoginLog-inor Sign-UpMy AccountMy Profile My PortfolioMy WatchlistMy Credit ScoreMy MessagesMy AlertsMy Profile My PROMy PortfolioMy WatchlistMy Credit ScoreMy MessagesMy AlertsLogoutChat with UsDownload AppFollow us on:\\n\\nUpgrade\\nQuestion: what is the price of Tiago iCNG?\\nRelevant text, if any:\"\n",
+      "  ]\n",
+      "}\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 5:llm:OpenAI] [1.01s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \" The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"stop\",\n",
+      "          \"logprobs\": null\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {\n",
+      "      \"total_tokens\": 1343,\n",
+      "      \"prompt_tokens\": 1269,\n",
+      "      \"completion_tokens\": 74\n",
+      "    },\n",
+      "    \"model_name\": \"text-davinci-003\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 6:llm:OpenAI] [1.01s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \" Tata Motors on Friday launched the CNG variant of its micro SUV Punch priced between Rs 7.1 lakh and Rs 9.68 lakh (ex-showroom, Delhi).\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"stop\",\n",
+      "          \"logprobs\": null\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {},\n",
+      "    \"model_name\": \"text-davinci-003\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 7:llm:OpenAI] [1.01s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \"  Tata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"stop\",\n",
+      "          \"logprobs\": null\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {},\n",
+      "    \"model_name\": \"text-davinci-003\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain > 8:llm:OpenAI] [1.01s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \" None\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"stop\",\n",
+      "          \"logprobs\": null\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {},\n",
+      "    \"model_name\": \"text-davinci-003\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 4:chain:LLMChain] [1.02s] Exiting Chain run with output:\n",
+      "\u001b[0m{\n",
+      "  \"outputs\": [\n",
+      "    {\n",
+      "      \"text\": \" The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"text\": \" Tata Motors on Friday launched the CNG variant of its micro SUV Punch priced between Rs 7.1 lakh and Rs 9.68 lakh (ex-showroom, Delhi).\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"text\": \"  Tata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\"\n",
+      "    },\n",
+      "    {\n",
+      "      \"text\": \" None\"\n",
+      "    }\n",
+      "  ]\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 9:chain:LLMChain] Entering Chain run with input:\n",
+      "\u001b[0m{\n",
+      "  \"question\": \"what is the price of Tiago iCNG?\",\n",
+      "  \"summaries\": \"Content:  The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n\\nContent:  Tata Motors on Friday launched the CNG variant of its micro SUV Punch priced between Rs 7.1 lakh and Rs 9.68 lakh (ex-showroom, Delhi).\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n\\nContent:   Tata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n\\nContent:  None\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\"\n",
+      "}\n",
+      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 9:chain:LLMChain > 10:llm:OpenAI] Entering LLM run with input:\n",
+      "\u001b[0m{\n",
+      "  \"prompts\": [\n",
+      "    \"Given the following extracted parts of a long document and a question, create a final answer with references (\\\"SOURCES\\\"). \\nIf you don't know the answer, just say that you don't know. Don't try to make up an answer.\\nALWAYS return a \\\"SOURCES\\\" part in your answer.\\n\\nQUESTION: Which state/country's law governs the interpretation of the contract?\\n=========\\nContent: This Agreement is governed by English law and the parties submit to the exclusive jurisdiction of the English courts in  relation to any dispute (contractual or non-contractual) concerning this Agreement save that either party may apply to any court for an  injunction or other relief to protect its Intellectual Property Rights.\\nSource: 28-pl\\nContent: No Waiver. Failure or delay in exercising any right or remedy under this Agreement shall not constitute a waiver of such (or any other)  right or remedy.\\n\\n11.7 Severability. The invalidity, illegality or unenforceability of any term (or part of a term) of this Agreement shall not affect the continuation  in force of the remainder of the term (if any) and this Agreement.\\n\\n11.8 No Agency. Except as expressly stated otherwise, nothing in this Agreement shall create an agency, partnership or joint venture of any  kind between the parties.\\n\\n11.9 No Third-Party Beneficiaries.\\nSource: 30-pl\\nContent: (b) if Google believes, in good faith, that the Distributor has violated or caused Google to violate any Anti-Bribery Laws (as  defined in Clause 8.5) or that such a violation is reasonably likely to occur,\\nSource: 4-pl\\n=========\\nFINAL ANSWER: This Agreement is governed by English law.\\nSOURCES: 28-pl\\n\\nQUESTION: What did the president say about Michael Jackson?\\n=========\\nContent: Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \\n\\nLast year COVID-19 kept us apart. This year we are finally together again. \\n\\nTonight, we meet as Democrats Republicans and Independents. But most importantly as Americans. \\n\\nWith a duty to one another to the American people to the Constitution. \\n\\nAnd with an unwavering resolve that freedom will always triumph over tyranny. \\n\\nSix days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to his menacing ways. But he badly miscalculated. \\n\\nHe thought he could roll into Ukraine and the world would roll over. Instead he met a wall of strength he never imagined. \\n\\nHe met the Ukrainian people. \\n\\nFrom President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world. \\n\\nGroups of citizens blocking tanks with their bodies. Everyone from students to retirees teachers turned soldiers defending their homeland.\\nSource: 0-pl\\nContent: And we won’t stop. \\n\\nWe have lost so much to COVID-19. Time with one another. And worst of all, so much loss of life. \\n\\nLet’s use this moment to reset. Let’s stop looking at COVID-19 as a partisan dividing line and see it for what it is: A God-awful disease.  \\n\\nLet’s stop seeing each other as enemies, and start seeing each other for who we really are: Fellow Americans.  \\n\\nWe can’t change how divided we’ve been. But we can change how we move forward—on COVID-19 and other issues we must face together. \\n\\nI recently visited the New York City Police Department days after the funerals of Officer Wilbert Mora and his partner, Officer Jason Rivera. \\n\\nThey were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \\n\\nOfficer Mora was 27 years old. \\n\\nOfficer Rivera was 22. \\n\\nBoth Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \\n\\nI spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves.\\nSource: 24-pl\\nContent: And a proud Ukrainian people, who have known 30 years  of independence, have repeatedly shown that they will not tolerate anyone who tries to take their country backwards.  \\n\\nTo all Americans, I will be honest with you, as I’ve always promised. A Russian dictator, invading a foreign country, has costs around the world. \\n\\nAnd I’m taking robust action to make sure the pain of our sanctions  is targeted at Russia’s economy. And I will use every tool at our disposal to protect American businesses and consumers. \\n\\nTonight, I can announce that the United States has worked with 30 other countries to release 60 Million barrels of oil from reserves around the world.  \\n\\nAmerica will lead that effort, releasing 30 Million barrels from our own Strategic Petroleum Reserve. And we stand ready to do more if necessary, unified with our allies.  \\n\\nThese steps will help blunt gas prices here at home. And I know the news about what’s happening can seem alarming. \\n\\nBut I want you to know that we are going to be okay.\\nSource: 5-pl\\nContent: More support for patients and families. \\n\\nTo get there, I call on Congress to fund ARPA-H, the Advanced Research Projects Agency for Health. \\n\\nIt’s based on DARPA—the Defense Department project that led to the Internet, GPS, and so much more.  \\n\\nARPA-H will have a singular purpose—to drive breakthroughs in cancer, Alzheimer’s, diabetes, and more. \\n\\nA unity agenda for the nation. \\n\\nWe can do this. \\n\\nMy fellow Americans—tonight , we have gathered in a sacred space—the citadel of our democracy. \\n\\nIn this Capitol, generation after generation, Americans have debated great questions amid great strife, and have done great things. \\n\\nWe have fought for freedom, expanded liberty, defeated totalitarianism and terror. \\n\\nAnd built the strongest, freest, and most prosperous nation the world has ever known. \\n\\nNow is the hour. \\n\\nOur moment of responsibility. \\n\\nOur test of resolve and conscience, of history itself. \\n\\nIt is in this moment that our character is formed. Our purpose is found. Our future is forged. \\n\\nWell I know this nation.\\nSource: 34-pl\\n=========\\nFINAL ANSWER: The president did not mention Michael Jackson.\\nSOURCES:\\n\\nQUESTION: what is the price of Tiago iCNG?\\n=========\\nContent:  The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n\\nContent:  Tata Motors on Friday launched the CNG variant of its micro SUV Punch priced between Rs 7.1 lakh and Rs 9.68 lakh (ex-showroom, Delhi).\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n\\nContent:   Tata Motors launches Punch iCNG, price starts at Rs 7.1 lakh\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n\\nContent:  None\\nSource: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\\n=========\\nFINAL ANSWER:\"\n",
+      "  ]\n",
+      "}\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 9:chain:LLMChain > 10:llm:OpenAI] [2.88s] Exiting LLM run with output:\n",
+      "\u001b[0m{\n",
+      "  \"generations\": [\n",
+      "    [\n",
+      "      {\n",
+      "        \"text\": \" The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\nSOURCES: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\",\n",
+      "        \"generation_info\": {\n",
+      "          \"finish_reason\": \"stop\",\n",
+      "          \"logprobs\": null\n",
+      "        }\n",
+      "      }\n",
+      "    ]\n",
+      "  ],\n",
+      "  \"llm_output\": {\n",
+      "    \"token_usage\": {\n",
+      "      \"total_tokens\": 2093,\n",
+      "      \"prompt_tokens\": 1976,\n",
+      "      \"completion_tokens\": 117\n",
+      "    },\n",
+      "    \"model_name\": \"text-davinci-003\"\n",
+      "  },\n",
+      "  \"run\": null\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain > 9:chain:LLMChain] [2.88s] Exiting Chain run with output:\n",
+      "\u001b[0m{\n",
+      "  \"text\": \" The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\nSOURCES: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\"\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain > 3:chain:MapReduceDocumentsChain] [3.90s] Exiting Chain run with output:\n",
+      "\u001b[0m{\n",
+      "  \"output_text\": \" The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\nSOURCES: https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\"\n",
+      "}\n",
+      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[1:chain:RetrievalQAWithSourcesChain] [4.09s] Exiting Chain run with output:\n",
+      "\u001b[0m{\n",
+      "  \"answer\": \" The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\n\",\n",
+      "  \"sources\": \"https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html\"\n",
+      "}\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'answer': ' The Tiago iCNG is priced between Rs 6.55 lakh and Rs 8.1 lakh.\\n',\n",
+       " 'sources': 'https://www.moneycontrol.com/news/business/markets/wall-street-rises-as-tesla-soars-on-ai-optimism-11351111.htmlhttps://www.moneycontrol.com/news/business/tata-motors-launches-punch-icng-price-starts-at-rs-7-1-lakh-11098751.html'}"
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "query = \"what is the price of Tiago iCNG?\"\n",
+    "# query = \"what are the main features of punch iCNG?\"\n",
+    "\n",
+    "langchain.debug=True\n",
+    "\n",
+    "chain({\"question\": query}, return_only_outputs=True)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

text_loaders_splitters.ipynb

@@ -0,0 +1,891 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "ae222ed8",
+   "metadata": {},
+   "source": [
+    "## Document Loaders In LangChain"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e65dada0",
+   "metadata": {},
+   "source": [
+    "#### TextLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "470ed0a2",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content=\"The stock of NVIDIA Corp (NASDAQ:NVDA) experienced a daily loss of -3.56% and a 3-month gain of 32.35%. With an Earnings Per Share (EPS) (EPS) of $1.92, the question arises: is the stock significantly overvalued? This article aims to provide a detailed valuation analysis of NVIDIA, offering insights into its financial strength, profitability, growth, and more. We invite you to delve into this comprehensive analysis.\\n\\nCompany Overview\\nWarning! GuruFocus has detected 10 Warning Signs with NVDA. Click here to check it out.\\n\\nNVDA 30-Year Financial Data\\n\\nThe intrinsic value of NVDA\\n\\n\\nNVIDIA Corp (NASDAQ:NVDA) is a leading designer of discrete graphics processing units that enhance the experience on computing platforms. The firm's chips are widely used in various end markets, including PC gaming and data centers. In recent years, NVIDIA has broadened its focus from traditional PC graphics applications such as gaming to more complex and favorable opportunities, including artificial intelligence and autonomous driving, leveraging the high-performance capabilities of its products.\\n\\nCurrently, NVIDIA's stock price stands at $418.01, significantly higher than the GF Value of $310.28, indicating the stock might be overvalued. With a market cap of $1 trillion, the valuation seems steep. The following analysis aims to delve deeper into the company's value.\\n\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nUnderstanding the GF Value\\nThe GF Value is a unique measure of the intrinsic value of a stock, calculated based on historical trading multiples, a GuruFocus adjustment factor, and future business performance estimates. If the stock price is significantly above the GF Value Line, it is overvalued, and its future return is likely to be poor. Conversely, if it is significantly below the GF Value Line, its future return will likely be higher.\\n\\nAccording to GuruFocus Value calculation, NVIDIA (NASDAQ:NVDA) appears to be significantly overvalued. The stock's current price of $418.01 per share and the market cap of $1 trillion further strengthen this assumption.\\n\\nGiven that NVIDIA is significantly overvalued, the long-term return of its stock is likely to be much lower than its future business growth.\\n\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nLink: These companies may deliver higher future returns at reduced risk.\\n\\nFinancial Strength of NVIDIA\\nExamining the financial strength of a company is crucial before investing in its stock. Companies with poor financial strength pose a higher risk of permanent loss. NVIDIA's cash-to-debt ratio of 1.27 is worse than 58.04% of companies in the Semiconductors industry. However, NVIDIA's overall financial strength is 8 out of 10, indicating a strong financial position.\\n\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nProfitability and Growth\\nConsistent profitability over the long term reduces the risk for investors. NVIDIA, with its profitability ranking of 10 out of 10, has been profitable for the past 10 years. The company's operating margin of 17.37% ranks better than 76.5% of companies in the Semiconductors industry.\\n\\nHowever, growth is a crucial factor in a company's valuation. NVIDIA's growth ranks worse than 52.99% of companies in the Semiconductors industry, with its 3-year average revenue growth rate better than 87.88% of companies in the industry.\\n\\nROIC vs WACC\\nComparing a company's return on invested capital (ROIC) to its weighted average cost of capital (WACC) is an effective way to evaluate its profitability. Over the past 12 months, NVIDIA's ROIC was 20.32 while its WACC was 16.74, suggesting that the company is creating value for its shareholders.\\n\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nIs NVIDIA's Stock Significantly Overvalued? A Comprehensive Valuation Analysis\\nConclusion\\nIn conclusion, NVIDIA (NASDAQ:NVDA) appears to be significantly overvalued. Despite its strong financial condition and profitability, its growth ranks lower than 52.99% of companies in the Semiconductors industry. To learn more about NVIDIA stock, you can check out its 30-Year Financials here.\\n\\nTo find out the high quality companies that may deliver above-average returns, please check out GuruFocus High Quality Low Capex Screener.\\n\\nThis article first appeared on GuruFocus.\", metadata={'source': 'nvda_news_1.txt'})]"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain.document_loaders import TextLoader\n",
+    "\n",
+    "loader = TextLoader(\"nvda_news_1.txt\")\n",
+    "loader.load()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "ca5844b4",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "langchain.document_loaders.text.TextLoader"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "type(loader)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "bb95e130",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'nvda_news_1.txt'"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "loader.file_path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4da25ff7",
+   "metadata": {},
+   "source": [
+    "#### CSVLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "ad67ba6e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders.csv_loader import CSVLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "220b3fcb",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='movie_id: 101\\ntitle: K.G.F: Chapter 2\\nindustry: Bollywood\\nrelease_year: 2022\\nimdb_rating: 8.4\\nstudio: Hombale Films\\nlanguage_id: 3\\nbudget: 1\\nrevenue: 12.5\\nunit: Billions\\ncurrency: INR', metadata={'source': 'movies.csv', 'row': 0}),\n",
+       " Document(page_content='movie_id: 102\\ntitle: Doctor Strange in the Multiverse of Madness\\nindustry: Hollywood\\nrelease_year: 2022\\nimdb_rating: 7\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 200\\nrevenue: 954.8\\nunit: Millions\\ncurrency: USD', metadata={'source': 'movies.csv', 'row': 1}),\n",
+       " Document(page_content='movie_id: 103\\ntitle: Thor: The Dark World\\nindustry: Hollywood\\nrelease_year: 2013\\nimdb_rating: 6.8\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 165\\nrevenue: 644.8\\nunit: Millions\\ncurrency: USD', metadata={'source': 'movies.csv', 'row': 2}),\n",
+       " Document(page_content='movie_id: 104\\ntitle: Thor: Ragnarok\\nindustry: Hollywood\\nrelease_year: 2017\\nimdb_rating: 7.9\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 180\\nrevenue: 854\\nunit: Millions\\ncurrency: USD', metadata={'source': 'movies.csv', 'row': 3}),\n",
+       " Document(page_content='movie_id: 105\\ntitle: Thor: Love and Thunder\\nindustry: Hollywood\\nrelease_year: 2022\\nimdb_rating: 6.8\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 250\\nrevenue: 670\\nunit: Millions\\ncurrency: USD', metadata={'source': 'movies.csv', 'row': 4}),\n",
+       " Document(page_content='movie_id: 106\\ntitle: Sholay\\nindustry: Bollywood\\nrelease_year: 1975\\nimdb_rating: 8.1\\nstudio: United Producers\\nlanguage_id: 1\\nbudget: Not Available\\nrevenue: Not Available\\nunit: Not Available\\ncurrency: Not Available', metadata={'source': 'movies.csv', 'row': 5}),\n",
+       " Document(page_content='movie_id: 107\\ntitle: Dilwale Dulhania Le Jayenge\\nindustry: Bollywood\\nrelease_year: 1995\\nimdb_rating: 8\\nstudio: Yash Raj Films\\nlanguage_id: 1\\nbudget: 400\\nrevenue: 2000\\nunit: Millions\\ncurrency: INR', metadata={'source': 'movies.csv', 'row': 6}),\n",
+       " Document(page_content='movie_id: 108\\ntitle: 3 Idiots\\nindustry: Bollywood\\nrelease_year: 2009\\nimdb_rating: 8.4\\nstudio: Vinod Chopra Films\\nlanguage_id: 1\\nbudget: 550\\nrevenue: 4000\\nunit: Millions\\ncurrency: INR', metadata={'source': 'movies.csv', 'row': 7}),\n",
+       " Document(page_content='movie_id: 109\\ntitle: Kabhi Khushi Kabhie Gham\\nindustry: Bollywood\\nrelease_year: 2001\\nimdb_rating: 7.4\\nstudio: Dharma Productions\\nlanguage_id: 1\\nbudget: 390\\nrevenue: 1360\\nunit: Millions\\ncurrency: INR', metadata={'source': 'movies.csv', 'row': 8})]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "loader = CSVLoader(file_path=\"movies.csv\")\n",
+    "data = loader.load()\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "4e1b3b9a",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Document(page_content='movie_id: 101\\ntitle: K.G.F: Chapter 2\\nindustry: Bollywood\\nrelease_year: 2022\\nimdb_rating: 8.4\\nstudio: Hombale Films\\nlanguage_id: 3\\nbudget: 1\\nrevenue: 12.5\\nunit: Billions\\ncurrency: INR', metadata={'source': 'movies.csv', 'row': 0})"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "e356cd2b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[Document(page_content='movie_id: 101\\ntitle: K.G.F: Chapter 2\\nindustry: Bollywood\\nrelease_year: 2022\\nimdb_rating: 8.4\\nstudio: Hombale Films\\nlanguage_id: 3\\nbudget: 1\\nrevenue: 12.5\\nunit: Billions\\ncurrency: INR', metadata={'source': 'K.G.F: Chapter 2', 'row': 0}),\n",
+       " Document(page_content='movie_id: 102\\ntitle: Doctor Strange in the Multiverse of Madness\\nindustry: Hollywood\\nrelease_year: 2022\\nimdb_rating: 7\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 200\\nrevenue: 954.8\\nunit: Millions\\ncurrency: USD', metadata={'source': 'Doctor Strange in the Multiverse of Madness', 'row': 1}),\n",
+       " Document(page_content='movie_id: 103\\ntitle: Thor: The Dark World\\nindustry: Hollywood\\nrelease_year: 2013\\nimdb_rating: 6.8\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 165\\nrevenue: 644.8\\nunit: Millions\\ncurrency: USD', metadata={'source': 'Thor: The Dark World', 'row': 2}),\n",
+       " Document(page_content='movie_id: 104\\ntitle: Thor: Ragnarok\\nindustry: Hollywood\\nrelease_year: 2017\\nimdb_rating: 7.9\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 180\\nrevenue: 854\\nunit: Millions\\ncurrency: USD', metadata={'source': 'Thor: Ragnarok', 'row': 3}),\n",
+       " Document(page_content='movie_id: 105\\ntitle: Thor: Love and Thunder\\nindustry: Hollywood\\nrelease_year: 2022\\nimdb_rating: 6.8\\nstudio: Marvel Studios\\nlanguage_id: 5\\nbudget: 250\\nrevenue: 670\\nunit: Millions\\ncurrency: USD', metadata={'source': 'Thor: Love and Thunder', 'row': 4}),\n",
+       " Document(page_content='movie_id: 106\\ntitle: Sholay\\nindustry: Bollywood\\nrelease_year: 1975\\nimdb_rating: 8.1\\nstudio: United Producers\\nlanguage_id: 1\\nbudget: Not Available\\nrevenue: Not Available\\nunit: Not Available\\ncurrency: Not Available', metadata={'source': 'Sholay', 'row': 5}),\n",
+       " Document(page_content='movie_id: 107\\ntitle: Dilwale Dulhania Le Jayenge\\nindustry: Bollywood\\nrelease_year: 1995\\nimdb_rating: 8\\nstudio: Yash Raj Films\\nlanguage_id: 1\\nbudget: 400\\nrevenue: 2000\\nunit: Millions\\ncurrency: INR', metadata={'source': 'Dilwale Dulhania Le Jayenge', 'row': 6}),\n",
+       " Document(page_content='movie_id: 108\\ntitle: 3 Idiots\\nindustry: Bollywood\\nrelease_year: 2009\\nimdb_rating: 8.4\\nstudio: Vinod Chopra Films\\nlanguage_id: 1\\nbudget: 550\\nrevenue: 4000\\nunit: Millions\\ncurrency: INR', metadata={'source': '3 Idiots', 'row': 7}),\n",
+       " Document(page_content='movie_id: 109\\ntitle: Kabhi Khushi Kabhie Gham\\nindustry: Bollywood\\nrelease_year: 2001\\nimdb_rating: 7.4\\nstudio: Dharma Productions\\nlanguage_id: 1\\nbudget: 390\\nrevenue: 1360\\nunit: Millions\\ncurrency: INR', metadata={'source': 'Kabhi Khushi Kabhie Gham', 'row': 8})]"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "loader = CSVLoader(file_path=\"movies.csv\", source_column=\"title\")\n",
+    "data = loader.load()\n",
+    "data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "42d13fd6",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'movie_id: 101\\ntitle: K.G.F: Chapter 2\\nindustry: Bollywood\\nrelease_year: 2022\\nimdb_rating: 8.4\\nstudio: Hombale Films\\nlanguage_id: 3\\nbudget: 1\\nrevenue: 12.5\\nunit: Billions\\ncurrency: INR'"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].page_content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "0d8913df",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'source': 'K.G.F: Chapter 2', 'row': 0}"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].metadata"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f0163924",
+   "metadata": {},
+   "source": [
+    "#### UnstructuredURLLoader"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "48644959",
+   "metadata": {},
+   "source": [
+    "UnstructuredURLLoader of Langchain internally uses unstructured python library to load the content from url's\n",
+    "\n",
+    "https://unstructured-io.github.io/unstructured/introduction.html\n",
+    "\n",
+    "https://pypi.org/project/unstructured/#description"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "2e1fbf77",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: unstructured in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (0.9.2)\n",
+      "Requirement already satisfied: libmagic in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (1.0)\n",
+      "Requirement already satisfied: python-magic in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (0.4.27)\n",
+      "Requirement already satisfied: python-magic-bin in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (0.4.14)\n",
+      "Requirement already satisfied: tabulate in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from unstructured) (0.9.0)\n",
+      "Requirement already satisfied: requests in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from unstructured) (2.31.0)\n",
+      "Requirement already satisfied: nltk in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from unstructured) (3.8.1)\n",
+      "Requirement already satisfied: chardet in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from unstructured) (5.2.0)\n",
+      "Requirement already satisfied: filetype in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from unstructured) (1.2.0)\n",
+      "Requirement already satisfied: lxml in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from unstructured) (4.9.3)\n",
+      "Requirement already satisfied: click in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->unstructured) (8.1.3)\n",
+      "Requirement already satisfied: tqdm in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->unstructured) (4.65.0)\n",
+      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->unstructured) (2023.8.8)\n",
+      "Requirement already satisfied: joblib in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from nltk->unstructured) (1.3.2)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->unstructured) (2.0.3)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->unstructured) (2023.5.7)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->unstructured) (3.4)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from requests->unstructured) (3.1.0)\n",
+      "Requirement already satisfied: colorama in c:\\users\\dhava\\appdata\\local\\programs\\python\\python310\\lib\\site-packages (from click->nltk->unstructured) (0.4.6)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "[notice] A new release of pip is available: 23.0.1 -> 23.2.1\n",
+      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+     ]
+    }
+   ],
+   "source": [
+    "#installing necessary libraries, libmagic is used for file type detection\n",
+    "!pip3 install unstructured libmagic python-magic python-magic-bin"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "9bfca934",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.document_loaders import UnstructuredURLLoader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "93ce75de",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "loader = UnstructuredURLLoader(\n",
+    "    urls = [\n",
+    "        \"https://www.moneycontrol.com/news/business/banks/hdfc-bank-re-appoints-sanmoy-chakrabarti-as-chief-risk-officer-11259771.html\",\n",
+    "        \"https://www.moneycontrol.com/news/business/markets/market-corrects-post-rbi-ups-inflation-forecast-icrr-bet-on-these-top-10-rate-sensitive-stocks-ideas-11142611.html\"\n",
+    "    ]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "be914bfa",
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "2"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = loader.load()\n",
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "id": "52838711",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'English\\n\\nHindi\\n\\nGujarati\\n\\nSpecials\\n\\nTrending Stocks\\n\\nIRFC\\xa0INE053F01010, IRFC, 543257\\n\\nTata Power\\xa0INE'"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].page_content[0:100]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "8b587350",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'source': 'https://www.moneycontrol.com/news/business/banks/hdfc-bank-re-appoints-sanmoy-chakrabarti-as-chief-risk-officer-11259771.html'}"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data[0].metadata"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "df45adf4",
+   "metadata": {},
+   "source": [
+    "## Text Splitters"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7ce2e736",
+   "metadata": {},
+   "source": [
+    "Why do we need text splitters in first place?\n",
+    "\n",
+    "LLM's have token limits. Hence we need to split the text which can be large into small chunks so that each chunk size is under the token limit. There are various text splitter classes in langchain that allows us to do this."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 80,
+   "id": "90431c5b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Taking some random text from wikipedia\n",
+    "\n",
+    "text = \"\"\"Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan. \n",
+    "It stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt Damon, and Michael Caine. \n",
+    "Set in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for humankind.\n",
+    "\n",
+    "Brothers Christopher and Jonathan Nolan wrote the screenplay, which had its origins in a script Jonathan developed in 2007 and was originally set to be directed by Steven Spielberg. \n",
+    "Kip Thorne, a Caltech theoretical physicist and 2017 Nobel laureate in Physics,[4] was an executive producer, acted as a scientific consultant, and wrote a tie-in book, The Science of Interstellar. \n",
+    "Cinematographer Hoyte van Hoytema shot it on 35 mm movie film in the Panavision anamorphic format and IMAX 70 mm. Principal photography began in late 2013 and took place in Alberta, Iceland, and Los Angeles. \n",
+    "Interstellar uses extensive practical and miniature effects, and the company Double Negative created additional digital effects.\n",
+    "\n",
+    "Interstellar premiered in Los Angeles on October 26, 2014. In the United States, it was first released on film stock, expanding to venues using digital projectors. The film received generally positive reviews from critics and grossed over $677 million worldwide ($715 million after subsequent re-releases), making it the tenth-highest-grossing film of 2014. \n",
+    "It has been praised by astronomers for its scientific accuracy and portrayal of theoretical astrophysics.[5][6][7] Interstellar was nominated for five awards at the 87th Academy Awards, winning Best Visual Effects, and received numerous other accolades.\"\"\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7af9a95b",
+   "metadata": {},
+   "source": [
+    "#### Manual approach of splitting the text into chunks"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 81,
+   "id": "a51fc70e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher N'"
+      ]
+     },
+     "execution_count": 81,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Say LLM token limit is 100, in that case we can do simple thing such as this\n",
+    "\n",
+    "text[0:100]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 82,
+   "id": "2adae99a",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "264"
+      ]
+     },
+     "execution_count": 82,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Well but we want complete words and want to do this for entire text, may be we can use Python's split funciton\n",
+    "\n",
+    "words = text.split(\" \")\n",
+    "len(words)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "id": "56ec5613",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chunks = []\n",
+    "\n",
+    "s = \"\"\n",
+    "for word in words:\n",
+    "    s += word + \" \"\n",
+    "    if len(s)>200:\n",
+    "        chunks.append(s)\n",
+    "        s = \"\"\n",
+    "        \n",
+    "chunks.append(s)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 84,
+   "id": "95d902bd",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan. \\nIt stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt ',\n",
+       " 'Damon, and Michael Caine. \\nSet in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in ']"
+      ]
+     },
+     "execution_count": 84,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chunks[:2]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ff06ebc4",
+   "metadata": {},
+   "source": [
+    "**Splitting data into chunks can be done in native python but it is a tidious process. Also if necessary, you may need to experiment with various delimiters in an iterative manner to ensure that each chunk does not exceed the token length limit of the respective LLM.**\n",
+    "\n",
+    "**Langchain provides a better way through text splitter classes.**"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f64b2909",
+   "metadata": {},
+   "source": [
+    "#### Using Text Splitter Classes from Langchain\n",
+    "\n",
+    "#### CharacterTextSplitter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 85,
+   "id": "9505bedd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import CharacterTextSplitter\n",
+    "\n",
+    "splitter = CharacterTextSplitter(\n",
+    "    separator = \"\\n\",\n",
+    "    chunk_size=200,\n",
+    "    chunk_overlap=0\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 86,
+   "id": "4d86bc63",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Created a chunk of size 210, which is longer than the specified 200\n",
+      "Created a chunk of size 208, which is longer than the specified 200\n",
+      "Created a chunk of size 358, which is longer than the specified 200\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "9"
+      ]
+     },
+     "execution_count": 86,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "chunks = splitter.split_text(text)\n",
+    "len(chunks)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 87,
+   "id": "e027b9de",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "105\n",
+      "120\n",
+      "210\n",
+      "181\n",
+      "197\n",
+      "207\n",
+      "128\n",
+      "357\n",
+      "253\n"
+     ]
+    }
+   ],
+   "source": [
+    "for chunk in chunks:\n",
+    "    print(len(chunk))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dcc73da3",
+   "metadata": {},
+   "source": [
+    "As you can see, all though we gave 200 as a chunk size since the split was based on \\n, it ended up creating chunks that are bigger than size 200. "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f86f3a10",
+   "metadata": {},
+   "source": [
+    "Another class from Langchain can be used to recursively split the text based on a list of separators. This class is RecursiveTextSplitter. Let's see how it works"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "00a61cf1",
+   "metadata": {},
+   "source": [
+    "#### RecursiveTextSplitter"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 88,
+   "id": "dacf5e8b",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan. \\nIt stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt Damon, and Michael Caine. \\nSet in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for humankind.\\n\\nBrothers Christopher and Jonathan Nolan wrote the screenplay, which had its origins in a script Jonathan developed in 2007 and was originally set to be directed by Steven Spielberg. \\nKip Thorne, a Caltech theoretical physicist and 2017 Nobel laureate in Physics,[4] was an executive producer, acted as a scientific consultant, and wrote a tie-in book, The Science of Interstellar. \\nCinematographer Hoyte van Hoytema shot it on 35 mm movie film in the Panavision anamorphic format and IMAX 70 mm. Principal photography began in late 2013 and took place in Alberta, Iceland, and Los Angeles. \\nInterstellar uses extensive practical and miniature effects, and the company Double Negative created additional digital effects.\\n\\nInterstellar premiered in Los Angeles on October 26, 2014. In the United States, it was first released on film stock, expanding to venues using digital projectors. The film received generally positive reviews from critics and grossed over $677 million worldwide ($715 million after subsequent re-releases), making it the tenth-highest-grossing film of 2014. \\nIt has been praised by astronomers for its scientific accuracy and portrayal of theoretical astrophysics.[5][6][7] Interstellar was nominated for five awards at the 87th Academy Awards, winning Best Visual Effects, and received numerous other accolades.'"
+      ]
+     },
+     "execution_count": 88,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "text"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 117,
+   "id": "848eae34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "r_splitter = RecursiveCharacterTextSplitter(\n",
+    "    separators = [\"\\n\\n\", \"\\n\", \" \"],  # List of separators based on requirement (defaults to [\"\\n\\n\", \"\\n\", \" \"])\n",
+    "    chunk_size = 200,  # size of each chunk created\n",
+    "    chunk_overlap  = 0,  # size of  overlap between chunks in order to maintain the context\n",
+    "    length_function = len  # Function to calculate size, currently we are using \"len\" which denotes length of string however you can pass any token counter)\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 118,
+   "id": "1151c51d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "105\n",
+      "120\n",
+      "199\n",
+      "10\n",
+      "181\n",
+      "197\n",
+      "198\n",
+      "8\n",
+      "128\n",
+      "191\n",
+      "165\n",
+      "198\n",
+      "54\n"
+     ]
+    }
+   ],
+   "source": [
+    "chunks = r_splitter.split_text(text)\n",
+    "\n",
+    "for chunk in chunks:\n",
+    "    print(len(chunk))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "32135f4d",
+   "metadata": {},
+   "source": [
+    "**Let's understand how exactly it formed these chunks**"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 109,
+   "id": "57ef6974",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan. \\nIt stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt Damon, and Michael Caine. \\nSet in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for humankind.'"
+      ]
+     },
+     "execution_count": 109,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "first_split = text.split(\"\\n\\n\")[0]\n",
+    "first_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 110,
+   "id": "2bc7719f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "439"
+      ]
+     },
+     "execution_count": 110,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(first_split)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e1018d24",
+   "metadata": {},
+   "source": [
+    "Recursive text splitter uses a list of separators, i.e.  separators = [\"\\n\\n\", \"\\n\", \".\"]\n",
+    "\n",
+    "So now it will first split using \\n\\n and then if the resulting chunk size is greater than the chunk_size parameter which is 200\n",
+    "in our case, then it will use the next separator which is \\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 119,
+   "id": "739cef71",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan. ',\n",
+       " 'It stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt Damon, and Michael Caine. ',\n",
+       " 'Set in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for humankind.']"
+      ]
+     },
+     "execution_count": 119,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "second_split = first_split.split(\"\\n\")\n",
+    "second_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 120,
+   "id": "903f5921",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "106\n",
+      "121\n",
+      "210\n"
+     ]
+    }
+   ],
+   "source": [
+    "for split in second_split:\n",
+    "    print(len(split))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ed1e69a7",
+   "metadata": {},
+   "source": [
+    "Third split exceeds chunk size 200. Now it will further try to split that using the third separator which is ' ' (space)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 115,
+   "id": "69f4da9d",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'Set in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for humankind.'"
+      ]
+     },
+     "execution_count": 115,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "second_split[2]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7fda659a",
+   "metadata": {},
+   "source": [
+    "When you split this using space (i.e. second_split[2].split(\" \")), it will separate out each word and then it will merge those \n",
+    "chunks such that their size is close to 200"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6f538e50",
+   "metadata": {},
+   "source": [
+    "<img src=\"chunk_size.jpg\"/>"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

vector_index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cdd74f971a1f9d3e73e47ac09fad6d1c27711fa92c9b2393a387838f405d033
+size 291235