{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0a6e1750-4cbb-489e-8194-3bd8e5933d20",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Text Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "64d4867c-f9db-4cab-99da-e42d92618089",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'langchain'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mlangchain\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdocument_loaders\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TextLoader\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'langchain'"
     ]
    }
   ],
   "source": [
    "from langchain.document_loaders import TextLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "539810ab-3bb7-4f3d-afde-a24784b0b511",
   "metadata": {},
   "outputs": [],
   "source": [
    "loader = TextLoader('example_fi.txt')\n",
    "data = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a0d00b71-9824-4d42-bf8e-0b0404928c86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'source': 'example_fi.txt'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0].page_content\n",
    "data[0].metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0bf4d69b-8ff3-4573-a48d-2eb7f1cddd51",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.document_loaders import UnstructuredURLLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4aa9ca90-f528-4b28-998b-d880cbee2b82",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Text Splitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "60aa3514-1190-4fcc-b388-65d13d63341b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Taking some random text from wikipedia\n",
    "\n",
    "text = \"\"\"Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan. \n",
    "It stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt Damon, and Michael Caine. \n",
    "Set in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for humankind.\n",
    "\n",
    "Brothers Christopher and Jonathan Nolan wrote the screenplay, which had its origins in a script Jonathan developed in 2007 and was originally set to be directed by Steven Spielberg. \n",
    "Kip Thorne, a Caltech theoretical physicist and 2017 Nobel laureate in Physics,[4] was an executive producer, acted as a scientific consultant, and wrote a tie-in book, The Science of Interstellar. \n",
    "Cinematographer Hoyte van Hoytema shot it on 35 mm movie film in the Panavision anamorphic format and IMAX 70 mm. Principal photography began in late 2013 and took place in Alberta, Iceland, and Los Angeles. \n",
    "Interstellar uses extensive practical and miniature effects, and the company Double Negative created additional digital effects.\n",
    "\n",
    "Interstellar premiered in Los Angeles on October 26, 2014. In the United States, it was first released on film stock, expanding to venues using digital projectors. The film received generally positive reviews from critics and grossed over $677 million worldwide ($715 million after subsequent re-releases), making it the tenth-highest-grossing film of 2014. \n",
    "It has been praised by astronomers for its scientific accuracy and portrayal of theoretical astrophysics.[5][6][7] Interstellar was nominated for five awards at the 87th Academy Awards, winning Best Visual Effects, and received numerous other accolades.\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7c2dbbc2-5634-4eb8-bb0b-f39f0bdf8342",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split Text using LangChain\n",
    "\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "# need to make sure that each of the chunks aren't too large...\n",
    "\n",
    "# RecursiveCharacterTextSplitter recursively splits the chunks using each of the separators that you've provided, instead of just one\n",
    "# Ie we first split the text by paragraph, then line, then space\n",
    "splitter = RecursiveCharacterTextSplitter(\n",
    "    separators=[\"\\n\\n\", \"\\n\", \" \"],\n",
    "    chunk_size=200,\n",
    "    chunk_overlap=0\n",
    ")\n",
    "\n",
    "chunks = splitter.split_text(text)\n",
    "len(chunks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "07b992fb-d042-4da4-8bb7-898bd0d8fd33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Interstellar is a 2014 epic science fiction film co-written, directed, and produced by Christopher Nolan.', 'It stars Matthew McConaughey, Anne Hathaway, Jessica Chastain, Bill Irwin, Ellen Burstyn, Matt Damon, and Michael Caine.', 'Set in a dystopian future where humanity is embroiled in a catastrophic blight and famine, the film follows a group of astronauts who travel through a wormhole near Saturn in search of a new home for', 'humankind.', 'Brothers Christopher and Jonathan Nolan wrote the screenplay, which had its origins in a script Jonathan developed in 2007 and was originally set to be directed by Steven Spielberg.', 'Kip Thorne, a Caltech theoretical physicist and 2017 Nobel laureate in Physics,[4] was an executive producer, acted as a scientific consultant, and wrote a tie-in book, The Science of Interstellar.', 'Cinematographer Hoyte van Hoytema shot it on 35 mm movie film in the Panavision anamorphic format and IMAX 70 mm. Principal photography began in late 2013 and took place in Alberta, Iceland, and Los', 'Angeles.', 'Interstellar uses extensive practical and miniature effects, and the company Double Negative created additional digital effects.', 'Interstellar premiered in Los Angeles on October 26, 2014. In the United States, it was first released on film stock, expanding to venues using digital projectors. The film received generally', 'positive reviews from critics and grossed over $677 million worldwide ($715 million after subsequent re-releases), making it the tenth-highest-grossing film of 2014.', 'It has been praised by astronomers for its scientific accuracy and portrayal of theoretical astrophysics.[5][6][7] Interstellar was nominated for five awards at the 87th Academy Awards, winning Best', 'Visual Effects, and received numerous other accolades.']\n"
     ]
    }
   ],
   "source": [
    "print(chunks)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}