Created using Colab

notebooks/LlamaIndex_101.ipynb

@@ -4,7 +4,10 @@
   "metadata": {
     "colab": {
       "provenance": [],
-      "authorship_tag": "ABX9TyNzaxKiokXX5SPot1IBiMhR",
+      "collapsed_sections": [
+        "HjI_gRaRutfj"
+      ],
+      "authorship_tag": "ABX9TyOzyI9AMRRR7iatfugkbz3Y",
       "include_colab_link": true
     },
     "kernelspec": {
@@ -36,16 +39,54 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "y_GAV7-zos0Y",
-        "outputId": "74d4a3c4-3576-455b-fbe2-1b8b67bf20d5"
+        "outputId": "aafcb14a-2caf-4424-c772-1d2bf6246cb2"
       },
-      "execution_count": 10,
+      "execution_count": 1,
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
             "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
-            "  Building wheel for wikipedia (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
+            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
+            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
+            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m219.0 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.8/526.8 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m273.8/273.8 kB\u001b[0m \u001b[31m1.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.2/92.2 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m1.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m2.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.5/15.5 MB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.1/141.1 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.8/62.8 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m295.8/295.8 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m3.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m283.9 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.2/49.2 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+            "\u001b[?25h  Building wheel for wikipedia (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+            "  Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n"
           ]
         }
       ]
@@ -62,7 +103,7 @@
       "metadata": {
         "id": "Ua0KNwgvyCaj"
       },
-      "execution_count": 49,
+      "execution_count": 2,
       "outputs": []
     },
     {
@@ -70,12 +111,12 @@
       "source": [
         "import os\n",
         "\n",
-        "os.environ['OPENAI_API_KEY'] = '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'"
+        "os.environ['OPENAI_API_KEY'] = '[OPENAI_API_KEY]'"
       ],
       "metadata": {
         "id": "--Q2zk06wElp"
       },
-      "execution_count": 2,
+      "execution_count": 3,
       "outputs": []
     },
     {
@@ -92,7 +133,7 @@
       "metadata": {
         "id": "tjwZjA8-wITr"
       },
-      "execution_count": 3,
+      "execution_count": 4,
       "outputs": []
     },
     {
@@ -359,24 +400,24 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "DrzbBAglwUo0",
-        "outputId": "73f30202-a708-4112-8491-9152e228c6cb"
+        "outputId": "5bd17a72-3733-4b8c-e421-5f363f05895f"
       },
-      "execution_count": 35,
+      "execution_count": 6,
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
-            "--2024-07-24 17:20:40--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt\n",
-            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.111.133, ...\n",
-            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n",
+            "--2024-07-24 18:48:21--  https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt\n",
+            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...\n",
+            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
             "HTTP request sent, awaiting response... 200 OK\n",
             "Length: 75042 (73K) [text/plain]\n",
             "Saving to: ‘./paul_graham/paul_graham_essay.txt’\n",
             "\n",
             "\r          ./paul_gr   0%[                    ]       0  --.-KB/s               \r./paul_graham/paul_ 100%[===================>]  73.28K  --.-KB/s    in 0.02s   \n",
             "\n",
-            "2024-07-24 17:20:40 (3.33 MB/s) - ‘./paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n",
+            "2024-07-24 18:48:21 (2.95 MB/s) - ‘./paul_graham/paul_graham_essay.txt’ saved [75042/75042]\n",
             "\n"
           ]
         }
@@ -393,7 +434,7 @@
       "metadata": {
         "id": "S8-QmnkCwIiU"
       },
-      "execution_count": 37,
+      "execution_count": 7,
       "outputs": []
     },
     {
@@ -410,7 +451,7 @@
       "metadata": {
         "id": "DfWglp75xc5f"
       },
-      "execution_count": 38,
+      "execution_count": 8,
       "outputs": []
     },
     {
@@ -418,13 +459,12 @@
       "source": [
         "from llama_index.core import StorageContext\n",
         "\n",
-        "storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
-        "storage_context.docstore.add_documents(nodes)"
+        "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
       ],
       "metadata": {
         "id": "-EVBlUC-xcj1"
       },
-      "execution_count": 39,
+      "execution_count": 11,
       "outputs": []
     },
     {
@@ -439,7 +479,7 @@
       "metadata": {
         "id": "8lMa4h9Cwn8b"
       },
-      "execution_count": 40,
+      "execution_count": 12,
       "outputs": []
     },
     {
@@ -450,7 +490,7 @@
       "metadata": {
         "id": "tJsfskjHxj0e"
       },
-      "execution_count": 44,
+      "execution_count": 13,
       "outputs": []
     },
     {
@@ -477,7 +517,7 @@
       "metadata": {
         "id": "yL9TsFwxxuoA"
       },
-      "execution_count": 50,
+      "execution_count": 14,
       "outputs": []
     },
     {
@@ -492,9 +532,9 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "JWc_n5Lhx2bq",
-        "outputId": "cd992d20-c701-4eb7-aaf2-30f790d1ca24"
+        "outputId": "972fa40f-065b-420a-d5e8-af36c1a2ad2d"
       },
-      "execution_count": 51,
+      "execution_count": 15,
       "outputs": [
         {
           "output_type": "stream",
@@ -504,46 +544,9 @@
             "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What did Paul Graham work on before Y Combinator?\n",
             "\u001b[0m\u001b[1;3;38;2;90;149;237m[pg_essay] Q: What did Paul Graham work on during Y Combinator?\n",
             "\u001b[0m\u001b[1;3;38;2;11;159;203m[pg_essay] Q: What did Paul Graham work on after Y Combinator?\n",
-            "\u001b[0mGenerated 1 sub questions.\n",
-            "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What did Paul Graham work on after Y Combinator?\n",
-            "\u001b[0mGenerated 1 sub questions.\n",
-            "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What is the title of Paul Graham's essay on What I Worked On?\n",
-            "\u001b[0mGenerated 1 sub questions.\n",
-            "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What is the title of Paul Graham's essay on What I Worked On?\n",
-            "\u001b[0mGenerated 1 sub questions.\n",
-            "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What did Paul Graham work on after Y Combinator?\n",
-            "\u001b[0mGenerated 1 sub questions.\n",
-            "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What is the title of Paul Graham's essay on What I Worked On?\n",
-            "\u001b[0m"
-          ]
-        },
-        {
-          "output_type": "stream",
-          "name": "stderr",
-          "text": [
-            "/usr/lib/python3.10/abc.py:123: RuntimeWarning: coroutine 'run_async_tasks.<locals>._gather' was never awaited\n",
-            "  return _abc_subclasscheck(cls, subclass)\n",
-            "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n",
-            "/usr/lib/python3.10/abc.py:123: RuntimeWarning: coroutine 'SubQuestionQueryEngine._aquery_subq' was never awaited\n",
-            "  return _abc_subclasscheck(cls, subclass)\n",
-            "RuntimeWarning: Enable tracemalloc to get the object allocation traceback\n"
-          ]
-        },
-        {
-          "output_type": "stream",
-          "name": "stdout",
-          "text": [
-            "Generated 1 sub questions.\n",
-            "\u001b[1;3;38;2;237;90;200m[pg_essay] Q: What is the title of Paul Graham's essay on What I Worked On?\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: After Y Combinator, Paul Graham started painting.\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: The title of Paul Graham's essay on What I Worked On is \"What I Worked On\".\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: After Y Combinator, Paul Graham started painting.\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: What I Worked On\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: What I Worked On\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: The title of Paul Graham's essay on What I Worked On is \"What I Worked On\".\n",
+            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: Paul Graham worked on building online stores through a web app called Viaweb before starting Y Combinator.\n",
             "\u001b[0m\u001b[1;3;38;2;11;159;203m[pg_essay] A: After Y Combinator, Paul Graham started painting.\n",
-            "\u001b[0m\u001b[1;3;38;2;90;149;237m[pg_essay] A: Paul Graham worked on various projects during his time at Y Combinator.\n",
-            "\u001b[0m\u001b[1;3;38;2;237;90;200m[pg_essay] A: Paul Graham worked on developing Viaweb before Y Combinator.\n",
+            "\u001b[0m\u001b[1;3;38;2;90;149;237m[pg_essay] A: Paul Graham worked on various aspects during Y Combinator, including being an angel firm, funding startups in batches, providing seed investments, and offering support to startups similar to what he had received when starting his own company.\n",
             "\u001b[0m"
           ]
         }
@@ -559,16 +562,16 @@
           "base_uri": "https://localhost:8080/"
         },
         "id": "T-ZC66Ltx5Za",
-        "outputId": "d7a1c85d-d73c-467d-e0df-9e06078622e2"
+        "outputId": "f45ec6bb-ee45-4526-80bd-ca7c30cdbf8f"
       },
-      "execution_count": 52,
+      "execution_count": 16,
       "outputs": [
         {
           "output_type": "stream",
           "name": "stdout",
           "text": [
             ">>> The final response:\n",
-            " Paul Graham worked on developing Viaweb before Y Combinator, on various projects during his time at Y Combinator, and started painting after Y Combinator.\n"
+            " Paul Graham's life involved building online stores through Viaweb before Y Combinator, working on various aspects within Y Combinator such as funding startups and providing support, and then transitioning to painting after Y Combinator.\n"
           ]
         }
       ]