{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already up-to-date: transformers in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (4.21.3)\n",
      "Requirement already up-to-date: torch in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (1.12.1)\n",
      "Requirement already up-to-date: torchvision in c:\\users\\divanma\\appdata\\roaming\\python\\python37\\site-packages (0.13.1)\n",
      "Requirement already satisfied, skipping upgrade: numpy>=1.17 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (1.21.6)\n",
      "Requirement already satisfied, skipping upgrade: pyyaml>=5.1 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (6.0)\n",
      "Requirement already satisfied, skipping upgrade: tokenizers!=0.11.3,<0.13,>=0.11.1 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (0.12.1)\n",
      "Requirement already satisfied, skipping upgrade: requests in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (2.28.1)\n",
      "Requirement already satisfied, skipping upgrade: importlib-metadata; python_version < \"3.8\" in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (4.11.3)\n",
      "Requirement already satisfied, skipping upgrade: packaging>=20.0 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (21.3)\n",
      "Requirement already satisfied, skipping upgrade: huggingface-hub<1.0,>=0.1.0 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (0.9.1)\n",
      "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (4.64.1)\n",
      "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (2022.9.11)\n",
      "Requirement already satisfied, skipping upgrade: filelock in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from transformers) (3.8.0)\n",
      "Requirement already satisfied, skipping upgrade: typing-extensions in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from torch) (4.1.1)\n",
      "Requirement already satisfied, skipping upgrade: pillow!=8.3.*,>=5.3.0 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from torchvision) (5.4.1)\n",
      "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from requests->transformers) (2022.6.15)\n",
      "Requirement already satisfied, skipping upgrade: charset-normalizer<3,>=2 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from requests->transformers) (2.0.4)\n",
      "Requirement already satisfied, skipping upgrade: urllib3<1.27,>=1.21.1 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from requests->transformers) (1.26.9)\n",
      "Requirement already satisfied, skipping upgrade: idna<4,>=2.5 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied, skipping upgrade: zipp>=0.5 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.8.0)\n",
      "Requirement already satisfied, skipping upgrade: pyparsing!=3.0.5,>=2.0.2 in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from packaging>=20.0->transformers) (3.0.4)\n",
      "Requirement already satisfied, skipping upgrade: colorama; platform_system == \"Windows\" in c:\\users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.5)\n"
     ]
    }
   ],
   "source": [
    "!pip install -U --user transformers torch torchvision"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForCausalLM\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load from Local"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_dir = os.getcwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(root_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mnti\u001b[1;34m(s)\u001b[0m\n\u001b[0;32m    186\u001b[0m             \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnts\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"ascii\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"strict\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 187\u001b[1;33m             \u001b[0mn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;34m\"0\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m8\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    188\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: invalid literal for int() with base 8: 'q\\x03ctorch'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mInvalidHeaderError\u001b[0m                        Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mnext\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   2288\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2289\u001b[1;33m                 \u001b[0mtarinfo\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtarinfo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfromtarfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2290\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mEOFHeaderError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mfromtarfile\u001b[1;34m(cls, tarfile)\u001b[0m\n\u001b[0;32m   1094\u001b[0m         \u001b[0mbuf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtarfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfileobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBLOCKSIZE\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1095\u001b[1;33m         \u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrombuf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1096\u001b[0m         \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moffset\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtarfile\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfileobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mBLOCKSIZE\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mfrombuf\u001b[1;34m(cls, buf, encoding, errors)\u001b[0m\n\u001b[0;32m   1036\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1037\u001b[1;33m         \u001b[0mchksum\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnti\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m148\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m156\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1038\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mchksum\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcalc_chksums\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mnti\u001b[1;34m(s)\u001b[0m\n\u001b[0;32m    188\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 189\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mInvalidHeaderError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"invalid header\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    190\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mn\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInvalidHeaderError\u001b[0m: invalid header",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mReadError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36m_load\u001b[1;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[0;32m    555\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 556\u001b[1;33m                 \u001b[0mstorage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    557\u001b[0m                 \u001b[0mstorage_dtype\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36mlegacy_load\u001b[1;34m(f)\u001b[0m\n\u001b[0;32m    466\u001b[0m             \u001b[1;31m# and the tensor back up with no problems in _this_ and future\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 467\u001b[1;33m             \u001b[1;31m# versions of pytorch, but in older versions, here's the problem:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    468\u001b[0m             \u001b[1;31m# the storage will be loaded up as a _UntypedStorage, and then the\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mopen\u001b[1;34m(cls, name, mode, fileobj, bufsize, **kwargs)\u001b[0m\n\u001b[0;32m   1590\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mCompressionError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"unknown compression type %r\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mcomptype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1591\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfilemode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfileobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1592\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mtaropen\u001b[1;34m(cls, name, mode, fileobj, **kwargs)\u001b[0m\n\u001b[0;32m   1620\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"mode must be 'r', 'a', 'w' or 'x'\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1621\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfileobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1622\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, name, mode, fileobj, format, tarinfo, dereference, ignore_zeros, encoding, errors, pax_headers, debug, errorlevel, copybufsize)\u001b[0m\n\u001b[0;32m   1483\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfirstmember\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1484\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfirstmember\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1485\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\tarfile.py\u001b[0m in \u001b[0;36mnext\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   2300\u001b[0m                 \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moffset\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2301\u001b[1;33m                     \u001b[1;32mraise\u001b[0m \u001b[0mReadError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2302\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mEmptyHeaderError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mReadError\u001b[0m: invalid header",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\transformers\\modeling_utils.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[1;34m(checkpoint_file)\u001b[0m\n\u001b[0;32m    366\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 367\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcheckpoint_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmap_location\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"cpu\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    368\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[0;32m    386\u001b[0m     \u001b[0mserialized_container_types\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 387\u001b[1;33m     \u001b[0mserialized_storages\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    388\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36m_load\u001b[1;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[0;32m    559\u001b[0m                 \u001b[0mstorage_numel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstorage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 560\u001b[1;33m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    561\u001b[0m             \u001b[1;31m# If storage is allocated, ensure that any other saved storages\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mRuntimeError\u001b[0m: c:\\dev\\P\\gpt-neo-1.3B-fiction-novel-generation\\pytorch_model.bin is a zip archive (did you mean to use torch.jit.load()?)",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_20996\\2464673473.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mAutoModelForCausalLM\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mroot_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\transformers\\models\\auto\\auto_factory.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[1;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[0;32m    444\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    445\u001b[0m             \u001b[0mmodel_class\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_get_model_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_model_mapping\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 446\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mmodel_class\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_pretrained\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpretrained_model_name_or_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    447\u001b[0m         raise ValueError(\n\u001b[0;32m    448\u001b[0m             \u001b[1;34mf\"Unrecognized configuration class {config.__class__} for this kind of AutoModel: {cls.__name__}.\\n\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\transformers\\modeling_utils.py\u001b[0m in \u001b[0;36mfrom_pretrained\u001b[1;34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[0m\n\u001b[0;32m   2065\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_sharded\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mstate_dict\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2066\u001b[0m                 \u001b[1;31m# Time to load the checkpoint\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2067\u001b[1;33m                 \u001b[0mstate_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mload_state_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresolved_archive_file\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2068\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2069\u001b[0m             \u001b[1;31m# set dtype to instantiate the model under:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\site-packages\\transformers\\modeling_utils.py\u001b[0m in \u001b[0;36mload_state_dict\u001b[1;34m(checkpoint_file)\u001b[0m\n\u001b[0;32m    369\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcheckpoint_file\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 371\u001b[1;33m                 \u001b[1;32mif\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"version\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    372\u001b[0m                     raise OSError(\n\u001b[0;32m    373\u001b[0m                         \u001b[1;34m\"You seem to have cloned a repository without having git-lfs installed. Please install \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\Users\\divanma\\.conda\\envs\\pytorchenv\\lib\\encodings\\cp1252.py\u001b[0m in \u001b[0;36mdecode\u001b[1;34m(self, input, final)\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mIncrementalDecoder\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mdecode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcharmap_decode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdecoding_table\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mStreamWriter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCodec\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcodecs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStreamWriter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model = AutoModelForCausalLM.from_pretrained(root_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inference Example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer('Hello, my dog is cute', return_tensors='pt')\n",
    "outputs = model(**inputs, labels=inputs['input_ids'])\n",
    "\n",
    "print(f'[OUTPUT] {outputs}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Valuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = outputs.loss\n",
    "logits = outputs.logits\n",
    "\n",
    "print(f'[LOSS] {loss}, [LOGITS] {logits}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.3 ('pytorchenv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a1f58ad6df42b3a9f00d8caf282612c40ca90330c75003a8465db9aa3eb9729c"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}