Upload example_images_tokenize.ipynb with huggingface_hub

example_images_tokenize.ipynb

@@ -1432,41 +1432,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "3ca6861c-4182-4d23-984f-92e0b5ba22f2",
-   "metadata": {},
-   "outputs": [
-    {
-     "ename": "AssertionError",
-     "evalue": "Key <|img_start|> is not a special token",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mAssertionError\u001b[0m                            Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[8], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mllama_tokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43madd_special_tokens\u001b[49m\u001b[43m(\u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m<|img_start|>\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43masdf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m\n",
-      "File \u001b[0;32m~/.local/lib/python3.11/site-packages/transformers/tokenization_utils_base.py:947\u001b[0m, in \u001b[0;36mSpecialTokensMixin.add_special_tokens\u001b[0;34m(self, special_tokens_dict, replace_additional_special_tokens)\u001b[0m\n\u001b[1;32m    945\u001b[0m added_tokens \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    946\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m special_tokens_dict\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m--> 947\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mSPECIAL_TOKENS_ATTRIBUTES, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mKey \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m is not a special token\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    949\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mverbose:\n\u001b[1;32m    950\u001b[0m         logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mAssigning \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mvalue\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m to the \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mkey\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m key of the tokenizer\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
-      "\u001b[0;31mAssertionError\u001b[0m: Key <|img_start|> is not a special token"
-     ]
-    }
-   ],
-   "source": [
-    "llama_tokenizer.regist"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "aaf0a3d1-dd5d-402b-9802-b692921c9079",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "llama_tokenizer.add_special_tokens({\"image start\":\"<|img_start|>\"})\n",
-    "llama_tokenizer.add_special_tokens(\"<|img_end|>\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 55,
+   "execution_count": 6,
    "id": "a3e9b926-2cd4-4870-b219-7f92cd9295c4",
    "metadata": {},
    "outputs": [],
@@ -1487,7 +1453,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
+   "execution_count": 7,
    "id": "7998066b-6fe7-4ca8-8ede-3870f37f3725",
    "metadata": {},
    "outputs": [],
@@ -1521,7 +1487,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 67,
+   "execution_count": 8,
    "id": "8919fa2e-4e07-48d9-876f-c033a0fd1ab8",
    "metadata": {},
    "outputs": [],
@@ -1531,7 +1497,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 69,
+   "execution_count": 9,
    "id": "c97bf893-aeda-43e4-b703-bdb425dcda3e",
    "metadata": {},
    "outputs": [],
@@ -1566,7 +1532,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 74,
+   "execution_count": 10,
    "id": "320c91a9-e0c0-4786-8eaa-424fa5e8e41e",
    "metadata": {
     "scrolled": true
@@ -1589,7 +1555,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 75,
+   "execution_count": 11,
    "id": "85037f9a-91aa-4991-ae41-251e926343b9",
    "metadata": {
     "scrolled": true
@@ -2601,7 +2567,7 @@
        " ...]"
       ]
      },
-     "execution_count": 75,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }