fix vocab json file

inference.ipynb

@@ -2,8 +2,8 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 7,
-   "id": "9abf3270",
+   "execution_count": 1,
+   "id": "438927ca",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,126 +15,45 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
-   "id": "6e0830a2",
+   "execution_count": 5,
+   "id": "27a57965",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
-    "# model = AutoModelForCTC.from_pretrained(\".\").to('cuda')\n",
-    "# processor = Wav2Vec2Processor.from_pretrained(\".\")"
+    "model = AutoModelForCTC.from_pretrained(\".\").to('cuda')\n",
+    "processor = Wav2Vec2Processor.from_pretrained(\".\")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
-   "id": "126e39e0",
-   "metadata": {},
+   "execution_count": 3,
+   "id": "1d4324df",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "outputs_hidden": true
+    }
+   },
    "outputs": [
     {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a2fdec3c288946a19a5b36618af4c26c",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/2.02k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ae6b614b82aa4627b55b8c742ca8898e",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/1.18G [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8c39ff01100c40aab1191514bb52399d",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/214 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "5078aede36ff4dc1a17e24a967cb46dd",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/260 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a4c7e0e6e26f4c5e8f40e79b67b61c17",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/795 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b55bcb16d8d042059a6487391e1a51de",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/23.0 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0546a666b47a4d418e62ccc8fec58bd4",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading:   0%|          | 0.00/309 [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "ename": "JSONDecodeError",
+     "evalue": "Expecting value: line 1 column 1 (char 0)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [3]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m model \u001b[38;5;241m=\u001b[39m AutoModelForCTC\u001b[38;5;241m.\u001b[39mfrom_pretrained(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvitouphy/xls-r-300m-km\u001b[39m\u001b[38;5;124m\"\u001b[39m)\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcuda\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m processor \u001b[38;5;241m=\u001b[39m \u001b[43mWav2Vec2Processor\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mvitouphy/xls-r-300m-km\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/models/wav2vec2/processing_wav2vec2.py:117\u001b[0m, in \u001b[0;36mWav2Vec2Processor.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, **kwargs)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[38;5;66;03m# load generic `AutoTokenizer`\u001b[39;00m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;66;03m# need fallback here for backward compatibility in case processor is\u001b[39;00m\n\u001b[1;32m    114\u001b[0m \u001b[38;5;66;03m# loaded from just a tokenizer file that does not have a `tokenizer_class` attribute\u001b[39;00m\n\u001b[1;32m    115\u001b[0m \u001b[38;5;66;03m# behavior should be deprecated in major future release\u001b[39;00m\n\u001b[1;32m    116\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 117\u001b[0m     tokenizer \u001b[38;5;241m=\u001b[39m \u001b[43mAutoTokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    118\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m:\n\u001b[1;32m    119\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(\n\u001b[1;32m    120\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mLoading a tokenizer inside \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m from a config that does not\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    121\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m include a `tokenizer_class` attribute is deprecated and will be \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[38;5;167;01mFutureWarning\u001b[39;00m,\n\u001b[1;32m    126\u001b[0m     )\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/models/auto/tokenization_auto.py:514\u001b[0m, in \u001b[0;36mAutoTokenizer.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *inputs, **kwargs)\u001b[0m\n\u001b[1;32m    510\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m tokenizer_class \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    511\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    512\u001b[0m             \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTokenizer class \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mtokenizer_class_candidate\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m does not exist or is not currently imported.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    513\u001b[0m         )\n\u001b[0;32m--> 514\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtokenizer_class\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    516\u001b[0m \u001b[38;5;66;03m# Otherwise we have to be creative.\u001b[39;00m\n\u001b[1;32m    517\u001b[0m \u001b[38;5;66;03m# if model is an encoder decoder, the encoder tokenizer class is used by default\u001b[39;00m\n\u001b[1;32m    518\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(config, EncoderDecoderConfig):\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1773\u001b[0m, in \u001b[0;36mPreTrainedTokenizerBase.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *init_inputs, **kwargs)\u001b[0m\n\u001b[1;32m   1770\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1771\u001b[0m         logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mloading file \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m from cache at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresolved_vocab_files[file_id]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1773\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_from_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1774\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresolved_vocab_files\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1775\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1776\u001b[0m \u001b[43m    \u001b[49m\u001b[43minit_configuration\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1777\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1778\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_auth_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_auth_token\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1779\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1780\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1781\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1908\u001b[0m, in \u001b[0;36mPreTrainedTokenizerBase._from_pretrained\u001b[0;34m(cls, resolved_vocab_files, pretrained_model_name_or_path, init_configuration, use_auth_token, cache_dir, *init_inputs, **kwargs)\u001b[0m\n\u001b[1;32m   1906\u001b[0m \u001b[38;5;66;03m# Instantiate tokenizer.\u001b[39;00m\n\u001b[1;32m   1907\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1908\u001b[0m     tokenizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1909\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m:\n\u001b[1;32m   1910\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\n\u001b[1;32m   1911\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnable to load vocabulary from file. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1912\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease check that the provided vocabulary is accessible and not corrupted.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1913\u001b[0m     )\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/models/wav2vec2/tokenization_wav2vec2.py:142\u001b[0m, in \u001b[0;36mWav2Vec2CTCTokenizer.__init__\u001b[0;34m(self, vocab_file, bos_token, eos_token, unk_token, pad_token, word_delimiter_token, do_lower_case, **kwargs)\u001b[0m\n\u001b[1;32m    139\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdo_lower_case \u001b[38;5;241m=\u001b[39m do_lower_case\n\u001b[1;32m    141\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(vocab_file, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m vocab_handle:\n\u001b[0;32m--> 142\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencoder \u001b[38;5;241m=\u001b[39m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvocab_handle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    143\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecoder \u001b[38;5;241m=\u001b[39m {v: k \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencoder\u001b[38;5;241m.\u001b[39mitems()}\n\u001b[1;32m    145\u001b[0m \u001b[38;5;66;03m# make sure that tokens made of several\u001b[39;00m\n\u001b[1;32m    146\u001b[0m \u001b[38;5;66;03m# characters are not split at tokenization\u001b[39;00m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/__init__.py:293\u001b[0m, in \u001b[0;36mload\u001b[0;34m(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload\u001b[39m(fp, \u001b[38;5;241m*\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, object_hook\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, parse_float\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    275\u001b[0m         parse_int\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, parse_constant\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, object_pairs_hook\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw):\n\u001b[1;32m    276\u001b[0m     \u001b[38;5;124;03m\"\"\"Deserialize ``fp`` (a ``.read()``-supporting file-like object containing\u001b[39;00m\n\u001b[1;32m    277\u001b[0m \u001b[38;5;124;03m    a JSON document) to a Python object.\u001b[39;00m\n\u001b[1;32m    278\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    291\u001b[0m \u001b[38;5;124;03m    kwarg; otherwise ``JSONDecoder`` is used.\u001b[39;00m\n\u001b[1;32m    292\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 293\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobject_hook\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobject_hook\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparse_float\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_float\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparse_int\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_int\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparse_constant\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_constant\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobject_pairs_hook\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobject_pairs_hook\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/__init__.py:357\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    352\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m kw[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mencoding\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m    354\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    355\u001b[0m         parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    356\u001b[0m         parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 357\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    358\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    359\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, s, _w\u001b[38;5;241m=\u001b[39mWHITESPACE\u001b[38;5;241m.\u001b[39mmatch):\n\u001b[1;32m    333\u001b[0m     \u001b[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m    334\u001b[0m \u001b[38;5;124;03m    containing a JSON document).\u001b[39;00m\n\u001b[1;32m    335\u001b[0m \n\u001b[1;32m    336\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m     obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_w\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    338\u001b[0m     end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m    339\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    353\u001b[0m     obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscan_once(s, idx)\n\u001b[1;32m    354\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpecting value\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, err\u001b[38;5;241m.\u001b[39mvalue) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m    356\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj, end\n",
+      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
+     ]
     }
    ],
    "source": [
@@ -144,8 +63,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
-   "id": "cf72163d",
+   "execution_count": 8,
+   "id": "3d61ff3b",
    "metadata": {},
    "outputs": [
     {
@@ -163,8 +82,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "id": "6b30d6ea",
+   "execution_count": 9,
+   "id": "a03f3af4",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -175,8 +94,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
-   "id": "bf9734cc",
+   "execution_count": 10,
+   "id": "9c88048b",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -185,8 +104,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
-   "id": "5e74effa",
+   "execution_count": 11,
+   "id": "f3bfc930",
    "metadata": {},
    "outputs": [
     {
@@ -199,7 +118,7 @@
        " 'sentence': 'ស៊ី ដាច់ ម៉ូតូ នៅ ពេល ដែល ប្រើ ឱ្យ ឌុប សម្ភារៈ គ្រឿង សង្ហារឹម យក ទៅ ឱ្យ ម៉ូយ នៅ ម្ដុំ វត្ដ សំរោងអណ្ដែត'}"
       ]
      },
-     "execution_count": 8,
+     "execution_count": 11,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -210,8 +129,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
-   "id": "c94de1a7",
+   "execution_count": 12,
+   "id": "122a898b",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -229,14 +148,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
-   "id": "c018376a",
+   "execution_count": 13,
+   "id": "153e7f45",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e9ec232c2caf4bdcb62b24696217a723",
+       "model_id": "a0dd47d98a4e448c9f786ce464348946",
        "version_major": 2,
        "version_minor": 0
       },
@@ -254,18 +173,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 45,
-   "id": "c0a02606",
+   "execution_count": 17,
+   "id": "8947d307",
    "metadata": {},
    "outputs": [],
    "source": [
-    "i = 21"
+    "i = 25"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 46,
-   "id": "19b8e75f",
+   "execution_count": 18,
+   "id": "3d6b46ca",
    "metadata": {},
    "outputs": [
     {
@@ -284,8 +203,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 47,
-   "id": "d15fb5a5",
+   "execution_count": 19,
+   "id": "d1550ddc",
    "metadata": {},
    "outputs": [
     {
@@ -293,10 +212,10 @@
      "output_type": "stream",
      "text": [
       "Prediction:\n",
-      "កាបូហ្វូនរ៉ាន និង កូហ្វាកទ័រ ជា មិត្ត និង គ្នា\n",
+      "ក្រុង ប៉យប៉ែត និង ខ្វាះទឹក ស្អាត ប្រើ ចាប់ ពិសព្ ប្តាល ក្រោយ ទៅ\n",
       "\n",
       "Reference:\n",
-      "កាបូហ្វូនរ៉ាន និង កូហ្វាក់ទ័រ ជា មិត្ត និង គ្នា\n"
+      "ក្រុង ប៉ោយប៉ែត នឹង ខ្វះ ទឹក ស្អាត ប្រើ ចាប់ ពី សប្តាហ៍ ក្រោយ ទៅ\n"
      ]
     }
    ],
@@ -313,7 +232,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "07bc1b8e",
+   "id": "5bbf1c82",
    "metadata": {},
    "outputs": [],
    "source": []
@@ -321,7 +240,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "b228faa1",
+   "id": "71b6f502",
    "metadata": {},
    "outputs": [],
    "source": []

preprocessor_config.json

@@ -0,0 +1,9 @@
+{
+  "do_normalize": true,
+  "feature_extractor_type": "Wav2Vec2FeatureExtractor",
+  "feature_size": 1,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

train_kh.ipynb

@@ -3,7 +3,7 @@
   {
    "cell_type": "code",
    "execution_count": 1,
-   "id": "1ffb676e",
+   "id": "1bf32ef8",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -16,7 +16,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "4aaaf656",
+   "id": "d2deec6c",
    "metadata": {
     "collapsed": true,
     "jupyter": {
@@ -19167,7 +19167,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "ae085c17",
+   "id": "6fe38e7a",
    "metadata": {},
    "source": [
     "### Load KH Data"
@@ -19176,7 +19176,7 @@
   {
    "cell_type": "code",
    "execution_count": 4,
-   "id": "4bc84318",
+   "id": "b75f1fec",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19197,7 +19197,7 @@
   {
    "cell_type": "code",
    "execution_count": 5,
-   "id": "e5bc62ba",
+   "id": "433fe749",
    "metadata": {},
    "outputs": [
     {
@@ -19307,7 +19307,7 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "36718ca2",
+   "id": "c6d633ad",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19321,7 +19321,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "ad45ec58",
+   "id": "acb914d0",
    "metadata": {},
    "source": [
     "### Clean Up the Text"
@@ -19330,7 +19330,7 @@
   {
    "cell_type": "code",
    "execution_count": 6,
-   "id": "cceeec4b",
+   "id": "bc3a017b",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19346,7 +19346,7 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "79539c6a",
+   "id": "4a7b6a10",
    "metadata": {
     "collapsed": true,
     "jupyter": {
@@ -19402,7 +19402,7 @@
   {
    "cell_type": "code",
    "execution_count": 7,
-   "id": "d06a790a",
+   "id": "7f511e3f",
    "metadata": {},
    "outputs": [
     {
@@ -19423,7 +19423,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "ca7019af",
+   "id": "205a6e23",
    "metadata": {},
    "source": [
     "### Build Character"
@@ -19432,7 +19432,7 @@
   {
    "cell_type": "code",
    "execution_count": 8,
-   "id": "9e6c5aeb",
+   "id": "48a97fac",
    "metadata": {},
    "outputs": [
     {
@@ -19480,7 +19480,7 @@
   {
    "cell_type": "code",
    "execution_count": 9,
-   "id": "04999981",
+   "id": "9b4ac5f7",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19491,7 +19491,7 @@
   {
    "cell_type": "code",
    "execution_count": 10,
-   "id": "0a62cbcf",
+   "id": "a9a07875",
    "metadata": {},
    "outputs": [
     {
@@ -19509,7 +19509,7 @@
   {
    "cell_type": "code",
    "execution_count": 11,
-   "id": "c388cffc",
+   "id": "8a3d39d8",
    "metadata": {},
    "outputs": [
     {
@@ -19536,7 +19536,7 @@
   {
    "cell_type": "code",
    "execution_count": 12,
-   "id": "6539221d",
+   "id": "934a4070",
    "metadata": {},
    "outputs": [
     {
@@ -19554,7 +19554,7 @@
   {
    "cell_type": "code",
    "execution_count": 13,
-   "id": "3e3bc229",
+   "id": "7f42a2b4",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19565,7 +19565,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "90ef7d2e",
+   "id": "9a504bc4",
    "metadata": {},
    "source": [
     "# Tokenizer"
@@ -19574,7 +19574,7 @@
   {
    "cell_type": "code",
    "execution_count": 14,
-   "id": "5dca790c",
+   "id": "0cec90b4",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19585,15 +19585,38 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
-   "id": "2a18f061",
+   "execution_count": 62,
+   "id": "dc9e79da",
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+      "Didn't find file ./tokenizer.json. We won't load it.\n",
+      "loading file ./vocab.json\n",
+      "loading file ./tokenizer_config.json\n",
+      "loading file ./added_tokens.json\n",
+      "loading file ./special_tokens_map.json\n",
+      "loading file None\n"
+     ]
+    },
+    {
+     "ename": "JSONDecodeError",
+     "evalue": "Expecting value: line 1 column 1 (char 0)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mJSONDecodeError\u001b[0m                           Traceback (most recent call last)",
+      "Input \u001b[0;32mIn [62]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m tokenizer \u001b[38;5;241m=\u001b[39m \u001b[43mWav2Vec2CTCTokenizer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_pretrained\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m./\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43munk_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m[UNK]\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpad_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m[PAD]\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mword_delimiter_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m|\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# './' load vocab.json in the current directory\u001b[39;00m\n\u001b[1;32m      2\u001b[0m feature_extractor \u001b[38;5;241m=\u001b[39m Wav2Vec2FeatureExtractor(feature_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m, sampling_rate\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m16000\u001b[39m, padding_value\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m0.0\u001b[39m, do_normalize\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, return_attention_mask\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)  \n\u001b[1;32m      3\u001b[0m processor \u001b[38;5;241m=\u001b[39m Wav2Vec2Processor(feature_extractor\u001b[38;5;241m=\u001b[39mfeature_extractor, tokenizer\u001b[38;5;241m=\u001b[39mtokenizer)\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1773\u001b[0m, in \u001b[0;36mPreTrainedTokenizerBase.from_pretrained\u001b[0;34m(cls, pretrained_model_name_or_path, *init_inputs, **kwargs)\u001b[0m\n\u001b[1;32m   1770\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m   1771\u001b[0m         logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mloading file \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile_path\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m from cache at \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresolved_vocab_files[file_id]\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m-> 1773\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_from_pretrained\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1774\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresolved_vocab_files\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1775\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpretrained_model_name_or_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1776\u001b[0m \u001b[43m    \u001b[49m\u001b[43minit_configuration\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1777\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_inputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1778\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_auth_token\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_auth_token\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1779\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcache_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcache_dir\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1780\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1781\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/tokenization_utils_base.py:1908\u001b[0m, in \u001b[0;36mPreTrainedTokenizerBase._from_pretrained\u001b[0;34m(cls, resolved_vocab_files, pretrained_model_name_or_path, init_configuration, use_auth_token, cache_dir, *init_inputs, **kwargs)\u001b[0m\n\u001b[1;32m   1906\u001b[0m \u001b[38;5;66;03m# Instantiate tokenizer.\u001b[39;00m\n\u001b[1;32m   1907\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1908\u001b[0m     tokenizer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_inputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minit_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1909\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m:\n\u001b[1;32m   1910\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\n\u001b[1;32m   1911\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUnable to load vocabulary from file. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1912\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease check that the provided vocabulary is accessible and not corrupted.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m   1913\u001b[0m     )\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/site-packages/transformers/models/wav2vec2/tokenization_wav2vec2.py:142\u001b[0m, in \u001b[0;36mWav2Vec2CTCTokenizer.__init__\u001b[0;34m(self, vocab_file, bos_token, eos_token, unk_token, pad_token, word_delimiter_token, do_lower_case, **kwargs)\u001b[0m\n\u001b[1;32m    139\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdo_lower_case \u001b[38;5;241m=\u001b[39m do_lower_case\n\u001b[1;32m    141\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(vocab_file, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mutf-8\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m vocab_handle:\n\u001b[0;32m--> 142\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencoder \u001b[38;5;241m=\u001b[39m \u001b[43mjson\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvocab_handle\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    143\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecoder \u001b[38;5;241m=\u001b[39m {v: k \u001b[38;5;28;01mfor\u001b[39;00m k, v \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencoder\u001b[38;5;241m.\u001b[39mitems()}\n\u001b[1;32m    145\u001b[0m \u001b[38;5;66;03m# make sure that tokens made of several\u001b[39;00m\n\u001b[1;32m    146\u001b[0m \u001b[38;5;66;03m# characters are not split at tokenization\u001b[39;00m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/__init__.py:293\u001b[0m, in \u001b[0;36mload\u001b[0;34m(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    274\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mload\u001b[39m(fp, \u001b[38;5;241m*\u001b[39m, \u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, object_hook\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, parse_float\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m    275\u001b[0m         parse_int\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, parse_constant\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, object_pairs_hook\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkw):\n\u001b[1;32m    276\u001b[0m     \u001b[38;5;124;03m\"\"\"Deserialize ``fp`` (a ``.read()``-supporting file-like object containing\u001b[39;00m\n\u001b[1;32m    277\u001b[0m \u001b[38;5;124;03m    a JSON document) to a Python object.\u001b[39;00m\n\u001b[1;32m    278\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    291\u001b[0m \u001b[38;5;124;03m    kwarg; otherwise ``JSONDecoder`` is used.\u001b[39;00m\n\u001b[1;32m    292\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 293\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    294\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mcls\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobject_hook\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobject_hook\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    295\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparse_float\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_float\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparse_int\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_int\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    296\u001b[0m \u001b[43m        \u001b[49m\u001b[43mparse_constant\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparse_constant\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mobject_pairs_hook\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobject_pairs_hook\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkw\u001b[49m\u001b[43m)\u001b[49m\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/__init__.py:357\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m    352\u001b[0m     \u001b[38;5;28;01mdel\u001b[39;00m kw[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mencoding\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m    354\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    355\u001b[0m         parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m    356\u001b[0m         parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 357\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    358\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    359\u001b[0m     \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m    332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode\u001b[39m(\u001b[38;5;28mself\u001b[39m, s, _w\u001b[38;5;241m=\u001b[39mWHITESPACE\u001b[38;5;241m.\u001b[39mmatch):\n\u001b[1;32m    333\u001b[0m     \u001b[38;5;124;03m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m    334\u001b[0m \u001b[38;5;124;03m    containing a JSON document).\u001b[39;00m\n\u001b[1;32m    335\u001b[0m \n\u001b[1;32m    336\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m     obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw_decode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43midx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m_w\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mend\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    338\u001b[0m     end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m    339\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n",
+      "File \u001b[0;32m/opt/conda/lib/python3.8/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m    353\u001b[0m     obj, end \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscan_once(s, idx)\n\u001b[1;32m    354\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mStopIteration\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExpecting value\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, err\u001b[38;5;241m.\u001b[39mvalue) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28mNone\u001b[39m\n\u001b[1;32m    356\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj, end\n",
+      "\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
      ]
     }
    ],
@@ -19606,7 +19629,7 @@
   {
    "cell_type": "code",
    "execution_count": 26,
-   "id": "4f1ff378",
+   "id": "61738038",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19623,7 +19646,7 @@
   {
    "cell_type": "code",
    "execution_count": 27,
-   "id": "1c8d8b0e",
+   "id": "4b72b9b8",
    "metadata": {},
    "outputs": [
     {
@@ -19663,7 +19686,7 @@
   {
    "cell_type": "code",
    "execution_count": 17,
-   "id": "4f5cbac0",
+   "id": "ccb4a36c",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19674,7 +19697,7 @@
   {
    "cell_type": "code",
    "execution_count": 18,
-   "id": "46a14b30",
+   "id": "cf9d1391",
    "metadata": {},
    "outputs": [
     {
@@ -19699,7 +19722,7 @@
   {
    "cell_type": "code",
    "execution_count": 19,
-   "id": "7b54ffa4",
+   "id": "57ea4c6f",
    "metadata": {},
    "outputs": [
     {
@@ -19746,7 +19769,7 @@
   {
    "cell_type": "code",
    "execution_count": 20,
-   "id": "0a51643b",
+   "id": "7cb9fd2a",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19768,7 +19791,7 @@
   {
    "cell_type": "code",
    "execution_count": 22,
-   "id": "e6a3137b",
+   "id": "42f2952f",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19779,7 +19802,7 @@
   {
    "cell_type": "code",
    "execution_count": 41,
-   "id": "ffe2e1e6",
+   "id": "fe093630",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19791,7 +19814,7 @@
   {
    "cell_type": "code",
    "execution_count": 25,
-   "id": "a1aec6d5",
+   "id": "a6efe782",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19851,7 +19874,7 @@
   {
    "cell_type": "code",
    "execution_count": 26,
-   "id": "1f73c038",
+   "id": "e82a3663",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19861,7 +19884,7 @@
   {
    "cell_type": "code",
    "execution_count": 27,
-   "id": "322220b4",
+   "id": "1df03ab8",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19872,7 +19895,7 @@
   {
    "cell_type": "code",
    "execution_count": 44,
-   "id": "19d15ad2",
+   "id": "8304f047",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19897,7 +19920,7 @@
   {
    "cell_type": "code",
    "execution_count": 45,
-   "id": "fa0dd3e4",
+   "id": "f92c9b4d",
    "metadata": {
     "collapsed": true,
     "jupyter": {
@@ -20048,7 +20071,7 @@
   {
    "cell_type": "code",
    "execution_count": 46,
-   "id": "6b01e8e0",
+   "id": "7f2dd147",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -20058,7 +20081,7 @@
   {
    "cell_type": "code",
    "execution_count": 47,
-   "id": "3fe63254",
+   "id": "3d27466c",
    "metadata": {},
    "outputs": [
     {
@@ -20095,7 +20118,7 @@
   {
    "cell_type": "code",
    "execution_count": 48,
-   "id": "745522a2",
+   "id": "014ac4c9",
    "metadata": {},
    "outputs": [
     {
@@ -20123,7 +20146,7 @@
   {
    "cell_type": "code",
    "execution_count": 49,
-   "id": "a2a214a5",
+   "id": "e6cb809a",
    "metadata": {
     "collapsed": true,
     "jupyter": {
@@ -20764,7 +20787,7 @@
   {
    "cell_type": "code",
    "execution_count": 57,
-   "id": "6cc9f33d",
+   "id": "57c2527b",
    "metadata": {},
    "outputs": [
     {
@@ -20784,7 +20807,7 @@
   {
    "cell_type": "code",
    "execution_count": 53,
-   "id": "72f2e951",
+   "id": "0211e267",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -20801,7 +20824,7 @@
   {
    "cell_type": "code",
    "execution_count": 54,
-   "id": "78d7353f",
+   "id": "62f6fd3e",
    "metadata": {},
    "outputs": [
     {
@@ -20819,52 +20842,54 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 58,
-   "id": "d9bb5fa1",
+   "execution_count": 60,
+   "id": "b050fb9f",
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Configuration saved in vitouphy/xls-r-300m-km/config.json\n",
-      "Model weights saved in vitouphy/xls-r-300m-km/pytorch_model.bin\n",
-      "Several commits (2) will be pushed upstream.\n",
-      "The progress bars may be unreliable.\n"
+      "/opt/conda/lib/python3.8/site-packages/huggingface_hub/hf_api.py:1001: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
+      "  warnings.warn(\n",
+      "Cloning https://huggingface.co/vitouphy/xls-r-300m-km into local empty directory.\n"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "65189fc5b517439b87208f0898179afd",
+       "model_id": "331db7acce774ee3b699aa82a0451092",
        "version_major": 2,
        "version_minor": 0
       },
       "text/plain": [
-       "Upload file pytorch_model.bin:   0%|          | 3.39k/1.18G [00:00<?, ?B/s]"
+       "Download file pytorch_model.bin:   0%|          | 3.47k/1.18G [00:00<?, ?B/s]"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "To https://huggingface.co/vitouphy/xls-r-300m-km\n",
-      "   e25c362..dff1f30  main -> main\n",
-      "\n"
-     ]
-    },
     {
      "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "db90465291c64e9f82988698d2473234",
+       "version_major": 2,
+       "version_minor": 0
+      },
       "text/plain": [
-       "'https://huggingface.co/vitouphy/xls-r-300m-km/commit/dff1f3008b5c2afbbbcab722e17fded4bf8f782b'"
+       "Clean file pytorch_model.bin:   0%|          | 1.00k/1.18G [00:00<?, ?B/s]"
       ]
      },
-     "execution_count": 58,
      "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Configuration saved in vitouphy/xls-r-300m-km/config.json\n",
+      "Model weights saved in vitouphy/xls-r-300m-km/pytorch_model.bin\n"
+     ]
     }
    ],
    "source": [
@@ -20873,8 +20898,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 59,
-   "id": "e56874c6",
+   "execution_count": 61,
+   "id": "9d7cb173",
    "metadata": {},
    "outputs": [
     {
@@ -20891,6 +20916,14 @@
    "source": [
     "trainer.save_model()"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8dc01ad4",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c93624168d989aabdf33bd32b0dbfa8b32857515b2a2f04190df98bd15ae4e61
+size 2991

vocab.json

@@ -1,5 +1 @@
-<<<<<<< HEAD
-{"\u1780": 1, "\u1781": 2, "\u1782": 3, "\u1783": 4, "\u1784": 5, "\u1785": 6, "\u1786": 7, "\u1787": 8, "\u1788": 9, "\u1789": 10, "\u178a": 11, "\u178b": 12, "\u178c": 13, "\u178d": 14, "\u178e": 15, "\u178f": 16, "\u1790": 17, "\u1791": 18, "\u1792": 19, "\u1793": 20, "\u1794": 21, "\u1795": 22, "\u1796": 23, "\u1797": 24, "\u1798": 25, "\u1799": 26, "\u179a": 27, "\u179b": 28, "\u179c": 29, "\u179f": 30, "\u17a0": 31, "\u17a1": 32, "\u17a2": 33, "\u17a5": 34, "\u17a7": 35, "\u17aa": 36, "\u17ab": 37, "\u17ac": 38, "\u17ad": 39, "\u17ae": 40, "\u17af": 41, "\u17b1": 42, "\u17b6": 43, "\u17b7": 44, "\u17b8": 45, "\u17b9": 46, "\u17ba": 47, "\u17bb": 48, "\u17bc": 49, "\u17bd": 50, "\u17be": 51, "\u17bf": 52, "\u17c0": 53, "\u17c1": 54, "\u17c2": 55, "\u17c3": 56, "\u17c4": 57, "\u17c5": 58, "\u17c6": 59, "\u17c7": 60, "\u17c8": 61, "\u17c9": 62, "\u17ca": 63, "\u17cb": 64, "\u17cc": 65, "\u17cd": 66, "\u17ce": 67, "\u17cf": 68, "\u17d0": 69, "\u17d2": 70, "|": 0, "[UNK]": 71, "[PAD]": 72}
-=======
-{"ក": 1, "ខ": 2, "គ": 3, "ឃ": 4, "ង": 5, "ច": 6, "ឆ": 7, "ជ": 8, "ឈ": 9, "ញ": 10, "ដ": 11, "ឋ": 12, "ឌ": 13, "ឍ": 14, "ណ": 15, "ត": 16, "ថ": 17, "ទ": 18, "ធ": 19, "ន": 20, "ប": 21, "ផ": 22, "ព": 23, "ភ": 24, "ម": 25, "យ": 26, "រ": 27, "ល": 28, "វ": 29, "ស": 30, "ហ": 31, "ឡ": 32, "អ": 33, "ឥ": 34, "ឧ": 35, "ឪ": 36, "ឫ": 37, "ឬ": 38, "ឭ": 39, "ឮ": 40, "ឯ": 41, "ឱ": 42, "ា": 43, "ិ": 44, "ី": 45, "ឹ": 46, "ឺ": 47, "ុ": 48, "ូ": 49, "ួ": 50, "ើ": 51, "ឿ": 52, "ៀ": 53, "េ": 54, "ែ": 55, "ៃ": 56, "ោ": 57, "ៅ": 58, "ំ": 59, "ះ": 60, "ៈ": 61, "៉": 62, "៊": 63, "់": 64, "៌": 65, "៍": 66, "៎": 67, "៏": 68, "័": 69, "្": 70, "|": 0, "[UNK]": 71, "[PAD]": 72}
->>>>>>> dff1f3008b5c2afbbbcab722e17fded4bf8f782b
+{"ក": 1, "ខ": 2, "គ": 3, "ឃ": 4, "ង": 5, "ច": 6, "ឆ": 7, "ជ": 8, "ឈ": 9, "ញ": 10, "ដ": 11, "ឋ": 12, "ឌ": 13, "ឍ": 14, "ណ": 15, "ត": 16, "ថ": 17, "ទ": 18, "ធ": 19, "ន": 20, "ប": 21, "ផ": 22, "ព": 23, "ភ": 24, "ម": 25, "យ": 26, "រ": 27, "ល": 28, "វ": 29, "ស": 30, "ហ": 31, "ឡ": 32, "អ": 33, "ឥ": 34, "ឧ": 35, "ឪ": 36, "ឫ": 37, "ឬ": 38, "ឭ": 39, "ឮ": 40, "ឯ": 41, "ឱ": 42, "ា": 43, "ិ": 44, "ី": 45, "ឹ": 46, "ឺ": 47, "ុ": 48, "ូ": 49, "ួ": 50, "ើ": 51, "ឿ": 52, "ៀ": 53, "េ": 54, "ែ": 55, "ៃ": 56, "ោ": 57, "ៅ": 58, "ំ": 59, "ះ": 60, "ៈ": 61, "៉": 62, "៊": 63, "់": 64, "៌": 65, "៍": 66, "៎": 67, "៏": 68, "័": 69, "្": 70, "|": 0, "[UNK]": 71, "[PAD]": 72}