Training in progress, step 4500

.ipynb_checkpoints/requirements-checkpoint.txt

@@ -6,3 +6,5 @@ jiwer~=2.3.0
 soundfile~=0.10.3
 transformers~=4.16.2
 datasets~=1.18.3
+pyctcdecode
+https://github.com/kpu/kenlm/archive/master.zip

.ipynb_checkpoints/with_ngram_LM-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63e0de4a374839516480d3ebd15f1a566884c85df3e200ae5ef67ad87896d6ba
+oid sha256:184eeda764638171e3397ddf0afb0061278cc440cdb956d065bdeb73bbfe8086
 size 1262058993

requirements.txt

@@ -6,3 +6,5 @@ jiwer~=2.3.0
 soundfile~=0.10.3
 transformers~=4.16.2
 datasets~=1.18.3
+pyctcdecode
+https://github.com/kpu/kenlm/archive/master.zip

runs/Feb06_18-52-28_job-0a778896-a7e2-46e9-bcf5-016f91f242cf/events.out.tfevents.1644173767.job-0a778896-a7e2-46e9-bcf5-016f91f242cf.841782.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:13ca3a32617dcb433d6774ca3ceadace283e127985b36df8d5d0b281378c3862
-size 13946
+oid sha256:e638063e5f5f1ea9473830277ba79d9fe975573b46c8800ded9eb193132d9a90
+size 15096

with_ngram_LM.ipynb

@@ -0,0 +1,339 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "7ed2fa91",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/workspace/xls-r-uzbek-cv8\n"
+     ]
+    }
+   ],
+   "source": [
+    "%cd ~/xls-r-uzbek-cv8"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "672bdf2b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[33mWARNING: Ignoring invalid distribution -ransformers (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ransformers (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0mCollecting https://github.com/kpu/kenlm/archive/master.zip (from -r requirements.txt (line 10))\n",
+      "  Using cached https://github.com/kpu/kenlm/archive/master.zip (541 kB)\n",
+      "  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25hRequirement already satisfied: unidecode in /opt/conda/lib/python3.8/site-packages (from -r requirements.txt (line 1)) (1.3.2)\n",
+      "Collecting tensorboard\n",
+      "  Using cached tensorboard-2.8.0-py3-none-any.whl (5.8 MB)\n",
+      "Requirement already satisfied: torch in /opt/conda/lib/python3.8/site-packages (from -r requirements.txt (line 3)) (1.10.2)\n",
+      "Requirement already satisfied: torchaudio in /opt/conda/lib/python3.8/site-packages (from -r requirements.txt (line 4)) (0.10.2)\n",
+      "Requirement already satisfied: jiwer~=2.3.0 in /opt/conda/lib/python3.8/site-packages (from -r requirements.txt (line 5)) (2.3.0)\n",
+      "Requirement already satisfied: soundfile~=0.10.3 in /opt/conda/lib/python3.8/site-packages (from -r requirements.txt (line 6)) (0.10.3.post1)\n",
+      "Collecting transformers~=4.16.2\n",
+      "  Using cached transformers-4.16.2-py3-none-any.whl (3.5 MB)\n",
+      "Collecting datasets~=1.18.3\n",
+      "  Using cached datasets-1.18.3-py3-none-any.whl (311 kB)\n",
+      "Requirement already satisfied: pyctcdecode in /opt/conda/lib/python3.8/site-packages (from -r requirements.txt (line 9)) (0.3.0)\n",
+      "Requirement already satisfied: protobuf>=3.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (3.19.4)\n",
+      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (1.8.1)\n",
+      "Collecting google-auth-oauthlib<0.5,>=0.4.1\n",
+      "  Using cached google_auth_oauthlib-0.4.6-py2.py3-none-any.whl (18 kB)\n",
+      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (2.6.0)\n",
+      "Requirement already satisfied: numpy>=1.12.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (1.19.2)\n",
+      "Requirement already satisfied: setuptools>=41.0.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (50.3.1.post20201107)\n",
+      "Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (2.24.0)\n",
+      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (3.3.6)\n",
+      "Requirement already satisfied: grpcio>=1.24.3 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (1.43.0)\n",
+      "Requirement already satisfied: wheel>=0.26 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (0.35.1)\n",
+      "Requirement already satisfied: absl-py>=0.4 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (1.0.0)\n",
+      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (2.0.2)\n",
+      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.8/site-packages (from tensorboard->-r requirements.txt (line 2)) (0.6.1)\n",
+      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.8/site-packages (from torch->-r requirements.txt (line 3)) (4.0.1)\n",
+      "Requirement already satisfied: python-Levenshtein==0.12.2 in /opt/conda/lib/python3.8/site-packages (from jiwer~=2.3.0->-r requirements.txt (line 5)) (0.12.2)\n",
+      "Requirement already satisfied: cffi>=1.0 in /opt/conda/lib/python3.8/site-packages (from soundfile~=0.10.3->-r requirements.txt (line 6)) (1.14.3)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (3.0.12)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (2022.1.18)\n",
+      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (0.0.47)\n",
+      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (21.3)\n",
+      "Requirement already satisfied: tokenizers!=0.11.3,>=0.10.1 in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (0.11.4)\n",
+      "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (0.4.0)\n",
+      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (5.4.1)\n",
+      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.8/site-packages (from transformers~=4.16.2->-r requirements.txt (line 7)) (4.62.3)\n",
+      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (0.70.12.2)\n",
+      "Requirement already satisfied: dill in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (0.3.4)\n",
+      "Requirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (1.4.0)\n",
+      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (3.8.1)\n",
+      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (2.0.2)\n",
+      "Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (6.0.1)\n",
+      "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.8/site-packages (from datasets~=1.18.3->-r requirements.txt (line 8)) (2022.1.0)\n",
+      "Requirement already satisfied: pygtrie<3.0,>=2.1 in /opt/conda/lib/python3.8/site-packages (from pyctcdecode->-r requirements.txt (line 9)) (2.4.2)\n",
+      "Requirement already satisfied: hypothesis<7,>=6.14 in /opt/conda/lib/python3.8/site-packages (from pyctcdecode->-r requirements.txt (line 9)) (6.36.1)\n",
+      "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from absl-py>=0.4->tensorboard->-r requirements.txt (line 2)) (1.15.0)\n",
+      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi>=1.0->soundfile~=0.10.3->-r requirements.txt (line 6)) (2.20)\n",
+      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 2)) (4.8)\n",
+      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 2)) (5.0.0)\n",
+      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 2)) (0.2.8)\n",
+      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r requirements.txt (line 2)) (1.3.1)\n",
+      "Requirement already satisfied: sortedcontainers<3.0.0,>=2.1.0 in /opt/conda/lib/python3.8/site-packages (from hypothesis<7,>=6.14->pyctcdecode->-r requirements.txt (line 9)) (2.4.0)\n",
+      "Requirement already satisfied: attrs>=19.2.0 in /opt/conda/lib/python3.8/site-packages (from hypothesis<7,>=6.14->pyctcdecode->-r requirements.txt (line 9)) (21.4.0)\n",
+      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.8/site-packages (from markdown>=2.6.8->tensorboard->-r requirements.txt (line 2)) (4.10.1)\n",
+      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=20.0->transformers~=4.16.2->-r requirements.txt (line 7)) (3.0.7)\n",
+      "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->-r requirements.txt (line 2)) (3.0.4)\n",
+      "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->-r requirements.txt (line 2)) (2.10)\n",
+      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->-r requirements.txt (line 2)) (1.25.11)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard->-r requirements.txt (line 2)) (2020.12.5)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets~=1.18.3->-r requirements.txt (line 8)) (1.7.2)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets~=1.18.3->-r requirements.txt (line 8)) (4.0.2)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets~=1.18.3->-r requirements.txt (line 8)) (1.2.0)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets~=1.18.3->-r requirements.txt (line 8)) (6.0.2)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets~=1.18.3->-r requirements.txt (line 8)) (1.3.0)\n",
+      "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets~=1.18.3->-r requirements.txt (line 8)) (2.0.10)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets~=1.18.3->-r requirements.txt (line 8)) (2.8.2)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets~=1.18.3->-r requirements.txt (line 8)) (2021.1)\n",
+      "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers~=4.16.2->-r requirements.txt (line 7)) (1.1.0)\n",
+      "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers~=4.16.2->-r requirements.txt (line 7)) (8.0.3)\n",
+      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.8/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard->-r requirements.txt (line 2)) (3.7.0)\n",
+      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard->-r requirements.txt (line 2)) (0.4.8)\n",
+      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard->-r requirements.txt (line 2)) (3.2.0)\n",
+      "Building wheels for collected packages: kenlm\n",
+      "  Building wheel for kenlm (setup.py) ... \u001b[?25ldone\n",
+      "\u001b[?25h  Created wheel for kenlm: filename=kenlm-0.0.0-cp38-cp38-linux_x86_64.whl size=2348591 sha256=d5c8e5430d89f59ddde39bc78aec471c1e66ef43b6cde792711b2e97d7b8b9dc\n",
+      "  Stored in directory: /tmp/pip-ephem-wheel-cache-hhcfnszu/wheels/ff/08/4e/a3ddc0e786e0f3c1fcd2e7a82c4324c02fc3ae2638471406d2\n",
+      "Successfully built kenlm\n",
+      "\u001b[33mWARNING: Ignoring invalid distribution -ransformers (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0mInstalling collected packages: kenlm, transformers, google-auth-oauthlib, tensorboard, datasets\n",
+      "\u001b[33m  WARNING: The script transformers-cli is installed in '/workspace/.local/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33m  WARNING: The script google-oauthlib-tool is installed in '/workspace/.local/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33m  WARNING: The script tensorboard is installed in '/workspace/.local/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33m  WARNING: The script datasets-cli is installed in '/workspace/.local/bin' which is not on PATH.\n",
+      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
+      "\u001b[0mSuccessfully installed datasets-1.18.3 google-auth-oauthlib-0.4.6 kenlm-0.0.0 tensorboard-2.8.0 transformers-4.16.2\n",
+      "\u001b[33mWARNING: Ignoring invalid distribution -ransformers (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -ip (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n",
+      "\u001b[0m"
+     ]
+    }
+   ],
+   "source": [
+    "!python -m pip install -r requirements.txt --user"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "722882a2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoFeatureExtractor, AutoTokenizer, pipeline\n",
+    "from datasets import Audio, Dataset, DatasetDict, load_dataset, load_metric\n",
+    "\n",
+    "import re\n",
+    "import string\n",
+    "import unidecode"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "e288d5c0",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Reusing dataset common_voice (/workspace/.cache/huggingface/datasets/mozilla-foundation___common_voice/uz/8.0.0/b8bc4d453193c06a43269b46cd87f075c70f152ac963b7f28f7a2760c45ec3e8)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a8aad37a859241ff81ac932edc204bf8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/5 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['client_id', 'path', 'audio', 'sentence', 'up_votes', 'down_votes', 'age', 'gender', 'accent', 'locale', 'segment'],\n",
+       "        num_rows: 39456\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['client_id', 'path', 'audio', 'sentence', 'up_votes', 'down_votes', 'age', 'gender', 'accent', 'locale', 'segment'],\n",
+       "        num_rows: 11598\n",
+       "    })\n",
+       "    validation: Dataset({\n",
+       "        features: ['client_id', 'path', 'audio', 'sentence', 'up_votes', 'down_votes', 'age', 'gender', 'accent', 'locale', 'segment'],\n",
+       "        num_rows: 10849\n",
+       "    })\n",
+       "    other: Dataset({\n",
+       "        features: ['client_id', 'path', 'audio', 'sentence', 'up_votes', 'down_votes', 'age', 'gender', 'accent', 'locale', 'segment'],\n",
+       "        num_rows: 119461\n",
+       "    })\n",
+       "    invalidated: Dataset({\n",
+       "        features: ['client_id', 'path', 'audio', 'sentence', 'up_votes', 'down_votes', 'age', 'gender', 'accent', 'locale', 'segment'],\n",
+       "        num_rows: 11276\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset_dict = load_dataset(\"mozilla-foundation/common_voice_8_0\", \"uz\", use_auth_token=True)\n",
+    "dataset_dict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "id": "86031b96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "chars_to_ignore_regex=f\"[{re.escape(string.punctuation)}]\" \n",
+    "\n",
+    "def remove_special_characters(batch):\n",
+    "    batch[\"text\"] = re.sub(\n",
+    "        chars_to_ignore_regex, \n",
+    "        \"\", \n",
+    "        re.sub(\"['`´]\", \"’\",   # elsewhere probably meant as glottal stop\n",
+    "               re.sub(\"([og])['`´]\", \"\\g<1>‘\",  # after o/g indicate modified char\n",
+    "                      unidecode.unidecode(batch[\"sentence\"]).lower()\n",
+    "                     )\n",
+    "              )\n",
+    "    ) + \" \"\n",
+    "    return batch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "id": "c2b27b8e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4b8d2f0df8ea46bdaee2c94996583c5e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "0ex [00:00, ?ex/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "dataset = dataset_dict[\"train\"].map(remove_special_characters, remove_columns=dataset_dict[\"train\"].column_names)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "02f89e8e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "      0  244494 2030240 uz_cv8_train.txt\n"
+     ]
+    }
+   ],
+   "source": [
+    "text_data = \"uz_cv8_train.txt\"\n",
+    "with open(text_data, \"w\") as fs:\n",
+    "    fs.write(\" \".join(dataset[\"text\"]))\n",
+    "\n",
+    "!wc $text_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "27e4f250",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "wc: invalid option -- 'a'\n",
+      "Try 'wc --help' for more information.\n"
+     ]
+    }
+   ],
+   "source": [
+    "!wc -a $text_data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e0238868",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}