Upload evaluate-gpu.ipynb

evaluate-gpu.ipynb

@@ -158,19 +158,22 @@
     {
      "data": {
       "text/plain": [
-       "[('mandarin-test/460.wav', 'ting qi lai you dian xiang zai chang de na zhong'),\n",
-       " ('mandarin-test/256.wav', 'zai jia hao wu liao a'),\n",
-       " ('singlish-test/2169.wav', 'controlling our environment is important'),\n",
-       " ('mandarin-test/400.wav', 'bo fang gu zheng de ge qu'),\n",
-       " ('singlish-test/1001.wav', 'because they are the one that badly need it'),\n",
-       " ('singlish-test/4.wav',\n",
-       "  'rescuers who used what appeared to be makeshift stretchers to carry the injured'),\n",
-       " ('singlish-test/392.wav', 'i attached a mirror to my closet door'),\n",
-       " ('singlish-test/2563.wav', 'do you know the answer'),\n",
-       " ('singlish-test/799.wav',\n",
-       "  'this kind of packaging can pose a danger to animals'),\n",
-       " ('singlish-test/1165.wav',\n",
-       "  'a lot of parents ive spoken to say they dont have the luxury to do that')]"
+       "[('malay-test/101.wav',\n",
+       "  'kenapa jews dan israelis mengejek iranian bukan agama lebih'),\n",
+       " ('singlish-test/978.wav',\n",
+       "  'but in the olympics time does not really matter what matters is winning'),\n",
+       " ('singlish-test/1189.wav',\n",
+       "  'and if joseph schooling was born with a tinier hand he might not have hit the wall first'),\n",
+       " ('singlish-test/1774.wav', 'melissa passed her number to the young man'),\n",
+       " ('singlish-test/21.wav',\n",
+       "  'but he really wanted to make satay and hung around satay sellers and memorized their ingredients'),\n",
+       " ('singlish-test/2164.wav', 'just check out their coverage'),\n",
+       " ('malay-test/397.wav', 'budaya cocorico french culture'),\n",
+       " ('mandarin-test/359.wav', 'xi shou jian deng da kai'),\n",
+       " ('singlish-test/1107.wav',\n",
+       "  'the party had a cool vibe but was for guests only'),\n",
+       " ('singlish-test/285.wav',\n",
+       "  'but in smaller towns there are not so many places and that is where they are most needed')]"
       ]
      },
      "execution_count": 8,
@@ -249,7 +252,7 @@
    "outputs": [],
    "source": [
     "model = AutoModelForCTC.from_pretrained(\n",
-    "    './wav2vec2-mixed-v3/checkpoint-55000',\n",
+    "    './wav2vec2-mixed-v3/checkpoint-97000',\n",
     "    ctc_loss_reduction=\"mean\",\n",
     "    pad_token_id=tokenizer.pad_token_id,\n",
     "    vocab_size=len(tokenizer),\n",
@@ -300,10 +303,10 @@
     {
      "data": {
       "text/plain": [
-       "['ting qi lai you dian xiang zai chang de na zhong',\n",
-       " 'zai jia hao wu liao wa',\n",
-       " 'controlling our environment is important',\n",
-       " 'bo fang gu zheng de ge qu']"
+       "['kenapa jius dan israelis mengejik iranian bukan agama lebih',\n",
+       " 'but in the olympics time does not really matter what matters is winning',\n",
+       " 'and if joseph schooling was born with a tinier hand he might not have hit the world first',\n",
+       " 'melissa passed her number to the young man']"
       ]
      },
      "execution_count": 14,
@@ -359,10 +362,10 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0 ting qi lai you dian xiang zai chang de na zhong\n",
-      "1 zai jia hao wu liao wa\n",
-      "2 controlling our environment is important\n",
-      "3 bo fang gu zheng de ge qu\n"
+      "0 kenapa jus dan israelis mengejek iranian bukan agama lebih\n",
+      "1 but in the olympics time does not really matter what matters is winning\n",
+      "2 and if joseph schooling was born with a tinier hand he might not have hit the world first\n",
+      "3 melissa passed her number to the young man\n"
      ]
     }
    ],
@@ -382,10 +385,10 @@
     {
      "data": {
       "text/plain": [
-       "['ting qi lai you dian xiang zai chang de na zhong',\n",
-       " 'zai jia hao wu liao a',\n",
-       " 'controlling our environment is important',\n",
-       " 'bo fang gu zheng de ge qu']"
+       "['kenapa jews dan israelis mengejek iranian bukan agama lebih',\n",
+       " 'but in the olympics time does not really matter what matters is winning',\n",
+       " 'and if joseph schooling was born with a tinier hand he might not have hit the wall first',\n",
+       " 'melissa passed her number to the young man']"
       ]
      },
      "execution_count": 18,
@@ -440,7 +443,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 1240/1240 [04:27<00:00,  4.63it/s]\n"
+      "100%|██████████| 1240/1240 [07:43<00:00,  2.67it/s]\n"
      ]
     }
    ],
@@ -478,20 +481,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 26,
    "id": "6c6ce8ef",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(0.14251665517797765,\n",
-       " 0.05082346216269688,\n",
-       " 0.10380217528405207,\n",
-       " 0.042868860764264445)"
+       "(0.12446419219809059,\n",
+       " 0.04363189219453221,\n",
+       " 0.09152993800218129,\n",
+       " 0.03621180629932558)"
       ]
      },
-     "execution_count": 21,
+     "execution_count": 26,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -502,7 +505,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 27,
    "id": "cf53914e",
    "metadata": {},
    "outputs": [],
@@ -514,20 +517,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 28,
    "id": "b1558987",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(0.21723938552369926,\n",
-       " 0.05027226867066105,\n",
-       " 0.13593624603428525,\n",
-       " 0.03601546154013878)"
+       "(0.22565751242221832,\n",
+       " 0.053659683623049854,\n",
+       " 0.14256712242006359,\n",
+       " 0.036930421149001316)"
       ]
      },
-     "execution_count": 23,
+     "execution_count": 28,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -538,20 +541,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 29,
    "id": "f340cde7",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(0.1331819722523124,\n",
-       " 0.05161275767676772,\n",
-       " 0.09859626021111582,\n",
-       " 0.04419848182804781)"
+       "(0.10734402150682842,\n",
+       " 0.04174804195104746,\n",
+       " 0.08103191123663189,\n",
+       " 0.03538238462620066)"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 29,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -562,20 +565,20 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 30,
    "id": "cbc2539f",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/plain": [
-       "(0.10382926344585862,\n",
-       " 0.04690941391603209,\n",
-       " 0.09411065398455744,\n",
-       " 0.0436573568867001)"
+       "(0.09817787449869257,\n",
+       " 0.04211892733885779,\n",
+       " 0.08913415903511501,\n",
+       " 0.040151154521006656)"
       ]
      },
-     "execution_count": 25,
+     "execution_count": 30,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -586,27 +589,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 31,
    "id": "4c543d0c",
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/utils/_deprecation.py:39: FutureWarning: Pass token='wav2vec2-xls-r-300m-mixed' as keyword args. From version 0.7 passing these as positional arguments will result in an error\n",
-      "  warnings.warn(\n",
-      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/hf_api.py:79: FutureWarning: `name` and `organization` input arguments are deprecated and will be removed in v0.7. Pass `repo_id` instead.\n",
-      "  warnings.warn(\n",
-      "/home/ubuntu/.local/lib/python3.8/site-packages/huggingface_hub/hf_api.py:596: FutureWarning: `create_repo` now takes `token` as an optional positional argument. Be sure to adapt your code!\n",
-      "  warnings.warn(\n",
-      "Cloning https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed into local empty directory.\n"
-     ]
-    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "10c88b815c83447b9d04f297f54fe1d9",
+       "model_id": "551516109d6a418b95be6884422d853e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -624,17 +614,17 @@
       "remote: Enforcing permissions...        \n",
       "remote: Allowed refs: all        \n",
       "To https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed\n",
-      "   33df917..7044629  main -> main\n",
+      "   3f5d181..7799685  main -> main\n",
       "\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "'https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed/commit/7044629625df853dec50f463f6b794afe61d391f'"
+       "'https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed/commit/77996855b40213396051061d8e23b67c2616e614'"
       ]
      },
-     "execution_count": 26,
+     "execution_count": 31,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -645,7 +635,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 32,
    "id": "05ec385e",
    "metadata": {},
    "outputs": [
@@ -653,24 +643,24 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "2022-06-01 09:29:07.148431: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.191068: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.192882: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.194967: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
+      "2022-06-01 19:14:20.564262: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.603610: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.605395: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.607506: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
       "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
-      "2022-06-01 09:29:07.196435: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.197071: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.197672: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.199082: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.199700: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.200318: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
-      "2022-06-01 09:29:07.201032: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
-      "2022-06-01 09:29:07.201159: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 17325 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
+      "2022-06-01 19:14:20.609495: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.610833: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.612207: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.615738: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.617302: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.618707: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
+      "2022-06-01 19:14:20.620281: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:39] Overriding allow_growth setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
+      "2022-06-01 19:14:20.620394: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 17119 MB memory:  -> device: 0, name: NVIDIA GeForce RTX 3090 Ti, pci bus id: 0000:01:00.0, compute capability: 8.6\n",
       "\n",
       "TFWav2Vec2ForCTC has backpropagation operations that are NOT supported on CPU. If you wish to train/fine-tine this model, you need a GPU or a TPU\n",
-      "2022-06-01 09:29:09.085113: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8100\n",
-      "2022-06-01 09:29:09.930887: I tensorflow/core/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
-      "2022-06-01 09:29:10.708302: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
+      "2022-06-01 19:14:22.857691: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8100\n",
+      "2022-06-01 19:14:24.326073: I tensorflow/core/platform/default/subprocess.cc:304] Start cannot spawn child process: No such file or directory\n",
+      "2022-06-01 19:14:25.725870: I tensorflow/stream_executor/cuda/cuda_blas.cc:1760] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n",
       "All PyTorch model weights were used when initializing TFWav2Vec2ForCTC.\n",
       "\n",
       "All the weights of TFWav2Vec2ForCTC were initialized from the PyTorch model.\n",
@@ -680,7 +670,7 @@
    ],
    "source": [
     "model_tf = TFWav2Vec2ForCTC.from_pretrained(\n",
-    "    './wav2vec2-mixed-v3/checkpoint-55000',\n",
+    "    './wav2vec2-mixed-v3/checkpoint-97000',\n",
     "    ctc_loss_reduction=\"mean\",\n",
     "    pad_token_id=tokenizer.pad_token_id,\n",
     "    vocab_size=len(tokenizer),\n",
@@ -690,21 +680,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 33,
    "id": "e0f3f749",
    "metadata": {},
    "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "2022-06-01 09:29:38.885075: W tensorflow/core/framework/cpu_allocator_impl.cc:80] Allocation of 33554432 exceeds 10% of free system memory.\n"
-     ]
-    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f70276237419473091977e7dcd4da591",
+       "model_id": "a0e5eeee5bf4499da3d5f4adbd5bfd4f",
        "version_major": 2,
        "version_minor": 0
       },
@@ -722,17 +705,17 @@
       "remote: Enforcing permissions...        \n",
       "remote: Allowed refs: all        \n",
       "To https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed\n",
-      "   7044629..86e9f45  main -> main\n",
+      "   7799685..0b9b0fb  main -> main\n",
       "\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "'https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed/commit/86e9f450fa80b3f51175f04f694b35f342a6a09e'"
+       "'https://huggingface.co/mesolitica/wav2vec2-xls-r-300m-mixed/commit/0b9b0fb66dc68a4f71ab793274fb28df9f19764f'"
       ]
      },
-     "execution_count": 28,
+     "execution_count": 33,
      "metadata": {},
      "output_type": "execute_result"
     }