diff --git "a/train_tr.ipynb" "b/train_tr.ipynb" new file mode 100644--- /dev/null +++ "b/train_tr.ipynb" @@ -0,0 +1,1781 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3fa7516b", + "metadata": { + "collapsed": true, + "jupyter": { + "outputs_hidden": true + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting datasets==1.13.3\n", + " Using cached datasets-1.13.3-py3-none-any.whl (287 kB)\n", + "Requirement already satisfied: pandas in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (1.4.0)\n", + "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (3.8.1)\n", + "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (2.24.0)\n", + "Requirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (6.0.1)\n", + "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (1.19.2)\n", + "Requirement already satisfied: dill in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (0.3.4)\n", + "Requirement already satisfied: fsspec[http]>=2021.05.0 in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (2022.1.0)\n", + "Collecting huggingface-hub<0.1.0,>=0.0.19\n", + " Using cached huggingface_hub-0.0.19-py3-none-any.whl (56 kB)\n", + "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (4.62.3)\n", + "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (0.70.12.2)\n", + "Requirement already satisfied: xxhash in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (2.0.2)\n", + "Requirement already satisfied: packaging in /opt/conda/lib/python3.8/site-packages (from datasets==1.13.3) (21.3)\n", + "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<0.1.0,>=0.0.19->datasets==1.13.3) (5.4.1)\n", + "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<0.1.0,>=0.0.19->datasets==1.13.3) (3.0.12)\n", + "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.8/site-packages (from huggingface-hub<0.1.0,>=0.0.19->datasets==1.13.3) (4.0.1)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging->datasets==1.13.3) (3.0.7)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.13.3) (3.0.4)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.13.3) (1.25.11)\n", + "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.13.3) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->datasets==1.13.3) (2020.12.5)\n", + "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (21.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (1.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (6.0.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (1.3.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (1.7.2)\n", + "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (2.0.10)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.8/site-packages (from aiohttp->datasets==1.13.3) (4.0.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets==1.13.3) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.8/site-packages (from pandas->datasets==1.13.3) (2021.1)\n", + "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.8.1->pandas->datasets==1.13.3) (1.15.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Installing collected packages: huggingface-hub, datasets\n", + " Attempting uninstall: huggingface-hub\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Found existing installation: huggingface-hub 0.4.0\n", + " Uninstalling huggingface-hub-0.4.0:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ERROR: Could not install packages due to an OSError: [Errno 13] Permission denied: 'INSTALLER'\n", + "Consider using the `--user` option or check the permissions.\n", + "\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: You are using pip version 21.3.1; however, version 22.0.2 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting transformers==4.11.3\n", + " Using cached transformers-4.11.3-py3-none-any.whl (2.9 MB)\n", + "Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (2.24.0)\n", + "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (0.0.47)\n", + "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (1.19.2)\n", + "Collecting tokenizers<0.11,>=0.10.1\n", + " Using cached tokenizers-0.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n", + "Requirement already satisfied: huggingface-hub>=0.0.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (0.4.0)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (21.3)\n", + "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (4.62.3)\n", + "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (3.0.12)\n", + "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (2022.1.18)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.8/site-packages (from transformers==4.11.3) (5.4.1)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.8/site-packages (from huggingface-hub>=0.0.17->transformers==4.11.3) (4.0.1)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=20.0->transformers==4.11.3) (3.0.7)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.11.3) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.11.3) (2.10)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.11.3) (1.25.11)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests->transformers==4.11.3) (2020.12.5)\n", + "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.11.3) (1.15.0)\n", + "Requirement already satisfied: click in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.11.3) (8.0.3)\n", + "Requirement already satisfied: joblib in /opt/conda/lib/python3.8/site-packages (from sacremoses->transformers==4.11.3) (1.1.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Installing collected packages: tokenizers, transformers\n", + " Attempting uninstall: tokenizers\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Found existing installation: tokenizers 0.11.4\n", + " Uninstalling tokenizers-0.11.4:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ERROR: Could not install packages due to an OSError: [Errno 13] Permission denied: 'INSTALLER'\n", + "Consider using the `--user` option or check the permissions.\n", + "\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: You are using pip version 21.3.1; however, version 22.0.2 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting huggingface_hub==0.1\n", + " Using cached huggingface_hub-0.1.0-py3-none-any.whl (59 kB)\n", + "Requirement already satisfied: tqdm in /opt/conda/lib/python3.8/site-packages (from huggingface_hub==0.1) (4.62.3)\n", + "Requirement already satisfied: requests in /opt/conda/lib/python3.8/site-packages (from huggingface_hub==0.1) (2.24.0)\n", + "Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.8/site-packages (from huggingface_hub==0.1) (21.3)\n", + "Requirement already satisfied: filelock in /opt/conda/lib/python3.8/site-packages (from huggingface_hub==0.1) (3.0.12)\n", + "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.8/site-packages (from huggingface_hub==0.1) (4.0.1)\n", + "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.8/site-packages (from huggingface_hub==0.1) (5.4.1)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=20.9->huggingface_hub==0.1) (3.0.7)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface_hub==0.1) (1.25.11)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface_hub==0.1) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface_hub==0.1) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests->huggingface_hub==0.1) (2020.12.5)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Installing collected packages: huggingface-hub\n", + " Attempting uninstall: huggingface-hub\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + " WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Found existing installation: huggingface-hub 0.4.0\n", + " Uninstalling huggingface-hub-0.4.0:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ERROR: Could not install packages due to an OSError: [Errno 13] Permission denied: 'INSTALLER'\n", + "Consider using the `--user` option or check the permissions.\n", + "\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: You are using pip version 21.3.1; however, version 22.0.2 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: torchaudio in /opt/conda/lib/python3.8/site-packages (0.10.2)\n", + "Requirement already satisfied: torch==1.10.2 in /opt/conda/lib/python3.8/site-packages (from torchaudio) (1.10.2)\n", + "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.8/site-packages (from torch==1.10.2->torchaudio) (4.0.1)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: You are using pip version 21.3.1; however, version 22.0.2 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: librosa in /opt/conda/lib/python3.8/site-packages (0.8.1)\n", + "Requirement already satisfied: joblib>=0.14 in /opt/conda/lib/python3.8/site-packages (from librosa) (1.1.0)\n", + "Requirement already satisfied: resampy>=0.2.2 in /opt/conda/lib/python3.8/site-packages (from librosa) (0.2.2)\n", + "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (21.3)\n", + "Requirement already satisfied: numpy>=1.15.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (1.19.2)\n", + "Requirement already satisfied: decorator>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (4.4.2)\n", + "Requirement already satisfied: audioread>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (2.1.9)\n", + "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (1.0.2)\n", + "Requirement already satisfied: numba>=0.43.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (0.55.1)\n", + "Requirement already satisfied: soundfile>=0.10.2 in /opt/conda/lib/python3.8/site-packages (from librosa) (0.10.3.post1)\n", + "Requirement already satisfied: scipy>=1.0.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (1.7.3)\n", + "Requirement already satisfied: pooch>=1.0 in /opt/conda/lib/python3.8/site-packages (from librosa) (1.6.0)\n", + "Requirement already satisfied: llvmlite<0.39,>=0.38.0rc1 in /opt/conda/lib/python3.8/site-packages (from numba>=0.43.0->librosa) (0.38.0)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from numba>=0.43.0->librosa) (50.3.1.post20201107)\n", + "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.8/site-packages (from packaging>=20.0->librosa) (3.0.7)\n", + "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.8/site-packages (from pooch>=1.0->librosa) (2.24.0)\n", + "Requirement already satisfied: appdirs>=1.3.0 in /opt/conda/lib/python3.8/site-packages (from pooch>=1.0->librosa) (1.4.4)\n", + "Requirement already satisfied: six>=1.3 in /opt/conda/lib/python3.8/site-packages (from resampy>=0.2.2->librosa) (1.15.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from scikit-learn!=0.19.0,>=0.14.0->librosa) (3.0.0)\n", + "Requirement already satisfied: cffi>=1.0 in /opt/conda/lib/python3.8/site-packages (from soundfile>=0.10.2->librosa) (1.14.3)\n", + "Requirement already satisfied: pycparser in /opt/conda/lib/python3.8/site-packages (from cffi>=1.0->soundfile>=0.10.2->librosa) (2.20)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (3.0.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2020.12.5)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (1.25.11)\n", + "Requirement already satisfied: idna<3,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2.10)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: You are using pip version 21.3.1; however, version 22.0.2 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: jiwer in /opt/conda/lib/python3.8/site-packages (2.3.0)\n", + "Requirement already satisfied: python-Levenshtein==0.12.2 in /opt/conda/lib/python3.8/site-packages (from jiwer) (0.12.2)\n", + "Requirement already satisfied: setuptools in /opt/conda/lib/python3.8/site-packages (from python-Levenshtein==0.12.2->jiwer) (50.3.1.post20201107)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -5ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -4ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -3ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -2ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -1ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -uggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -okenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -kenizers (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -0ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution -ggingface-hub (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: Ignoring invalid distribution - (/opt/conda/lib/python3.8/site-packages)\n", + "WARNING: You are using pip version 21.3.1; however, version 22.0.2 is available.\n", + "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\n" + ] + } + ], + "source": [ + "%%bash\n", + "pip install datasets==1.13.3\n", + "pip install transformers==4.11.3\n", + "pip install huggingface_hub==0.1\n", + "pip install torchaudio\n", + "pip install librosa\n", + "pip install jiwer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "09a1f5ea", + "metadata": {}, + "outputs": [], + "source": [ + "%" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c57800bb", + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset, load_metric, Audio, Dataset\n", + "import os\n", + "import torchaudio\n", + "from tqdm.auto import tqdm" + ] + }, + { + "cell_type": "markdown", + "id": "98314e83", + "metadata": {}, + "source": [ + "### Common Voice Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "c9e549d0", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Reusing dataset common_voice (/workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd)\n", + "Reusing dataset common_voice (/workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd)\n" + ] + } + ], + "source": [ + "common_voice_train = load_dataset(\"common_voice\", \"tr\", split=\"train+validation\")\n", + "common_voice_test = load_dataset(\"common_voice\", \"tr\", split=\"test\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c1ca3246", + "metadata": {}, + "outputs": [], + "source": [ + "# remove unnecceesary attributes\n", + "common_voice_train = common_voice_train.remove_columns([\"accent\", \"age\", \"client_id\", \"down_votes\", \"gender\", \"locale\", \"segment\", \"up_votes\"])\n", + "common_voice_test = common_voice_test.remove_columns([\"accent\", \"age\", \"client_id\", \"down_votes\", \"gender\", \"locale\", \"segment\", \"up_votes\"])" + ] + }, + { + "cell_type": "markdown", + "id": "7574225f", + "metadata": {}, + "source": [ + "### Clean Up the Text" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "af63fb74", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove character\n", + "import re\n", + "chars_to_remove_regex = '[\\,\\?\\.\\!\\-\\;\\:\\\"\\“\\%\\‘\\”\\�\\']'\n", + "\n", + "def remove_special_characters(batch):\n", + " batch[\"sentence\"] = re.sub(chars_to_remove_regex, '', batch[\"sentence\"])\n", + " return batch" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "e1714034", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f2af4ccfa69a4f7e817db9785810fa26", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "0ex [00:00, ?ex/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "0d95f7cf9b114038932dc6a0a174146f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "0ex [00:00, ?ex/s]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "common_voice_train = common_voice_train.map(remove_special_characters)\n", + "common_voice_test = common_voice_test.map(remove_special_characters)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d348cf4a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'path': 'common_voice_tr_17346025.mp3',\n", + " 'audio': {'path': 'cv-corpus-6.1-2020-12-11/tr/clips/common_voice_tr_17346025.mp3',\n", + " 'array': array([0. , 0. , 0. , ..., 0.0002867 , 0.00025517,\n", + " 0.00025791], dtype=float32),\n", + " 'sampling_rate': 48000},\n", + " 'sentence': 'Hırvat makamları değişiklik istiyorlar'}" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "common_voice_train[0]" + ] + }, + { + "cell_type": "markdown", + "id": "53070ed8", + "metadata": {}, + "source": [ + "### Build Character" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "dc3b3351", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a573372b6d2343868e15af3920f983ba", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/3478 [00:00\n", + " \n", + " Your browser does not support the audio element.\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import IPython.display as ipd\n", + "import numpy as np\n", + "import random\n", + "\n", + "rand_int = random.randint(0, len(common_voice_train)-1)\n", + "\n", + "print(\"Target text:\", common_voice_train[rand_int][\"sentence\"])\n", + "print(\"Input array shape:\", common_voice_train[rand_int][\"audio\"][\"array\"].shape)\n", + "print(\"Sampling rate:\", common_voice_train[rand_int][\"audio\"][\"sampling_rate\"])\n", + "ipd.Audio(data=common_voice_train[rand_int][\"audio\"][\"array\"], autoplay=True, rate=16000)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "8e1df31f", + "metadata": {}, + "outputs": [], + "source": [ + "# This does not prepare the input for the Transformer model.\n", + "# This will resample the data and convert the sentence into indices\n", + "# Batch here is just for one entry (row)\n", + "def prepare_dataset(batch):\n", + " audio = batch[\"audio\"]\n", + " \n", + " # batched output is \"un-batched\"\n", + " batch[\"input_values\"] = processor(audio[\"array\"], sampling_rate=audio[\"sampling_rate\"]).input_values[0]\n", + " batch[\"input_length\"] = len(batch[\"input_values\"])\n", + " \n", + " with processor.as_target_processor():\n", + " batch[\"labels\"] = processor(batch[\"sentence\"]).input_ids\n", + " return batch" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "88b304c0", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-e3ff506f96ec6817.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-00a0dacd1c387ee8.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-89839f1a29958c06.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-ea97d53e6e03248b.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-74c31e1ede89718b.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-b4485d5ec10af59a.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-87741a8a8705e488.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-2aa5c421e49dbb8a.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-6fa3756abc090cb1.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-7082faf01a7536d9.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-dbf56923bad5550e.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-cfa541d30ccf3270.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-9f28af78c8d178d8.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-4fc740b07e55a01b.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-ec4bd65c3d0c2b80.arrow\n", + "Loading cached processed dataset at /workspace/.cache/huggingface/datasets/common_voice/tr/6.1.0/5693bfc0feeade582a78c2fb250bc88f52bd86f0a7f1bb22bfee67e715de30fd/cache-033c2e0fab0f0e8a.arrow\n" + ] + } + ], + "source": [ + "common_voice_train = common_voice_train.map(prepare_dataset, remove_columns=common_voice_train.column_names, num_proc=16)\n", + "common_voice_test = common_voice_test.map(prepare_dataset, remove_columns=common_voice_test.column_names, num_proc=16)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "5a9767e7", + "metadata": {}, + "outputs": [], + "source": [ + "# In case the dataset is too long which can lead to OOM. We should filter them out.\n", + "# max_input_length_in_sec = 5.0\n", + "# common_voice_train = common_voice_train.filter(lambda x: x < max_input_length_in_sec * processor.feature_extractor.sampling_rate, input_columns=[\"input_length\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "df2feb3b", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "\n", + "from dataclasses import dataclass, field\n", + "from typing import Any, Dict, List, Optional, Union\n", + "\n", + "@dataclass\n", + "class DataCollatorCTCWithPadding:\n", + " \"\"\"\n", + " Data collator that will dynamically pad the inputs received.\n", + " Args:\n", + " processor (:class:`~transformers.Wav2Vec2Processor`)\n", + " The processor used for proccessing the data.\n", + " padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):\n", + " Select a strategy to pad the returned sequences (according to the model's padding side and padding index)\n", + " among:\n", + " * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single\n", + " sequence if provided).\n", + " * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the\n", + " maximum acceptable input length for the model if that argument is not provided.\n", + " * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of\n", + " different lengths).\n", + " \"\"\"\n", + "\n", + " processor: Wav2Vec2Processor\n", + " padding: Union[bool, str] = True\n", + "\n", + " def __call__(self, features: List[Dict[str, Union[List[int], torch.Tensor]]]) -> Dict[str, torch.Tensor]:\n", + " # split inputs and labels since they have to be of different lenghts and need\n", + " # different padding methods\n", + " input_features = [{\"input_values\": feature[\"input_values\"]} for feature in features]\n", + " label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n", + "\n", + " batch = self.processor.pad(\n", + " input_features,\n", + " padding=self.padding,\n", + " return_tensors=\"pt\",\n", + " )\n", + "\n", + " with self.processor.as_target_processor():\n", + " labels_batch = self.processor.pad(\n", + " label_features,\n", + " padding=self.padding,\n", + " return_tensors=\"pt\",\n", + " )\n", + "\n", + " # replace padding with -100 to ignore loss correctly\n", + " labels = labels_batch[\"input_ids\"].masked_fill(labels_batch.attention_mask.ne(1), -100)\n", + "\n", + " batch[\"labels\"] = labels\n", + "\n", + " return batch" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "bf081d03", + "metadata": {}, + "outputs": [], + "source": [ + "data_collator = DataCollatorCTCWithPadding(processor=processor, padding=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "2059e2de", + "metadata": {}, + "outputs": [], + "source": [ + "wer_metric = load_metric(\"wer\")\n", + "# cer_metric = load_metric(\"cer\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "ac4bb898", + "metadata": {}, + "outputs": [], + "source": [ + "def compute_metrics(pred):\n", + " pred_logits = pred.predictions\n", + " pred_ids = np.argmax(pred_logits, axis=-1)\n", + "\n", + " pred.label_ids[pred.label_ids == -100] = tokenizer.pad_token_id\n", + "\n", + " pred_str = tokenizer.batch_decode(pred_ids)\n", + " # we do not want to group tokens when computing the metrics\n", + " label_str = tokenizer.batch_decode(pred.label_ids, group_tokens=False)\n", + "\n", + " wer = wer_metric.compute(predictions=pred_str, references=label_str)\n", + "# cer = cer_metric.compute(predictions=pred_str, references=label_str)\n", + "\n", + " return {\"wer\": wer}\n", + "# return {\"cer\": cer}" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "43669353", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of the model checkpoint at facebook/wav2vec2-xls-r-300m were not used when initializing Wav2Vec2ForCTC: ['quantizer.weight_proj.weight', 'project_q.bias', 'project_hid.bias', 'project_q.weight', 'quantizer.weight_proj.bias', 'quantizer.codevectors', 'project_hid.weight']\n", + "- This IS expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", + "- This IS NOT expected if you are initializing Wav2Vec2ForCTC from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", + "Some weights of Wav2Vec2ForCTC were not initialized from the model checkpoint at facebook/wav2vec2-xls-r-300m and are newly initialized: ['lm_head.weight', 'lm_head.bias']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "from transformers import Wav2Vec2ForCTC\n", + "\n", + "model = Wav2Vec2ForCTC.from_pretrained(\n", + " \"facebook/wav2vec2-xls-r-300m\", \n", + " attention_dropout=0.0,\n", + " hidden_dropout=0.0,\n", + " feat_proj_dropout=0.0,\n", + " mask_time_prob=0.05,\n", + " layerdrop=0.0,\n", + " ctc_loss_reduction=\"mean\", \n", + " pad_token_id=tokenizer.pad_token_id,\n", + " vocab_size=len(processor.tokenizer),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "f6a6f61e", + "metadata": {}, + "outputs": [], + "source": [ + "model.freeze_feature_encoder()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "54b7d33d", + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import TrainingArguments\n", + "\n", + "training_args = TrainingArguments(\n", + " output_dir='.',\n", + " group_by_length=True,\n", + " per_device_train_batch_size=8,\n", + " gradient_accumulation_steps=2,\n", + " evaluation_strategy=\"steps\",\n", + " gradient_checkpointing=True,\n", + " fp16=True,\n", + " num_train_epochs=25,\n", + " save_steps=500,\n", + " eval_steps=500,\n", + " logging_steps=100,\n", + " learning_rate=5e-5,\n", + " warmup_steps=1000,\n", + " save_total_limit=3\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "929dc96c", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using amp half precision backend\n" + ] + } + ], + "source": [ + "from transformers import Trainer\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " data_collator=data_collator,\n", + " args=training_args,\n", + " compute_metrics=compute_metrics,\n", + " train_dataset=common_voice_train,\n", + " eval_dataset=common_voice_test,\n", + " tokenizer=processor.feature_extractor,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "3460b4cd", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the training set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "/opt/conda/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use thePyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", + " warnings.warn(\n", + "***** Running training *****\n", + " Num examples = 3478\n", + " Num Epochs = 25\n", + " Instantaneous batch size per device = 8\n", + " Total train batch size (w. parallel, distributed & accumulation) = 16\n", + " Gradient Accumulation steps = 2\n", + " Total optimization steps = 5425\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [5425/5425 1:31:08, Epoch 24/25]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining LossValidation LossWer
5003.8859003.7607851.000000
10001.8193001.5307821.000613
15000.5986000.7295361.005616
20000.3992000.6185581.013377
25000.3199000.5972451.012254
30000.2388000.5555721.010109
35000.1882000.5172811.014092
40000.1604000.5170091.018278
45000.1443000.5267381.018380
50000.1404000.5366641.016747

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-500\n", + "Configuration saved in ./checkpoint-500/config.json\n", + "Model weights saved in ./checkpoint-500/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-500/preprocessor_config.json\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-1000\n", + "Configuration saved in ./checkpoint-1000/config.json\n", + "Model weights saved in ./checkpoint-1000/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-1000/preprocessor_config.json\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-1500\n", + "Configuration saved in ./checkpoint-1500/config.json\n", + "Model weights saved in ./checkpoint-1500/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-1500/preprocessor_config.json\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-2000\n", + "Configuration saved in ./checkpoint-2000/config.json\n", + "Model weights saved in ./checkpoint-2000/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-2000/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-500] due to args.save_total_limit\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-2500\n", + "Configuration saved in ./checkpoint-2500/config.json\n", + "Model weights saved in ./checkpoint-2500/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-2500/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-1000] due to args.save_total_limit\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-3000\n", + "Configuration saved in ./checkpoint-3000/config.json\n", + "Model weights saved in ./checkpoint-3000/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-3000/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-1500] due to args.save_total_limit\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-3500\n", + "Configuration saved in ./checkpoint-3500/config.json\n", + "Model weights saved in ./checkpoint-3500/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-3500/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-2000] due to args.save_total_limit\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-4000\n", + "Configuration saved in ./checkpoint-4000/config.json\n", + "Model weights saved in ./checkpoint-4000/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-4000/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-2500] due to args.save_total_limit\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-4500\n", + "Configuration saved in ./checkpoint-4500/config.json\n", + "Model weights saved in ./checkpoint-4500/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-4500/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-3000] due to args.save_total_limit\n", + "The following columns in the evaluation set don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.\n", + "***** Running Evaluation *****\n", + " Num examples = 1647\n", + " Batch size = 8\n", + "Saving model checkpoint to ./checkpoint-5000\n", + "Configuration saved in ./checkpoint-5000/config.json\n", + "Model weights saved in ./checkpoint-5000/pytorch_model.bin\n", + "Configuration saved in ./checkpoint-5000/preprocessor_config.json\n", + "Deleting older checkpoint [checkpoint-3500] due to args.save_total_limit\n", + "\n", + "\n", + "Training completed. Do not forget to share your model on huggingface.co/models =)\n", + "\n", + "\n" + ] + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=5425, training_loss=1.241710463449153, metrics={'train_runtime': 5469.9405, 'train_samples_per_second': 15.896, 'train_steps_per_second': 0.992, 'total_flos': 1.0590512839529611e+19, 'train_loss': 1.241710463449153, 'epoch': 25.0})" + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df904372", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}