diff --git "a/e5_interleaving-cl.ipynb" "b/e5_interleaving-cl.ipynb" --- "a/e5_interleaving-cl.ipynb" +++ "b/e5_interleaving-cl.ipynb" @@ -92,8 +92,8 @@ "Building dependency tree \n", "Reading state information... Done\n", "git-lfs is already the newest version (2.9.2-1).\n", - "0 upgraded, 0 newly installed, 0 to remove and 156 not upgraded.\n", - "Error: Failed to call git rev-parse --git-dir: exit status 128 \n", + "0 upgraded, 0 newly installed, 0 to remove and 155 not upgraded.\n", + "Updated git hooks.\n", "Git LFS initialized.\n" ] } @@ -120,25 +120,181 @@ "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: pip in ./.local/lib/python3.8/site-packages (22.3.1)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: torch in ./.local/lib/python3.8/site-packages (1.13.0)\n", - "Requirement already satisfied: torchaudio in ./.local/lib/python3.8/site-packages (0.13.0)\n", - "Requirement already satisfied: torchvision in ./.local/lib/python3.8/site-packages (0.14.0)\n", - "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in ./.local/lib/python3.8/site-packages (from torch) (11.10.3.66)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in ./.local/lib/python3.8/site-packages (from torch) (11.7.99)\n", - "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in ./.local/lib/python3.8/site-packages (from torch) (8.5.0.96)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in ./.local/lib/python3.8/site-packages (from torch) (11.7.99)\n", - "Requirement already satisfied: typing-extensions in ./.local/lib/python3.8/site-packages (from torch) (4.4.0)\n", - "Requirement already satisfied: wheel in /usr/lib/python3/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (0.34.2)\n", - "Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch) (45.2.0)\n", - "Requirement already satisfied: requests in ./.local/lib/python3.8/site-packages (from torchvision) (2.28.1)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/lib/python3/dist-packages (from torchvision) (7.0.0)\n", - "Requirement already satisfied: numpy in ./.local/lib/python3.8/site-packages (from torchvision) (1.23.5)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->torchvision) (2019.11.28)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in ./.local/lib/python3.8/site-packages (from requests->torchvision) (2.1.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->torchvision) (2.8)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.local/lib/python3.8/site-packages (from requests->torchvision) (1.26.13)\n" + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: pip in ./.local/lib/python3.8/site-packages (22.3.1)\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mDefaulting to user installation because normal site-packages is not writeable\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mCollecting torch\n", + " Using cached torch-1.13.0-cp38-cp38-manylinux1_x86_64.whl (890.2 MB)\n", + "Collecting torchaudio\n", + " Using cached torchaudio-0.13.0-cp38-cp38-manylinux1_x86_64.whl (4.2 MB)\n", + "Collecting torchvision\n", + " Using cached torchvision-0.14.0-cp38-cp38-manylinux1_x86_64.whl (24.3 MB)\n", + "Collecting nvidia-cudnn-cu11==8.5.0.96\n", + " Using cached nvidia_cudnn_cu11-8.5.0.96-2-py3-none-manylinux1_x86_64.whl (557.1 MB)\n", + "Collecting nvidia-cublas-cu11==11.10.3.66\n", + " Using cached nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl (317.1 MB)\n", + "Collecting typing-extensions\n", + " Using cached typing_extensions-4.4.0-py3-none-any.whl (26 kB)\n", + "Collecting nvidia-cuda-runtime-cu11==11.7.99\n", + " Using cached nvidia_cuda_runtime_cu11-11.7.99-py3-none-manylinux1_x86_64.whl (849 kB)\n", + "Collecting nvidia-cuda-nvrtc-cu11==11.7.99\n", + " Using cached nvidia_cuda_nvrtc_cu11-11.7.99-2-py3-none-manylinux1_x86_64.whl (21.0 MB)\n", + "Collecting wheel\n", + " Using cached wheel-0.38.4-py3-none-any.whl (36 kB)\n", + "Collecting setuptools\n", + " Using cached setuptools-65.6.3-py3-none-any.whl (1.2 MB)\n", + "Collecting numpy\n", + " Using cached numpy-1.24.0rc2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.3 MB)\n", + "Collecting pillow!=8.3.*,>=5.3.0\n", + " Using cached Pillow-9.3.0-cp38-cp38-manylinux_2_28_x86_64.whl (3.3 MB)\n", + "Collecting requests\n", + " Using cached requests-2.28.1-py3-none-any.whl (62 kB)\n", + "Collecting idna<4,>=2.5\n", + " Using cached idna-3.4-py3-none-any.whl (61 kB)\n", + "Collecting urllib3<1.27,>=1.21.1\n", + " Using cached urllib3-1.26.13-py2.py3-none-any.whl (140 kB)\n", + "Collecting charset-normalizer<3,>=2\n", + " Using cached charset_normalizer-2.1.1-py3-none-any.whl (39 kB)\n", + "Collecting certifi>=2017.4.17\n", + " Using cached certifi-2022.12.7-py3-none-any.whl (155 kB)\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mInstalling collected packages: wheel, urllib3, typing-extensions, setuptools, pillow, nvidia-cuda-nvrtc-cu11, numpy, idna, charset-normalizer, certifi, requests, nvidia-cuda-runtime-cu11, nvidia-cublas-cu11, nvidia-cudnn-cu11, torch, torchvision, torchaudio\n", + " Attempting uninstall: wheel\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: wheel 0.38.4\n", + " Uninstalling wheel-0.38.4:\n", + " Successfully uninstalled wheel-0.38.4\n", + " Attempting uninstall: urllib3\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: urllib3 1.26.13\n", + " Uninstalling urllib3-1.26.13:\n", + " Successfully uninstalled urllib3-1.26.13\n", + " Attempting uninstall: typing-extensions\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: typing_extensions 4.4.0\n", + " Uninstalling typing_extensions-4.4.0:\n", + " Successfully uninstalled typing_extensions-4.4.0\n", + " Attempting uninstall: setuptools\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: setuptools 65.6.3\n", + " Uninstalling setuptools-65.6.3:\n", + " Successfully uninstalled setuptools-65.6.3\n", + " Attempting uninstall: pillow\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: Pillow 9.3.0\n", + " Uninstalling Pillow-9.3.0:\n", + " Successfully uninstalled Pillow-9.3.0\n", + " Attempting uninstall: nvidia-cuda-nvrtc-cu11\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: nvidia-cuda-nvrtc-cu11 11.7.99\n", + " Uninstalling nvidia-cuda-nvrtc-cu11-11.7.99:\n", + " Successfully uninstalled nvidia-cuda-nvrtc-cu11-11.7.99\n", + " Attempting uninstall: numpy\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: numpy 1.23.5\n", + " Uninstalling numpy-1.23.5:\n", + " Successfully uninstalled numpy-1.23.5\n", + " Attempting uninstall: idna\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: idna 3.4\n", + " Uninstalling idna-3.4:\n", + " Successfully uninstalled idna-3.4\n", + " Attempting uninstall: charset-normalizer\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: charset-normalizer 2.1.1\n", + " Uninstalling charset-normalizer-2.1.1:\n", + " Successfully uninstalled charset-normalizer-2.1.1\n", + " Attempting uninstall: certifi\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: certifi 2022.12.7\n", + " Uninstalling certifi-2022.12.7:\n", + " Successfully uninstalled certifi-2022.12.7\n", + " Attempting uninstall: requests\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: requests 2.28.1\n", + " Uninstalling requests-2.28.1:\n", + " Successfully uninstalled requests-2.28.1\n", + " Attempting uninstall: nvidia-cuda-runtime-cu11\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: nvidia-cuda-runtime-cu11 11.7.99\n", + " Uninstalling nvidia-cuda-runtime-cu11-11.7.99:\n", + " Successfully uninstalled nvidia-cuda-runtime-cu11-11.7.99\n", + " Attempting uninstall: nvidia-cublas-cu11\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: nvidia-cublas-cu11 11.10.3.66\n", + " Uninstalling nvidia-cublas-cu11-11.10.3.66:\n", + " Successfully uninstalled nvidia-cublas-cu11-11.10.3.66\n", + " Attempting uninstall: nvidia-cudnn-cu11\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: nvidia-cudnn-cu11 8.5.0.96\n", + " Uninstalling nvidia-cudnn-cu11-8.5.0.96:\n", + " Successfully uninstalled nvidia-cudnn-cu11-8.5.0.96\n", + " Attempting uninstall: torch\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: torch 1.13.0\n", + " Uninstalling torch-1.13.0:\n", + " Successfully uninstalled torch-1.13.0\n", + " Attempting uninstall: torchvision\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: torchvision 0.14.0\n", + " Uninstalling torchvision-0.14.0:\n", + " Successfully uninstalled torchvision-0.14.0\n", + " Attempting uninstall: torchaudio\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: torchaudio 0.13.0\n", + " Uninstalling torchaudio-0.13.0:\n", + " Successfully uninstalled torchaudio-0.13.0\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "launchpadlib 1.10.13 requires testresources, which is not installed.\n", + "pandas-profiling 3.4.0 requires numpy<1.24,>=1.16.0, but you have numpy 1.24.0rc2 which is incompatible.\n", + "numba 0.56.4 requires numpy<1.24,>=1.18, but you have numpy 1.24.0rc2 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed certifi-2022.12.7 charset-normalizer-2.1.1 idna-3.4 numpy-1.24.0rc2 nvidia-cublas-cu11-11.10.3.66 nvidia-cuda-nvrtc-cu11-11.7.99 nvidia-cuda-runtime-cu11-11.7.99 nvidia-cudnn-cu11-8.5.0.96 pillow-9.3.0 requests-2.28.1 setuptools-65.6.3 torch-1.13.0 torchaudio-0.13.0 torchvision-0.14.0 typing-extensions-4.4.0 urllib3-1.26.13 wheel-0.38.4\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m" ] } ], @@ -148,7 +304,7 @@ "!pip3 install --upgrade packaging>=20.9\n", "!pip3 install --upgrade typing-extensions>=3.7.4.3\n", "\n", - "!pip3 install --pre torch torchaudio torchvision --upgrade\n", + "!pip3 install --pre torch torchaudio torchvision --force-reinstall\n", "\n", "#!pip3 install bitsandbytes\n", "\n", @@ -185,7 +341,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "d74b38c5-a1fb-4214-b4f4-b5bf0869f169", "metadata": { "colab": { @@ -200,7 +356,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Wed Dec 14 10:05:55 2022 \n", + "Mon Dec 12 18:05:43 2022 \n", "+-----------------------------------------------------------------------------+\n", "| NVIDIA-SMI 515.65.01 Driver Version: 515.65.01 CUDA Version: 11.7 |\n", "|-------------------------------+----------------------+----------------------+\n", @@ -209,7 +365,7 @@ "| | | MIG M. |\n", "|===============================+======================+======================|\n", "| 0 NVIDIA A100-SXM... On | 00000000:06:00.0 Off | 0 |\n", - "| N/A 31C P0 47W / 400W | 0MiB / 40960MiB | 0% Default |\n", + "| N/A 74C P0 350W / 400W | 36965MiB / 40960MiB | 100% Default |\n", "| | | Disabled |\n", "+-------------------------------+----------------------+----------------------+\n", " \n", @@ -218,7 +374,7 @@ "| GPU GI CI PID Type Process name GPU Memory |\n", "| ID ID Usage |\n", "|=============================================================================|\n", - "| No running processes found |\n", + "| 0 N/A N/A 3651714 C python 36963MiB |\n", "+-----------------------------------------------------------------------------+\n" ] } @@ -261,40 +417,40 @@ "output_type": "stream", "text": [ "Get:1 https://nvidia.github.io/libnvidia-container/stable/ubuntu18.04/amd64 InRelease [1484 B]\n", - "Hit:2 https://download.docker.com/linux/ubuntu focal InRelease \n", - "Hit:3 https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu18.04/amd64 InRelease\n", - "Hit:4 https://packages.cloud.google.com/apt cloud-sdk InRelease \n", - "Hit:5 http://archive.lambdalabs.com/ubuntu focal InRelease \n", - "Get:6 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB] \n", + "Hit:2 https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu18.04/amd64 InRelease\n", + "Hit:3 http://security.ubuntu.com/ubuntu focal-security InRelease \n", + "Hit:4 https://download.docker.com/linux/ubuntu focal InRelease \n", + "Hit:5 https://packages.cloud.google.com/apt cloud-sdk InRelease \n", + "Ign:6 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal InRelease \n", "Hit:7 https://packages.microsoft.com/repos/azure-cli focal InRelease \n", "Hit:8 http://archive.ubuntu.com/ubuntu focal InRelease \n", - "Hit:9 https://pkg.cloudflare.com/cloudflared focal InRelease \n", - "Ign:10 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal InRelease \n", - "Get:11 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB] \n", - "Err:12 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal Release \n", + "Hit:9 http://archive.lambdalabs.com/ubuntu focal InRelease \n", + "Hit:10 https://pkg.cloudflare.com/cloudflared focal InRelease \n", + "Err:11 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal Release\n", " 404 Not Found [IP: 185.125.190.52 80]\n", - "Hit:13 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal InRelease \n", - "Get:14 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB]\n", - "Reading package lists... Done \n", + "Hit:12 http://archive.ubuntu.com/ubuntu focal-updates InRelease\n", + "Hit:13 http://archive.ubuntu.com/ubuntu focal-backports InRelease\n", + "Hit:14 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal InRelease\n", + "Reading package lists... Done \n", "E: The repository 'http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal Release' does not have a Release file.\n", "N: Updating from such a repository can't be done securely, and is therefore disabled by default.\n", "N: See apt-secure(8) manpage for repository creation and user configuration details.\n", "Hit:1 https://download.docker.com/linux/ubuntu focal InRelease\n", "Get:2 https://nvidia.github.io/libnvidia-container/stable/ubuntu18.04/amd64 InRelease [1484 B]\n", - "Hit:3 https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu18.04/amd64 InRelease\n", - "Hit:4 https://packages.cloud.google.com/apt cloud-sdk InRelease \u001b[0m\u001b[33m\n", - "Hit:5 http://archive.lambdalabs.com/ubuntu focal InRelease \u001b[0m\u001b[33m\n", - "Hit:6 https://packages.microsoft.com/repos/azure-cli focal InRelease \u001b[0m\u001b[33m\n", - "Ign:7 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal InRelease \u001b[0m\u001b[33m\n", - "Hit:8 http://archive.ubuntu.com/ubuntu focal InRelease \u001b[0m\n", - "Hit:9 http://security.ubuntu.com/ubuntu focal-security InRelease \u001b[0m\u001b[33m\n", - "Hit:10 https://pkg.cloudflare.com/cloudflared focal InRelease \u001b[0m\n", + "Hit:3 http://security.ubuntu.com/ubuntu focal-security InRelease \u001b[0m\n", + "Hit:4 https://nvidia.github.io/nvidia-container-runtime/stable/ubuntu18.04/amd64 InRelease\n", + "Hit:5 https://packages.cloud.google.com/apt cloud-sdk InRelease \u001b[0m\n", + "Ign:6 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal InRelease \u001b[0m\u001b[33m\u001b[33m\n", + "Hit:7 http://archive.ubuntu.com/ubuntu focal InRelease \u001b[0m\n", + "Hit:8 https://packages.microsoft.com/repos/azure-cli focal InRelease \u001b[0m\u001b[33m\n", + "Hit:9 http://archive.lambdalabs.com/ubuntu focal InRelease \u001b[0m\n", + "Hit:10 https://pkg.cloudflare.com/cloudflared focal InRelease \u001b[0m\u001b[33m\n", "Hit:11 http://archive.ubuntu.com/ubuntu focal-updates InRelease \u001b[0m\n", - "Hit:12 http://archive.ubuntu.com/ubuntu focal-backports InRelease \u001b[0m\u001b[33m\n", - "Err:13 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal Release \u001b[33m\n", + "Hit:12 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal InRelease \n", + "Err:13 http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal Release\n", " 404 Not Found [IP: 185.125.190.52 80]\n", - "Hit:14 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu focal InRelease\n", - "Reading package lists... Done\u001b[33m\u001b[33m\n", + "Hit:14 http://archive.ubuntu.com/ubuntu focal-backports InRelease\n", + "Reading package lists... Done\u001b[33m\u001b[33m\u001b[33m\u001b[33m\n", "\u001b[1;31mE: \u001b[0mThe repository 'http://ppa.launchpad.net/jonathonf/ffmpeg-4/ubuntu focal Release' does not have a Release file.\u001b[0m\n", "\u001b[33mN: \u001b[0mUpdating from such a repository can't be done securely, and is therefore disabled by default.\u001b[0m\n", "\u001b[33mN: \u001b[0mSee apt-secure(8) manpage for repository creation and user configuration details.\u001b[0m\n", @@ -302,7 +458,7 @@ "Building dependency tree \n", "Reading state information... Done\n", "ffmpeg is already the newest version (7:4.2.7-0ubuntu0.1).\n", - "0 upgraded, 0 newly installed, 0 to remove and 156 not upgraded.\n" + "0 upgraded, 0 newly installed, 0 to remove and 154 not upgraded.\n" ] } ], @@ -344,165 +500,210 @@ "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", - "Collecting git+https://github.com/huggingface/datasets\n", - " Cloning https://github.com/huggingface/datasets to /tmp/pip-req-build-klxr8_0g\n", - " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/datasets /tmp/pip-req-build-klxr8_0g\n", - " Resolved https://github.com/huggingface/datasets to commit c902456677116a081f762fa2b4aad13a0aa04d6e\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mCollecting git+https://github.com/huggingface/datasets\n", + " Cloning https://github.com/huggingface/datasets to /tmp/pip-req-build-2byfy50h\n", + " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/datasets /tmp/pip-req-build-2byfy50h\n", + " Resolved https://github.com/huggingface/datasets to commit 5266c81430628edc175013692f02f5f2747ff29e\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", - "\u001b[?25hRequirement already satisfied: packaging in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (22.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets==2.7.1.dev0) (5.3.1)\n", - "Requirement already satisfied: multiprocess in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (0.70.14)\n", - "Requirement already satisfied: dill<0.3.7 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (0.3.6)\n", - "Requirement already satisfied: fsspec[http]>=2021.11.1 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (2022.11.0)\n", - "Requirement already satisfied: aiohttp in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (3.8.3)\n", - "Requirement already satisfied: pyarrow>=6.0.0 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (10.0.1)\n", - "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (0.11.1)\n", - "Requirement already satisfied: responses<0.19 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (0.18.0)\n", - "Requirement already satisfied: numpy>=1.17 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (1.23.5)\n", - "Requirement already satisfied: xxhash in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (3.1.0)\n", - "Requirement already satisfied: requests>=2.19.0 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (2.28.1)\n", - "Requirement already satisfied: pandas in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (1.5.1)\n", + "\u001b[?25hRequirement already satisfied: xxhash in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (3.1.0)\n", + "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (0.11.1)\n", + "Requirement already satisfied: multiprocess in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (0.70.14)\n", + "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (2022.11.0)\n", + "Requirement already satisfied: packaging in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (22.0)\n", + "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (0.18.0)\n", "Requirement already satisfied: tqdm>=4.62.1 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (4.64.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (1.8.2)\n", - "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (2.1.1)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (4.0.2)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (1.3.1)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (1.3.3)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (6.0.3)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (3.8.3)\n", + "Requirement already satisfied: numpy>=1.17 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (1.24.0rc2)\n", + "Requirement already satisfied: pandas in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (1.5.1)\n", + "Requirement already satisfied: requests>=2.19.0 in ./.local/lib/python3.8/site-packages (from datasets==2.7.1.dev0) (2.28.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from datasets==2.7.1.dev0) (5.3.1)\n", + "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (0.3.6)\n", + "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets==2.7.1.dev0) (10.0.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->datasets==2.7.1.dev0) (19.3.0)\n", - "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets==2.7.1.dev0) (3.0.12)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets==2.7.1.dev0) (1.3.1)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets==2.7.1.dev0) (1.3.3)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets==2.7.1.dev0) (6.0.3)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets==2.7.1.dev0) (4.0.2)\n", + "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in ./.local/lib/python3.8/site-packages (from aiohttp->datasets==2.7.1.dev0) (2.1.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets==2.7.1.dev0) (1.8.2)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.local/lib/python3.8/site-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets==2.7.1.dev0) (4.4.0)\n", + "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets==2.7.1.dev0) (3.0.12)\n", + "Requirement already satisfied: certifi>=2017.4.17 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->datasets==2.7.1.dev0) (2022.12.7)\n", + "Requirement already satisfied: idna<4,>=2.5 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->datasets==2.7.1.dev0) (3.4)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->datasets==2.7.1.dev0) (1.26.13)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets==2.7.1.dev0) (2.8)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->datasets==2.7.1.dev0) (2019.11.28)\n", "Requirement already satisfied: pytz>=2020.1 in ./.local/lib/python3.8/site-packages (from pandas->datasets==2.7.1.dev0) (2022.5)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in ./.local/lib/python3.8/site-packages (from pandas->datasets==2.7.1.dev0) (2.8.2)\n", "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.1->pandas->datasets==2.7.1.dev0) (1.14.0)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Collecting git+https://github.com/huggingface/transformers\n", - " Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-fqy16y_p\n", - " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-fqy16y_p\n", - " Resolved https://github.com/huggingface/transformers to commit d994473b05a83ea398d9f10ca458855df095e22d\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mDefaulting to user installation because normal site-packages is not writeable\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mCollecting git+https://github.com/huggingface/transformers\n", + " Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-rez3g9mu\n", + " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-rez3g9mu\n", + " Resolved https://github.com/huggingface/transformers to commit 799cea64ac1029d66e9e58f18bc6f47892270723\n", " Installing build dependencies ... \u001b[?25ldone\n", "\u001b[?25h Getting requirements to build wheel ... \u001b[?25ldone\n", "\u001b[?25h Preparing metadata (pyproject.toml) ... \u001b[?25ldone\n", - "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (4.64.1)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers==4.26.0.dev0) (5.3.1)\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from transformers==4.26.0.dev0) (0.11.1)\n", + "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers==4.26.0.dev0) (3.0.12)\n", + "Requirement already satisfied: numpy>=1.17 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (1.24.0rc2)\n", "Requirement already satisfied: requests in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (2.28.1)\n", - "Requirement already satisfied: numpy>=1.17 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (1.23.5)\n", + "Requirement already satisfied: tqdm>=4.27 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (4.64.1)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers==4.26.0.dev0) (2022.10.31)\n", + "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.8/dist-packages (from transformers==4.26.0.dev0) (0.13.2)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/lib/python3/dist-packages (from transformers==4.26.0.dev0) (5.3.1)\n", "Requirement already satisfied: packaging>=20.0 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (22.0)\n", - "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (0.13.2)\n", - "Requirement already satisfied: filelock in /usr/lib/python3/dist-packages (from transformers==4.26.0.dev0) (3.0.12)\n", - "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (0.11.1)\n", - "Requirement already satisfied: regex!=2019.12.17 in ./.local/lib/python3.8/site-packages (from transformers==4.26.0.dev0) (2022.10.31)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in ./.local/lib/python3.8/site-packages (from huggingface-hub<1.0,>=0.10.0->transformers==4.26.0.dev0) (4.4.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers==4.26.0.dev0) (2.8)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.local/lib/python3.8/site-packages (from requests->transformers==4.26.0.dev0) (1.26.13)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests->transformers==4.26.0.dev0) (2019.11.28)\n", "Requirement already satisfied: charset-normalizer<3,>=2 in ./.local/lib/python3.8/site-packages (from requests->transformers==4.26.0.dev0) (2.1.1)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: librosa in ./.local/lib/python3.8/site-packages (0.9.2)\n", - "Requirement already satisfied: scipy>=1.2.0 in ./.local/lib/python3.8/site-packages (from librosa) (1.9.3)\n", - "Requirement already satisfied: numpy>=1.17.0 in ./.local/lib/python3.8/site-packages (from librosa) (1.23.5)\n", - "Requirement already satisfied: audioread>=2.1.9 in ./.local/lib/python3.8/site-packages (from librosa) (3.0.0)\n", + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.local/lib/python3.8/site-packages (from requests->transformers==4.26.0.dev0) (1.26.13)\n", + "Requirement already satisfied: idna<4,>=2.5 in ./.local/lib/python3.8/site-packages (from requests->transformers==4.26.0.dev0) (3.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in ./.local/lib/python3.8/site-packages (from requests->transformers==4.26.0.dev0) (2022.12.7)\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mDefaulting to user installation because normal site-packages is not writeable\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: librosa in /usr/local/lib/python3.8/dist-packages (0.9.2)\n", + "Requirement already satisfied: packaging>=20.0 in ./.local/lib/python3.8/site-packages (from librosa) (22.0)\n", + "Requirement already satisfied: soundfile>=0.10.2 in /usr/local/lib/python3.8/dist-packages (from librosa) (0.11.0)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.8/dist-packages (from librosa) (3.0.0)\n", + "Requirement already satisfied: resampy>=0.2.2 in /usr/local/lib/python3.8/dist-packages (from librosa) (0.4.2)\n", "Requirement already satisfied: decorator>=4.0.10 in /usr/lib/python3/dist-packages (from librosa) (4.4.2)\n", + "Requirement already satisfied: pooch>=1.0 in /usr/local/lib/python3.8/dist-packages (from librosa) (1.6.0)\n", + "Requirement already satisfied: numba>=0.45.1 in /usr/local/lib/python3.8/dist-packages (from librosa) (0.56.4)\n", + "Requirement already satisfied: numpy>=1.17.0 in ./.local/lib/python3.8/site-packages (from librosa) (1.24.0rc2)\n", "Requirement already satisfied: joblib>=0.14 in ./.local/lib/python3.8/site-packages (from librosa) (1.2.0)\n", - "Requirement already satisfied: packaging>=20.0 in ./.local/lib/python3.8/site-packages (from librosa) (22.0)\n", - "Requirement already satisfied: resampy>=0.2.2 in ./.local/lib/python3.8/site-packages (from librosa) (0.4.2)\n", - "Requirement already satisfied: numba>=0.45.1 in ./.local/lib/python3.8/site-packages (from librosa) (0.56.4)\n", - "Requirement already satisfied: soundfile>=0.10.2 in ./.local/lib/python3.8/site-packages (from librosa) (0.11.0)\n", - "Requirement already satisfied: pooch>=1.0 in ./.local/lib/python3.8/site-packages (from librosa) (1.6.0)\n", "Requirement already satisfied: scikit-learn>=0.19.1 in /usr/lib/python3/dist-packages (from librosa) (0.22.2.post1)\n", - "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in ./.local/lib/python3.8/site-packages (from numba>=0.45.1->librosa) (0.39.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in ./.local/lib/python3.8/site-packages (from librosa) (1.9.3)\n", "Requirement already satisfied: importlib-metadata in ./.local/lib/python3.8/site-packages (from numba>=0.45.1->librosa) (5.0.0)\n", - "Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from numba>=0.45.1->librosa) (45.2.0)\n", + "Requirement already satisfied: setuptools in ./.local/lib/python3.8/site-packages (from numba>=0.45.1->librosa) (65.6.3)\n", + "Collecting numpy>=1.17.0\n", + " Using cached numpy-1.23.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.1 MB)\n", + "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.8/dist-packages (from numba>=0.45.1->librosa) (0.39.1)\n", "Requirement already satisfied: appdirs>=1.3.0 in /usr/lib/python3/dist-packages (from pooch>=1.0->librosa) (1.4.3)\n", "Requirement already satisfied: requests>=2.19.0 in ./.local/lib/python3.8/site-packages (from pooch>=1.0->librosa) (2.28.1)\n", "Requirement already satisfied: cffi>=1.0 in /usr/lib/python3/dist-packages (from soundfile>=0.10.2->librosa) (1.14.0)\n", - "Requirement already satisfied: charset-normalizer<3,>=2 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2.1.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2.8)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/lib/python3/dist-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2019.11.28)\n", + "Requirement already satisfied: idna<4,>=2.5 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (3.4)\n", + "Requirement already satisfied: certifi>=2017.4.17 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2022.12.7)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (1.26.13)\n", + "Requirement already satisfied: charset-normalizer<3,>=2 in ./.local/lib/python3.8/site-packages (from requests>=2.19.0->pooch>=1.0->librosa) (2.1.1)\n", "Requirement already satisfied: zipp>=0.5 in /usr/lib/python3/dist-packages (from importlib-metadata->numba>=0.45.1->librosa) (1.0.0)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: gradio in ./.local/lib/python3.8/site-packages (3.13.0)\n", - "Requirement already satisfied: pillow in /usr/lib/python3/dist-packages (from gradio) (7.0.0)\n", - "Requirement already satisfied: pydub in ./.local/lib/python3.8/site-packages (from gradio) (0.25.1)\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mInstalling collected packages: numpy\n", + " Attempting uninstall: numpy\n", + "\u001b[33m WARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m Found existing installation: numpy 1.24.0rc2\n", + " Uninstalling numpy-1.24.0rc2:\n", + " Successfully uninstalled numpy-1.24.0rc2\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mSuccessfully installed numpy-1.23.5\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mDefaulting to user installation because normal site-packages is not writeable\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: jiwer in /usr/local/lib/python3.8/dist-packages (2.5.1)\n", + "Requirement already satisfied: levenshtein==0.20.2 in /usr/local/lib/python3.8/dist-packages (from jiwer) (0.20.2)\n", + "Requirement already satisfied: rapidfuzz<3.0.0,>=2.3.0 in /usr/local/lib/python3.8/dist-packages (from levenshtein==0.20.2->jiwer) (2.13.5)\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mDefaulting to user installation because normal site-packages is not writeable\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: gradio in ./.local/lib/python3.8/site-packages (3.12.0)\n", + "Requirement already satisfied: h11<0.13,>=0.11 in ./.local/lib/python3.8/site-packages (from gradio) (0.12.0)\n", "Requirement already satisfied: pydantic in ./.local/lib/python3.8/site-packages (from gradio) (1.10.2)\n", - "Requirement already satisfied: ffmpy in ./.local/lib/python3.8/site-packages (from gradio) (0.3.0)\n", + "Requirement already satisfied: jinja2 in ./.local/lib/python3.8/site-packages (from gradio) (3.1.2)\n", "Requirement already satisfied: markdown-it-py[linkify,plugins] in ./.local/lib/python3.8/site-packages (from gradio) (2.1.0)\n", - "Requirement already satisfied: fastapi in ./.local/lib/python3.8/site-packages (from gradio) (0.88.0)\n", - "Requirement already satisfied: orjson in ./.local/lib/python3.8/site-packages (from gradio) (3.8.3)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.8/dist-packages (from gradio) (2022.11.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from gradio) (3.8.3)\n", "Requirement already satisfied: pandas in ./.local/lib/python3.8/site-packages (from gradio) (1.5.1)\n", - "Requirement already satisfied: pyyaml in /usr/lib/python3/dist-packages (from gradio) (5.3.1)\n", - "Requirement already satisfied: websockets>=10.0 in ./.local/lib/python3.8/site-packages (from gradio) (10.4)\n", - "Requirement already satisfied: requests in ./.local/lib/python3.8/site-packages (from gradio) (2.28.1)\n", - "Requirement already satisfied: h11<0.13,>=0.11 in ./.local/lib/python3.8/site-packages (from gradio) (0.12.0)\n", + "Requirement already satisfied: httpx in ./.local/lib/python3.8/site-packages (from gradio) (0.23.1)\n", "Requirement already satisfied: numpy in ./.local/lib/python3.8/site-packages (from gradio) (1.23.5)\n", - "Requirement already satisfied: python-multipart in ./.local/lib/python3.8/site-packages (from gradio) (0.0.5)\n", + "Requirement already satisfied: ffmpy in ./.local/lib/python3.8/site-packages (from gradio) (0.3.0)\n", "Requirement already satisfied: pycryptodome in ./.local/lib/python3.8/site-packages (from gradio) (3.16.0)\n", - "Requirement already satisfied: jinja2 in ./.local/lib/python3.8/site-packages (from gradio) (3.1.2)\n", - "Requirement already satisfied: fsspec in ./.local/lib/python3.8/site-packages (from gradio) (2022.11.0)\n", + "Requirement already satisfied: python-multipart in ./.local/lib/python3.8/site-packages (from gradio) (0.0.5)\n", + "Requirement already satisfied: websockets>=10.0 in ./.local/lib/python3.8/site-packages (from gradio) (10.4)\n", + "Requirement already satisfied: pyyaml in /usr/lib/python3/dist-packages (from gradio) (5.3.1)\n", "Requirement already satisfied: matplotlib in ./.local/lib/python3.8/site-packages (from gradio) (3.5.3)\n", + "Requirement already satisfied: orjson in ./.local/lib/python3.8/site-packages (from gradio) (3.8.3)\n", + "Requirement already satisfied: requests in ./.local/lib/python3.8/site-packages (from gradio) (2.28.1)\n", "Requirement already satisfied: uvicorn in ./.local/lib/python3.8/site-packages (from gradio) (0.20.0)\n", "Requirement already satisfied: paramiko in ./.local/lib/python3.8/site-packages (from gradio) (2.12.0)\n", - "Requirement already satisfied: httpx in ./.local/lib/python3.8/site-packages (from gradio) (0.23.1)\n", - "Requirement already satisfied: aiohttp in ./.local/lib/python3.8/site-packages (from gradio) (3.8.3)\n", - "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (2.1.1)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (6.0.3)\n", + "Requirement already satisfied: pydub in ./.local/lib/python3.8/site-packages (from gradio) (0.25.1)\n", + "Requirement already satisfied: fastapi in ./.local/lib/python3.8/site-packages (from gradio) (0.88.0)\n", + "Requirement already satisfied: pillow in ./.local/lib/python3.8/site-packages (from gradio) (9.3.0)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/lib/python3/dist-packages (from aiohttp->gradio) (19.3.0)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (1.8.2)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (1.3.1)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (1.3.3)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (4.0.2)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio) (1.3.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio) (1.8.2)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio) (6.0.3)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio) (4.0.2)\n", + "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in ./.local/lib/python3.8/site-packages (from aiohttp->gradio) (2.1.1)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->gradio) (1.3.3)\n", "Requirement already satisfied: starlette==0.22.0 in ./.local/lib/python3.8/site-packages (from fastapi->gradio) (0.22.0)\n", - "Requirement already satisfied: anyio<5,>=3.4.0 in ./.local/lib/python3.8/site-packages (from starlette==0.22.0->fastapi->gradio) (3.6.2)\n", "Requirement already satisfied: typing-extensions>=3.10.0 in ./.local/lib/python3.8/site-packages (from starlette==0.22.0->fastapi->gradio) (4.4.0)\n", - "Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from httpx->gradio) (2019.11.28)\n", + "Requirement already satisfied: anyio<5,>=3.4.0 in ./.local/lib/python3.8/site-packages (from starlette==0.22.0->fastapi->gradio) (3.6.2)\n", "Requirement already satisfied: sniffio in ./.local/lib/python3.8/site-packages (from httpx->gradio) (1.3.0)\n", + "Requirement already satisfied: certifi in ./.local/lib/python3.8/site-packages (from httpx->gradio) (2022.12.7)\n", "Requirement already satisfied: httpcore<0.17.0,>=0.15.0 in ./.local/lib/python3.8/site-packages (from httpx->gradio) (0.15.0)\n", "Requirement already satisfied: rfc3986[idna2008]<2,>=1.3 in ./.local/lib/python3.8/site-packages (from httpx->gradio) (1.5.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in ./.local/lib/python3.8/site-packages (from jinja2->gradio) (2.1.1)\n", "Requirement already satisfied: mdurl~=0.1 in ./.local/lib/python3.8/site-packages (from markdown-it-py[linkify,plugins]->gradio) (0.1.2)\n", "Requirement already satisfied: linkify-it-py~=1.0 in ./.local/lib/python3.8/site-packages (from markdown-it-py[linkify,plugins]->gradio) (1.0.3)\n", "Requirement already satisfied: mdit-py-plugins in ./.local/lib/python3.8/site-packages (from markdown-it-py[linkify,plugins]->gradio) (0.3.3)\n", - "Requirement already satisfied: python-dateutil>=2.7 in ./.local/lib/python3.8/site-packages (from matplotlib->gradio) (2.8.2)\n", - "Requirement already satisfied: packaging>=20.0 in ./.local/lib/python3.8/site-packages (from matplotlib->gradio) (22.0)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/lib/python3/dist-packages (from matplotlib->gradio) (0.10.0)\n", "Requirement already satisfied: pyparsing>=2.2.1 in /usr/lib/python3/dist-packages (from matplotlib->gradio) (2.4.6)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/lib/python3/dist-packages (from matplotlib->gradio) (1.0.1)\n", - "Requirement already satisfied: cycler>=0.10 in /usr/lib/python3/dist-packages (from matplotlib->gradio) (0.10.0)\n", + "Requirement already satisfied: packaging>=20.0 in ./.local/lib/python3.8/site-packages (from matplotlib->gradio) (22.0)\n", + "Requirement already satisfied: python-dateutil>=2.7 in ./.local/lib/python3.8/site-packages (from matplotlib->gradio) (2.8.2)\n", "Requirement already satisfied: fonttools>=4.22.0 in ./.local/lib/python3.8/site-packages (from matplotlib->gradio) (4.38.0)\n", "Requirement already satisfied: pytz>=2020.1 in ./.local/lib/python3.8/site-packages (from pandas->gradio) (2022.5)\n", "Requirement already satisfied: pynacl>=1.0.1 in /usr/lib/python3/dist-packages (from paramiko->gradio) (1.3.0)\n", "Requirement already satisfied: cryptography>=2.5 in /usr/lib/python3/dist-packages (from paramiko->gradio) (2.8)\n", "Requirement already satisfied: six in /usr/lib/python3/dist-packages (from paramiko->gradio) (1.14.0)\n", "Requirement already satisfied: bcrypt>=3.1.3 in ./.local/lib/python3.8/site-packages (from paramiko->gradio) (4.0.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->gradio) (2.8)\n", + "Requirement already satisfied: idna<4,>=2.5 in ./.local/lib/python3.8/site-packages (from requests->gradio) (3.4)\n", "Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./.local/lib/python3.8/site-packages (from requests->gradio) (1.26.13)\n", "Requirement already satisfied: click>=7.0 in /usr/lib/python3/dist-packages (from uvicorn->gradio) (7.0)\n", "Requirement already satisfied: uc-micro-py in ./.local/lib/python3.8/site-packages (from linkify-it-py~=1.0->markdown-it-py[linkify,plugins]->gradio) (1.0.1)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: more-itertools in /usr/lib/python3/dist-packages (4.2.0)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: jiwer in ./.local/lib/python3.8/site-packages (2.5.1)\n", - "Requirement already satisfied: bitsandbytes in ./.local/lib/python3.8/site-packages (0.35.4)\n", - "Requirement already satisfied: accelerate in ./.local/lib/python3.8/site-packages (0.15.0)\n", - "Requirement already satisfied: levenshtein==0.20.2 in ./.local/lib/python3.8/site-packages (from jiwer) (0.20.2)\n", - "Requirement already satisfied: rapidfuzz<3.0.0,>=2.3.0 in ./.local/lib/python3.8/site-packages (from levenshtein==0.20.2->jiwer) (2.13.6)\n", - "Requirement already satisfied: torch>=1.4.0 in ./.local/lib/python3.8/site-packages (from accelerate) (1.13.0)\n", - "Requirement already satisfied: packaging>=20.0 in ./.local/lib/python3.8/site-packages (from accelerate) (22.0)\n", - "Requirement already satisfied: numpy>=1.17 in ./.local/lib/python3.8/site-packages (from accelerate) (1.23.5)\n", - "Requirement already satisfied: pyyaml in /usr/lib/python3/dist-packages (from accelerate) (5.3.1)\n", - "Requirement already satisfied: psutil in /usr/lib/python3/dist-packages (from accelerate) (5.5.1)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in ./.local/lib/python3.8/site-packages (from torch>=1.4.0->accelerate) (11.7.99)\n", - "Requirement already satisfied: typing-extensions in ./.local/lib/python3.8/site-packages (from torch>=1.4.0->accelerate) (4.4.0)\n", - "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in ./.local/lib/python3.8/site-packages (from torch>=1.4.0->accelerate) (11.10.3.66)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in ./.local/lib/python3.8/site-packages (from torch>=1.4.0->accelerate) (11.7.99)\n", - "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in ./.local/lib/python3.8/site-packages (from torch>=1.4.0->accelerate) (8.5.0.96)\n", - "Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.4.0->accelerate) (45.2.0)\n", - "Requirement already satisfied: wheel in /usr/lib/python3/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.4.0->accelerate) (0.34.2)\n" + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mDefaulting to user installation because normal site-packages is not writeable\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0mRequirement already satisfied: more-itertools in /usr/local/lib/python3.8/dist-packages (9.0.0)\n", + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m" ] } ], @@ -512,10 +713,9 @@ "!pip3 install numexpr>=2.7.3\n", "!pip install librosa\n", "!pip install evaluate>=0.3.0\n", + "!pip install jiwer\n", "!pip install gradio\n", - "!pip install more-itertools\n", - "!pip install jiwer \n", - "#!pip install bitsandbytes accelerate" + "!pip install more-itertools" ] }, { @@ -535,7 +735,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 1, "id": "dff27c76-575c-432b-8916-b1b810efef4a", "metadata": { "colab": { @@ -568,7 +768,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "54def7c3052c464b960445d9650f571a", + "model_id": "6a2bfa275b6f4cdba66b6abdab11e859", "version_major": 2, "version_minor": 0 }, @@ -650,12 +850,26 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "id": "065a8cf7-e54f-4ac3-900e-609c80714fca", "metadata": { "id": "065a8cf7-e54f-4ac3-900e-609c80714fca" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m\u001b[33mWARNING: Ignoring invalid distribution -orch (/home/ubuntu/.local/lib/python3.8/site-packages)\u001b[0m\u001b[33m\n", + "\u001b[0m" + ] + } + ], "source": [ "!pip3 install --upgrade numexpr>=2.7.3\n", "\n", @@ -684,7 +898,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "id": "a2787582-554f-44ce-9f38-4180a5ed6b44", "metadata": { "id": "a2787582-554f-44ce-9f38-4180a5ed6b44", @@ -727,7 +941,7 @@ " all_datasets = []\n", " # iterate over the datasets we want to interleave\n", " for i, dataset_name in enumerate(dataset_names):\n", - " dataset = load_dataset(dataset_name, dataset_config_names[i], split=splits[i], streaming=False, **kwargs)\n", + " dataset = load_dataset(dataset_name, dataset_config_names[i], split=splits[i], streaming=True, **kwargs)\n", " # resample to specified sampling rate\n", " dataset = dataset.cast_column(\"audio\", Audio(sampling_rate))\n", " # normalise columns to [\"audio\", \"sentence\"]\n", @@ -746,22 +960,13 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, "id": "qOwlctMhNmCG", "metadata": { "id": "qOwlctMhNmCG", "tags": [] }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset common_voice_11_0 (/home/ubuntu/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/el/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f)\n", - "Found cached dataset fleurs (/home/ubuntu/.cache/huggingface/datasets/google___fleurs/el_gr/2.0.0/aabb39fb29739c495517ac904e2886819b6e344702f0a5b5283cb178b087c94a)\n" - ] - } - ], + "outputs": [], "source": [ "ds = load_multiple_streaming_datasets(dataset_names, dataset_config_names=dataset_config_names, text_column_names=text_column_names, use_auth_token=True)\n", "\n" @@ -779,31 +984,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 4, "id": "imRHJOpm4V_j", "metadata": { "id": "imRHJOpm4V_j" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset common_voice_11_0 (/home/ubuntu/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/el/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f)\n", - "Found cached dataset fleurs (/home/ubuntu/.cache/huggingface/datasets/google___fleurs/el_gr/2.0.0/aabb39fb29739c495517ac904e2886819b6e344702f0a5b5283cb178b087c94a)\n", - "Found cached dataset common_voice_11_0 (/home/ubuntu/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/el/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f)\n" - ] - } - ], + "outputs": [], "source": [ "from datasets import IterableDatasetDict\n", "raw_datasets = IterableDatasetDict()\n", "\n", "raw_datasets[\"train\"] = load_multiple_streaming_datasets(dataset_names, dataset_config_names=dataset_config_names, text_column_names=text_column_names, use_auth_token=True)\n", - "raw_datasets[\"test\"] = load_dataset(\"mozilla-foundation/common_voice_11_0\", \"el\", split=\"test\", streaming=False, use_auth_token=True)\n", - "\n", - "\n", - "#raw_datasets = raw_datasets.remove_columns([\"accent\", \"age\", \"client_id\", \"down_votes\", \"gender\", \"locale\", \"path\", \"segment\", \"up_votes\"])\n", + "raw_datasets[\"test\"] = load_dataset(\"mozilla-foundation/common_voice_11_0\", \"el\", split=\"test\", streaming=True, use_auth_token=True)\n", "\n", "#raw_datasets[\"train\"] = load_multiple_streaming_datasets(dataset_names, \"el\", split=\"\", use_auth_token=True)\n", "#raw_datasets[\"test\"] = load_multiple_streaming_datasets(dataset_names, \"el\", split=\"\", use_auth_token=True)\n", @@ -839,7 +1031,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 5, "id": "77d9f0c5-8607-4642-a8ac-c3ab2e223ea6", "metadata": { "id": "77d9f0c5-8607-4642-a8ac-c3ab2e223ea6" @@ -849,7 +1041,6 @@ "from transformers import WhisperProcessor\n", "\n", "processor = WhisperProcessor.from_pretrained(\"emilios/whisper-medium-el\", language=\"Greek\", task=\"transcribe\")\n", - "#processor = WhisperProcessor.from_pretrained(\"emilios/whisper-medium-el\", language=\"Greek\", task=\"transcribe\", load_in_8bit=True)\n", "#processor = WhisperProcessor.from_pretrained(\"farsipal/whisper-small-el\", language=\"Greek\", task=\"transcribe\")\n" ] }, @@ -875,7 +1066,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "id": "ab5a13b4-9bd4-4aa0-aef2-b3de9b762988", "metadata": { "colab": { @@ -892,7 +1083,7 @@ " 'sentence': Value(dtype='string', id=None)}" ] }, - "execution_count": 12, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -920,7 +1111,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 7, "id": "3ab6a724-3d1e-478b-a9e9-d2f85feb6c39", "metadata": { "id": "3ab6a724-3d1e-478b-a9e9-d2f85feb6c39" @@ -944,7 +1135,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 8, "id": "d041650e-1c48-4439-87b3-5b6f4a514107", "metadata": { "id": "d041650e-1c48-4439-87b3-5b6f4a514107" @@ -954,11 +1145,9 @@ "import string\n", "import re\n", "\n", - "do_lower_case = True\n", + "do_lower_case = False\n", "do_remove_punctuation = False\n", "\n", - "#normalizer = BasicTextNormalizer()\n", - "\n", "punctuation_to_remove = string.punctuation.replace(\"'\", \"\") # don't remove apostrophes\n", "punctuation_to_remove_regex = f\"[{''.join(punctuation_to_remove)}]\"\n", "\n", @@ -982,7 +1171,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 9, "id": "c085911c-a10a-41ef-8874-306e0503e9bb", "metadata": { "id": "c085911c-a10a-41ef-8874-306e0503e9bb" @@ -1023,34 +1212,12 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 10, "id": "a37a7cdb-9013-427f-8de9-6a8d0e9dc684", "metadata": { "id": "a37a7cdb-9013-427f-8de9-6a8d0e9dc684" }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "1d964121a24b4f4dbd3ce7b21112b05c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/6430 [00:00\n", - " \n", - " \n", - " [7000/7000 1:21:59, Epoch 33.98/35]\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining LossValidation LossWer
70000.0004000.38536911.450594

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: down_votes, locale, age, client_id, accent, input_length, gender, path, segment, up_votes. If down_votes, locale, age, client_id, accent, input_length, gender, path, segment, up_votes are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message.\n", - "***** Running Evaluation *****\n", - " Num examples = 1696\n", - " Batch size = 16\n", - "Saving model checkpoint to ./whisper-medium-el/checkpoint-7000\n", - "Configuration saved in ./whisper-medium-el/checkpoint-7000/config.json\n", - "Model weights saved in ./whisper-medium-el/checkpoint-7000/pytorch_model.bin\n", - "Feature extractor saved in ./whisper-medium-el/checkpoint-7000/preprocessor_config.json\n", - "tokenizer config file saved in ./whisper-medium-el/checkpoint-7000/tokenizer_config.json\n", - "Special tokens file saved in ./whisper-medium-el/checkpoint-7000/special_tokens_map.json\n", - "added tokens file saved in ./whisper-medium-el/checkpoint-7000/added_tokens.json\n", - "Feature extractor saved in ./whisper-medium-el/preprocessor_config.json\n", - "tokenizer config file saved in ./whisper-medium-el/tokenizer_config.json\n", - "Special tokens file saved in ./whisper-medium-el/special_tokens_map.json\n", - "added tokens file saved in ./whisper-medium-el/added_tokens.json\n" - ] - }, - { - "ename": "OSError", - "evalue": "error: insufficient permission for adding an object to repository database .git/objects\nerror: e5_interleaving-resume.ipynb: failed to insert into database\nerror: unable to index file 'e5_interleaving-resume.ipynb'\nfatal: adding files failed\n", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mCalledProcessError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/huggingface_hub/repository.py\u001b[0m in \u001b[0;36mgit_add\u001b[0;34m(self, pattern, auto_lfs_track)\u001b[0m\n\u001b[1;32m 1121\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_subprocess\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"git add -v\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mpattern\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlocal_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1123\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Adding to index:\\n{result.stdout}\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/huggingface_hub/utils/_subprocess.py\u001b[0m in \u001b[0;36mrun_subprocess\u001b[0;34m(command, folder, check, **kwargs)\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m return subprocess.run(\n\u001b[0m\u001b[1;32m 62\u001b[0m \u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/lib/python3.8/subprocess.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(input, capture_output, timeout, check, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m 515\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcheck\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mretcode\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 516\u001b[0;31m raise CalledProcessError(retcode, process.args,\n\u001b[0m\u001b[1;32m 517\u001b[0m output=stdout, stderr=stderr)\n", - "\u001b[0;31mCalledProcessError\u001b[0m: Command '['git', 'add', '-v', '.']' returned non-zero exit status 128.", - "\nDuring handling of the above exception, another exception occurred:\n", - "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;31m#trainer.train()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m 1532\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_inner_training_loop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_train_batch_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_find_batch_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1533\u001b[0m )\n\u001b[0;32m-> 1534\u001b[0;31m return inner_training_loop(\n\u001b[0m\u001b[1;32m 1535\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1536\u001b[0m \u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mresume_from_checkpoint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m 1857\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_step_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1858\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1859\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_log_save_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtr_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_keys_for_eval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1860\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1861\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_substep_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_maybe_log_save_evaluate\u001b[0;34m(self, tr_loss, model, trial, epoch, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m 2124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2125\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_save\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2126\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_save_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrial\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2127\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_save\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_save_checkpoint\u001b[0;34m(self, model, trial, metrics)\u001b[0m\n\u001b[1;32m 2268\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2269\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpush_to_hub\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2270\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_push_from_checkpoint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2271\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2272\u001b[0m \u001b[0;31m# Maybe delete some older checkpoints.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/transformers/trainer.py\u001b[0m in \u001b[0;36m_push_from_checkpoint\u001b[0;34m(self, checkpoint_folder)\u001b[0m\n\u001b[1;32m 3440\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3441\u001b[0m \u001b[0mcommit_message\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34mf\"Training in progress, epoch {int(self.state.epoch)}\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3442\u001b[0;31m _, self.push_in_progress = self.repo.push_to_hub(\n\u001b[0m\u001b[1;32m 3443\u001b[0m \u001b[0mcommit_message\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcommit_message\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblocking\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mauto_lfs_prune\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3444\u001b[0m )\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/huggingface_hub/repository.py\u001b[0m in \u001b[0;36mpush_to_hub\u001b[0;34m(self, commit_message, blocking, clean_ok, auto_lfs_prune)\u001b[0m\n\u001b[1;32m 1428\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Repo currently clean. Ignoring push_to_hub\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1429\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1430\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgit_add\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mauto_lfs_track\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1431\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgit_commit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommit_message\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1432\u001b[0m return self.git_push(\n", - "\u001b[0;32m~/.local/lib/python3.8/site-packages/huggingface_hub/repository.py\u001b[0m in \u001b[0;36mgit_add\u001b[0;34m(self, pattern, auto_lfs_track)\u001b[0m\n\u001b[1;32m 1123\u001b[0m \u001b[0mlogger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Adding to index:\\n{result.stdout}\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1124\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCalledProcessError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1125\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mEnvironmentError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstderr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1127\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgit_commit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcommit_message\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"commit files to HF hub\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mOSError\u001b[0m: error: insufficient permission for adding an object to repository database .git/objects\nerror: e5_interleaving-resume.ipynb: failed to insert into database\nerror: unable to index file 'e5_interleaving-resume.ipynb'\nfatal: adding files failed\n" - ] - } - ], + "outputs": [], "source": [ "#trainer.train()\n", "trainer.train(resume_from_checkpoint = True)\n" @@ -2469,7 +2511,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "6dd0e310-9b07-4133-ac14-2ed2d7524e22", "metadata": { "id": "6dd0e310-9b07-4133-ac14-2ed2d7524e22" @@ -2485,7 +2527,8 @@ " \"model_name\": \"Whisper Medium El Greco\", # a 'pretty' name for your model\n", " \"finetuned_from\": \"openai/whisper-medium\",\n", " \"tasks\": \"automatic-speech-recognition\",\n", - " \"tags\": \"hf-asr-leaderboard, whisper-medium, mozilla-foundation/common_voice_11_0, greek, whisper-event\",\n", + " \"tags\": \"hf-asr-leaderboard\",\n", + " #\"tags\": \"hf-asr-leaderboard, whisper-medium, mozilla-foundation/common_voice_11_0, greek, whisper-event\",\n", "}" ] }, @@ -2501,262 +2544,12 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": null, "id": "95737cda-c5dd-4887-a4d0-dfcb0d61d977", "metadata": { "id": "95737cda-c5dd-4887-a4d0-dfcb0d61d977" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Saving model checkpoint to ./whisper-medium-el\n", - "Configuration saved in ./whisper-medium-el/config.json\n", - "Model weights saved in ./whisper-medium-el/pytorch_model.bin\n", - "Feature extractor saved in ./whisper-medium-el/preprocessor_config.json\n", - "tokenizer config file saved in ./whisper-medium-el/tokenizer_config.json\n", - "Special tokens file saved in ./whisper-medium-el/special_tokens_map.json\n", - "added tokens file saved in ./whisper-medium-el/added_tokens.json\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2cbda43ce4ff41d5b06481239fded50f", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Upload file pytorch_model.bin: 0%| | 32.0k/2.85G [00:00 main\n", - "\n", - "Dropping the following result as it does not have all the necessary fields:\n", - "{'task': {'name': 'Automatic Speech Recognition', 'type': 'automatic-speech-recognition'}, 'dataset': {'name': 'Common Voice 11.0', 'type': 'mozilla-foundation/common_voice_11_0', 'config': 'el', 'split': 'test', 'args': 'el'}}\n", - "To https://huggingface.co/emilios/whisper-medium-el\n", - " d6004cb..0fcacdf main -> main\n", - "\n" - ] - }, - { - "data": { - "text/plain": [ - "'https://huggingface.co/emilios/whisper-medium-el/commit/d6004cbf9a0795ad80eecbd7642d13a34329d1ca'" - ] - }, - "execution_count": 33, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "trainer.push_to_hub(**kwargs)" ] @@ -2764,31 +2557,7 @@ { "cell_type": "code", "execution_count": null, - "id": "085a8a52-4191-4d3e-a8b2-6829d948c8b8", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82a24484-5d1a-468b-bf29-57f0a124f65e", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a7ae995c-67a9-4ded-8b61-2c22d203c7a4", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ea7a64f-17bb-44bb-9636-66bfc10ff0d0", + "id": "afa8a29a-59ab-463f-b2bf-2fbf2bae9b82", "metadata": {}, "outputs": [], "source": []