{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# HuggingFace challenge - Debugger notebook\n", "Run this notebook to verify your libraries versions, check GPU config and run a quick training" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "id": "T2utsYSKszvv" }, "outputs": [], "source": [ "import platform\n", "import multiprocessing\n", "\n", "import torch\n", "import transformers\n", "import datasets\n", "\n", "import soundfile" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Print main infos" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Check your GPU informations (if any)\n", "If you launched an AI Training job with GPU resources, they should be listed below (Tesla V100s 32GB).\n", "Driver and CUDA version " ] }, { "cell_type": "markdown", "metadata": { "id": "TorMtpwPv6RQ" }, "source": [ "## Quick training run with a dummy model and data\n", "more information on https://github.com/huggingface/transformers/tree/master/examples/pytorch/speech-recognition" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "fevoJD15u4Ss", "outputId": "5861d34e-745b-45ee-e780-ed363043e655" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "--2022-01-31 17:09:10-- https://raw.githubusercontent.com/huggingface/transformers/master/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.111.133, 185.199.108.133, 185.199.110.133, ...\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.111.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 30360 (30K) [text/plain]\n", "Saving to: ‘run_speech_recognition_ctc.py’\n", "\n", "run_speech_recognit 100%[===================>] 29.65K --.-KB/s in 0.001s \n", "\n", "2022-01-31 17:09:10 (55.6 MB/s) - ‘run_speech_recognition_ctc.py’ saved [30360/30360]\n", "\n" ] } ], "source": [ "!wget -O run_speech_recognition_ctc.py https://raw.githubusercontent.com/huggingface/transformers/master/examples/pytorch/speech-recognition/run_speech_recognition_ctc.py" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Mz4bubhxxsad", "outputId": "23398525-cc19-43c2-9fec-497e06214f29" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "01/31/2022 17:10:15 - WARNING - __main__ - Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: True\n", "01/31/2022 17:10:15 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n", "_n_gpu=1,\n", "adafactor=False,\n", "adam_beta1=0.9,\n", "adam_beta2=0.999,\n", "adam_epsilon=1e-08,\n", "bf16=False,\n", "bf16_full_eval=False,\n", "dataloader_drop_last=False,\n", "dataloader_num_workers=0,\n", "dataloader_pin_memory=True,\n", "ddp_bucket_cap_mb=None,\n", "ddp_find_unused_parameters=None,\n", "debug=[],\n", "deepspeed=None,\n", "disable_tqdm=False,\n", "do_eval=True,\n", "do_predict=False,\n", "do_train=True,\n", "eval_accumulation_steps=None,\n", "eval_steps=500,\n", "evaluation_strategy=IntervalStrategy.STEPS,\n", "fp16=True,\n", "fp16_backend=auto,\n", "fp16_full_eval=False,\n", "fp16_opt_level=O1,\n", "gradient_accumulation_steps=1,\n", "gradient_checkpointing=True,\n", "greater_is_better=None,\n", "group_by_length=True,\n", "half_precision_backend=auto,\n", "hub_model_id=None,\n", "hub_strategy=HubStrategy.EVERY_SAVE,\n", "hub_token=,\n", "ignore_data_skip=False,\n", "label_names=None,\n", "label_smoothing_factor=0.0,\n", "learning_rate=0.0003,\n", "length_column_name=input_length,\n", "load_best_model_at_end=False,\n", "local_rank=-1,\n", "log_level=-1,\n", "log_level_replica=-1,\n", "log_on_each_node=True,\n", "logging_dir=./runs/Jan31_17-10-15_job-6a6be32c-c82d-4385-805b-1f7606124d5b,\n", "logging_first_step=False,\n", "logging_nan_inf_filter=True,\n", "logging_steps=500,\n", "logging_strategy=IntervalStrategy.STEPS,\n", "lr_scheduler_type=SchedulerType.LINEAR,\n", "max_grad_norm=1.0,\n", "max_steps=10,\n", "metric_for_best_model=None,\n", "mp_parameters=,\n", "no_cuda=False,\n", "num_train_epochs=3.0,\n", "optim=OptimizerNames.ADAMW_HF,\n", "output_dir=./,\n", "overwrite_output_dir=True,\n", "past_index=-1,\n", "per_device_eval_batch_size=8,\n", "per_device_train_batch_size=2,\n", "prediction_loss_only=False,\n", "push_to_hub=True,\n", "push_to_hub_model_id=None,\n", "push_to_hub_organization=None,\n", "push_to_hub_token=,\n", "remove_unused_columns=True,\n", "report_to=[],\n", "resume_from_checkpoint=None,\n", "run_name=./,\n", "save_on_each_node=False,\n", "save_steps=5,\n", "save_strategy=IntervalStrategy.STEPS,\n", "save_total_limit=1,\n", "seed=42,\n", "sharded_ddp=[],\n", "skip_memory_metrics=True,\n", "tf32=None,\n", "tpu_metrics_debug=False,\n", "tpu_num_cores=None,\n", "use_legacy_prediction_loop=False,\n", "warmup_ratio=0.0,\n", "warmup_steps=0,\n", "weight_decay=0.0,\n", "xpu_backend=None,\n", ")\n", "Downloading: 100%|█████████████████████████| 10.1k/10.1k [00:00<00:00, 7.28MB/s]\n", "Downloading: 100%|█████████████████████████| 2.98k/2.98k [00:00<00:00, 3.39MB/s]\n", "Downloading: 100%|██████████████████████████| 53.1k/53.1k [00:00<00:00, 325kB/s]\n", "Downloading and preparing dataset common_voice/ab to /workspace/.cache/huggingface/datasets/mozilla-foundation___common_voice/ab/8.0.0/b8bc4d453193c06a43269b46cd87f075c70f152ac963b7f28f7a2760c45ec3e8...\n", "Downloading: 100%|█████████████████████████| 1.72G/1.72G [01:31<00:00, 18.8MB/s]\n", "9124 examples [00:13, 710.66 examples/s] " ] } ], "source": [ "!python run_speech_recognition_ctc.py \\\n", "\t--dataset_name=\"mozilla-foundation/common_voice_8_0\" \\\n", "\t--model_name_or_path=\"hf-test/xls-r-dummy\" \\\n", "\t--dataset_config_name=\"ab\" \\\n", "\t--output_dir=\"./\" \\\n", "\t--overwrite_output_dir \\\n", "\t--max_steps=\"10\" \\\n", "\t--per_device_train_batch_size=\"2\" \\\n", "\t--learning_rate=\"3e-4\" \\\n", "\t--save_total_limit=\"1\" \\\n", "\t--evaluation_strategy=\"steps\" \\\n", "\t--text_column_name=\"sentence\" \\\n", "\t--length_column_name=\"input_length\" \\\n", "\t--save_steps=\"5\" \\\n", "\t--layerdrop=\"0.0\" \\\n", "\t--freeze_feature_encoder \\\n", "\t--gradient_checkpointing \\\n", "\t--fp16 \\\n", "\t--group_by_length \\\n", "\t--push_to_hub \\\n", "\t--use_auth_token \\\n", "\t--do_train --do_eva" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "accelerator": "GPU", "colab": { "authorship_tag": "ABX9TyM3OaMlm9YQtKpl28c8gBBd", "include_colab_link": true, "name": "DebugOVHTransformers.ipynb", "provenance": [] }, "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.8" } }, "nbformat": 4, "nbformat_minor": 4 }