{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import notebook_login\n",
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset, DatasetDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['path', 'audio', 'transcription', 'english_transcription', 'intent_class', 'lang_id'],\n",
       "    num_rows: 563\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minds14_train = load_dataset(\n",
    "    \"PolyAI/minds14\", \n",
    "    \"en-US\",\n",
    "    split=\"train\"\n",
    ")\n",
    "minds14_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['path', 'audio', 'transcription', 'english_transcription', 'intent_class', 'lang_id'],\n",
       "        num_rows: 450\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['path', 'audio', 'transcription', 'english_transcription', 'intent_class', 'lang_id'],\n",
       "        num_rows: 113\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minds14 = DatasetDict()\n",
    "\n",
    "minds14[\"train\"] = minds14_train.select(range(450))\n",
    "minds14[\"test\"] = minds14_train.select(range(450, 563))\n",
    "\n",
    "minds14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['audio', 'transcription'],\n",
       "        num_rows: 450\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['audio', 'transcription'],\n",
       "        num_rows: 113\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minds14 = minds14.select_columns(['audio', 'transcription'])\n",
    "minds14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import WhisperProcessor\n",
    "\n",
    "processor = WhisperProcessor.from_pretrained(\n",
    "    \"openai/whisper-tiny\", language=\"english\", task=\"transcribe\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'audio': Audio(sampling_rate=8000, mono=True, decode=True, id=None),\n",
       " 'transcription': Value(dtype='string', id=None)}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minds14[\"train\"].features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Audio\n",
    "\n",
    "sampling_rate = processor.feature_extractor.sampling_rate\n",
    "minds14 = minds14.cast_column(\"audio\", Audio(sampling_rate=sampling_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_dataset(example):\n",
    "    audio = example[\"audio\"]\n",
    "\n",
    "    example = processor(\n",
    "        audio=audio[\"array\"],\n",
    "        sampling_rate=audio[\"sampling_rate\"],\n",
    "        text=example[\"transcription\"],\n",
    "    )\n",
    "\n",
    "    # compute input length of audio sample in seconds\n",
    "    example[\"input_length\"] = len(audio[\"array\"]) / audio[\"sampling_rate\"]\n",
    "\n",
    "    return example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "356d0ccec48f41b9ad10504ae0ca4813",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/450 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef753a60316c4115924c49052eeb411d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/113 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "minds14 = minds14.map(\n",
    "    prepare_dataset, remove_columns=minds14.column_names[\"train\"], num_proc=1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_input_length = 30.0\n",
    "def is_audio_in_length_range(length):\n",
    "    return length < max_input_length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2292d10d955d4d958e07849f0abb57c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/450 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "minds14[\"train\"] = minds14[\"train\"].filter(\n",
    "    is_audio_in_length_range,\n",
    "    input_columns=[\"input_length\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_features', 'labels', 'input_length'],\n",
       "    num_rows: 445\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minds14['train']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training and Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "from dataclasses import dataclass\n",
    "from typing import Any, Dict, List, Union\n",
    "\n",
    "\n",
    "@dataclass\n",
    "class DataCollatorSpeechSeq2SeqWithPadding:\n",
    "    processor: Any\n",
    "\n",
    "    def __call__(\n",
    "        self, features: List[Dict[str, Union[List[int], torch.Tensor]]]\n",
    "    ) -> Dict[str, torch.Tensor]:\n",
    "        # split inputs and labels since they have to be of different lengths and need different padding methods\n",
    "        # first treat the audio inputs by simply returning torch tensors\n",
    "        input_features = [\n",
    "            {\"input_features\": feature[\"input_features\"][0]} for feature in features\n",
    "        ]\n",
    "        batch = self.processor.feature_extractor.pad(input_features, return_tensors=\"pt\")\n",
    "\n",
    "        # get the tokenized label sequences\n",
    "        label_features = [{\"input_ids\": feature[\"labels\"]} for feature in features]\n",
    "        # pad the labels to max length\n",
    "        labels_batch = self.processor.tokenizer.pad(label_features, return_tensors=\"pt\")\n",
    "\n",
    "        # replace padding with -100 to ignore loss correctly\n",
    "        labels = labels_batch[\"input_ids\"].masked_fill(\n",
    "            labels_batch.attention_mask.ne(1), -100\n",
    "        )\n",
    "\n",
    "        # if bos token is appended in previous tokenization step,\n",
    "        # cut bos token here as it's append later anyways\n",
    "        if (labels[:, 0] == self.processor.tokenizer.bos_token_id).all().cpu().item():\n",
    "            labels = labels[:, 1:]\n",
    "\n",
    "        batch[\"labels\"] = labels\n",
    "\n",
    "        return batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "from transformers.models.whisper.english_normalizer import BasicTextNormalizer\n",
    "\n",
    "metric = evaluate.load(\"wer\")\n",
    "normalizer = BasicTextNormalizer()\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    pred_ids = pred.predictions\n",
    "    label_ids = pred.label_ids\n",
    "\n",
    "    # replace -100 with the pad_token_id\n",
    "    label_ids[label_ids == -100] = processor.tokenizer.pad_token_id\n",
    "\n",
    "    # we do not want to group tokens when computing the metrics\n",
    "    pred_str = processor.batch_decode(pred_ids, skip_special_tokens=True)\n",
    "    label_str = processor.batch_decode(label_ids, skip_special_tokens=True)\n",
    "\n",
    "    # compute orthographic wer\n",
    "    wer_ortho = 100 * metric.compute(predictions=pred_str, references=label_str)\n",
    "\n",
    "    # compute normalised WER\n",
    "    pred_str_norm = [normalizer(pred) for pred in pred_str]\n",
    "    label_str_norm = [normalizer(label) for label in label_str]\n",
    "    # filtering step to only evaluate the samples that correspond to non-zero references:\n",
    "    pred_str_norm = [\n",
    "        pred_str_norm[i] for i in range(len(pred_str_norm)) if len(label_str_norm[i]) > 0\n",
    "    ]\n",
    "    label_str_norm = [\n",
    "        label_str_norm[i]\n",
    "        for i in range(len(label_str_norm))\n",
    "        if len(label_str_norm[i]) > 0\n",
    "    ]\n",
    "\n",
    "    wer = 100 * metric.compute(predictions=pred_str_norm, references=label_str_norm)\n",
    "\n",
    "    return {\"wer_ortho\": wer_ortho, \"wer\": wer}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import WhisperForConditionalGeneration\n",
    "model = WhisperForConditionalGeneration.from_pretrained(\"openai/whisper-tiny\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "\n",
    "# disable cache during training since it's incompatible with gradient checkpointing\n",
    "model.config.use_cache = False\n",
    "\n",
    "# set language and task for generation and re-enable cache\n",
    "model.generate = partial(\n",
    "    model.generate, language=\"english\", task=\"transcribe\", use_cache=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Seq2SeqTrainingArguments\n",
    "\n",
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=\"./whisper-tiny-en-us-minds14\",  # name on the HF Hub\n",
    "    per_device_train_batch_size=16,\n",
    "    gradient_accumulation_steps=1,  # increase by 2x for every 2x decrease in batch size\n",
    "    learning_rate=1e-5,\n",
    "    lr_scheduler_type=\"constant_with_warmup\",\n",
    "    warmup_steps=50,\n",
    "    max_steps=4000,  # increase to 4000 if you have your own GPU or a Colab paid plan\n",
    "    gradient_checkpointing=True,\n",
    "    # fp16=True,\n",
    "    # fp16_full_eval=True,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    per_device_eval_batch_size=16,\n",
    "    predict_with_generate=True,\n",
    "    generation_max_length=225,\n",
    "    save_steps=500,\n",
    "    eval_steps=500,\n",
    "    logging_steps=25,\n",
    "    report_to=[\"tensorboard\"],\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=\"wer\",\n",
    "    greater_is_better=False,\n",
    "    # push_to_hub=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Seq2SeqTrainer\n",
    "\n",
    "trainer = Seq2SeqTrainer(\n",
    "    args=training_args,\n",
    "    model=model,\n",
    "    train_dataset=minds14[\"train\"],\n",
    "    eval_dataset=minds14[\"test\"],\n",
    "    data_collator=data_collator,\n",
    "    compute_metrics=compute_metrics,\n",
    "    tokenizer=processor,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9dcf642e434e48468854ec1cbaa6120c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 1.584, 'learning_rate': 5e-06, 'epoch': 0.89}\n",
      "{'loss': 0.6567, 'learning_rate': 1e-05, 'epoch': 1.79}\n",
      "{'loss': 0.1857, 'learning_rate': 1e-05, 'epoch': 2.68}\n",
      "{'loss': 0.1218, 'learning_rate': 1e-05, 'epoch': 3.57}\n",
      "{'loss': 0.0876, 'learning_rate': 1e-05, 'epoch': 4.46}\n",
      "{'loss': 0.0512, 'learning_rate': 1e-05, 'epoch': 5.36}\n",
      "{'loss': 0.0299, 'learning_rate': 1e-05, 'epoch': 6.25}\n",
      "{'loss': 0.016, 'learning_rate': 1e-05, 'epoch': 7.14}\n",
      "{'loss': 0.0085, 'learning_rate': 1e-05, 'epoch': 8.04}\n",
      "{'loss': 0.0038, 'learning_rate': 1e-05, 'epoch': 8.93}\n",
      "{'loss': 0.0028, 'learning_rate': 1e-05, 'epoch': 9.82}\n",
      "{'loss': 0.0023, 'learning_rate': 1e-05, 'epoch': 10.71}\n",
      "{'loss': 0.0015, 'learning_rate': 1e-05, 'epoch': 11.61}\n",
      "{'loss': 0.0012, 'learning_rate': 1e-05, 'epoch': 12.5}\n",
      "{'loss': 0.0011, 'learning_rate': 1e-05, 'epoch': 13.39}\n",
      "{'loss': 0.0009, 'learning_rate': 1e-05, 'epoch': 14.29}\n",
      "{'loss': 0.0008, 'learning_rate': 1e-05, 'epoch': 15.18}\n",
      "{'loss': 0.0007, 'learning_rate': 1e-05, 'epoch': 16.07}\n",
      "{'loss': 0.0007, 'learning_rate': 1e-05, 'epoch': 16.96}\n",
      "{'loss': 0.0006, 'learning_rate': 1e-05, 'epoch': 17.86}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6448ea85978f4e14ad837324e482d808",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.25609758496284485, 'eval_wer_ortho': 35.90376310919186, 'eval_wer': 35.30106257378985, 'eval_runtime': 27.7439, 'eval_samples_per_second': 4.073, 'eval_steps_per_second': 0.288, 'epoch': 17.86}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0006, 'learning_rate': 1e-05, 'epoch': 18.75}\n",
      "{'loss': 0.0005, 'learning_rate': 1e-05, 'epoch': 19.64}\n",
      "{'loss': 0.0005, 'learning_rate': 1e-05, 'epoch': 20.54}\n",
      "{'loss': 0.0005, 'learning_rate': 1e-05, 'epoch': 21.43}\n",
      "{'loss': 0.0004, 'learning_rate': 1e-05, 'epoch': 22.32}\n",
      "{'loss': 0.0004, 'learning_rate': 1e-05, 'epoch': 23.21}\n",
      "{'loss': 0.0004, 'learning_rate': 1e-05, 'epoch': 24.11}\n",
      "{'loss': 0.0003, 'learning_rate': 1e-05, 'epoch': 25.0}\n",
      "{'loss': 0.0003, 'learning_rate': 1e-05, 'epoch': 25.89}\n",
      "{'loss': 0.0003, 'learning_rate': 1e-05, 'epoch': 26.79}\n",
      "{'loss': 0.0003, 'learning_rate': 1e-05, 'epoch': 27.68}\n",
      "{'loss': 0.0003, 'learning_rate': 1e-05, 'epoch': 28.57}\n",
      "{'loss': 0.0003, 'learning_rate': 1e-05, 'epoch': 29.46}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 30.36}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 31.25}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 32.14}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 33.04}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 33.93}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 34.82}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 35.71}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb97f0dd1de841f4a6904e6240ffa58a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.2792435586452484, 'eval_wer_ortho': 36.4589759407773, 'eval_wer': 35.9504132231405, 'eval_runtime': 20.8669, 'eval_samples_per_second': 5.415, 'eval_steps_per_second': 0.383, 'epoch': 35.71}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 36.61}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 37.5}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 38.39}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 39.29}\n",
      "{'loss': 0.0002, 'learning_rate': 1e-05, 'epoch': 40.18}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 41.07}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 41.96}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 42.86}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 43.75}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 44.64}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 45.54}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 46.43}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 47.32}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 48.21}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 49.11}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 50.0}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 50.89}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 51.79}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 52.68}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 53.57}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f8a8ea4cd774a72a6b89f714f17a78e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.29441583156585693, 'eval_wer_ortho': 36.705737199259715, 'eval_wer': 36.36363636363637, 'eval_runtime': 20.6363, 'eval_samples_per_second': 5.476, 'eval_steps_per_second': 0.388, 'epoch': 53.57}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 54.46}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 55.36}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 56.25}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 57.14}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 58.04}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 58.93}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 59.82}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 60.71}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 61.61}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 62.5}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 63.39}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 64.29}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 65.18}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 66.07}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 66.96}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 67.86}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 68.75}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 69.64}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 70.54}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 71.43}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3e15e770b014f84beff76935f5e1069",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.30616462230682373, 'eval_wer_ortho': 36.76742751388032, 'eval_wer': 36.481700118063756, 'eval_runtime': 20.6248, 'eval_samples_per_second': 5.479, 'eval_steps_per_second': 0.388, 'epoch': 71.43}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 72.32}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 73.21}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 74.11}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 75.0}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 75.89}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 76.79}\n",
      "{'loss': 0.0001, 'learning_rate': 1e-05, 'epoch': 77.68}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 78.57}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 79.46}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 80.36}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 81.25}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 82.14}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 83.04}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 83.93}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 84.82}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 85.71}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 86.61}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 87.5}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 88.39}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 89.29}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "287e32b32c004d56bfaabf8398bc0b57",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.31588611006736755, 'eval_wer_ortho': 36.82911782850093, 'eval_wer': 36.77685950413223, 'eval_runtime': 20.6213, 'eval_samples_per_second': 5.48, 'eval_steps_per_second': 0.388, 'epoch': 89.29}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 90.18}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 91.07}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 91.96}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 92.86}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 93.75}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 94.64}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 95.54}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 96.43}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 97.32}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 98.21}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 99.11}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 100.0}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 100.89}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 101.79}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 102.68}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 103.57}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 104.46}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 105.36}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 106.25}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 107.14}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4c5b0d4040949408a37a90be1cc106b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3247106671333313, 'eval_wer_ortho': 36.705737199259715, 'eval_wer': 36.658795749704844, 'eval_runtime': 20.5021, 'eval_samples_per_second': 5.512, 'eval_steps_per_second': 0.39, 'epoch': 107.14}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mkhojira/Projects/mml/audio-course/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 108.04}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 108.93}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 109.82}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 110.71}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 111.61}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 112.5}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 113.39}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 114.29}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 115.18}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 116.07}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 116.96}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 117.86}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 118.75}\n",
      "{'loss': 0.0, 'learning_rate': 1e-05, 'epoch': 119.64}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/mkhojira/Projects/mml/audio-course/unit5/hands_on.ipynb Cell 22\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/mkhojira/Projects/mml/audio-course/unit5/hands_on.ipynb#X34sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m trainer\u001b[39m.\u001b[39;49mtrain()\n",
      "File \u001b[0;32m~/Projects/mml/audio-course/venv/lib/python3.8/site-packages/transformers/trainer.py:1555\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   1553\u001b[0m         hf_hub_utils\u001b[39m.\u001b[39menable_progress_bars()\n\u001b[1;32m   1554\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1555\u001b[0m     \u001b[39mreturn\u001b[39;00m inner_training_loop(\n\u001b[1;32m   1556\u001b[0m         args\u001b[39m=\u001b[39;49margs,\n\u001b[1;32m   1557\u001b[0m         resume_from_checkpoint\u001b[39m=\u001b[39;49mresume_from_checkpoint,\n\u001b[1;32m   1558\u001b[0m         trial\u001b[39m=\u001b[39;49mtrial,\n\u001b[1;32m   1559\u001b[0m         ignore_keys_for_eval\u001b[39m=\u001b[39;49mignore_keys_for_eval,\n\u001b[1;32m   1560\u001b[0m     )\n",
      "File \u001b[0;32m~/Projects/mml/audio-course/venv/lib/python3.8/site-packages/transformers/trainer.py:1862\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   1859\u001b[0m \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39maccelerator\u001b[39m.\u001b[39maccumulate(model):\n\u001b[1;32m   1860\u001b[0m     tr_loss_step \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining_step(model, inputs)\n\u001b[0;32m-> 1862\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m   1863\u001b[0m     args\u001b[39m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   1864\u001b[0m     \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m is_torch_tpu_available()\n\u001b[1;32m   1865\u001b[0m     \u001b[39mand\u001b[39;00m (torch\u001b[39m.\u001b[39misnan(tr_loss_step) \u001b[39mor\u001b[39;00m torch\u001b[39m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   1866\u001b[0m ):\n\u001b[1;32m   1867\u001b[0m     \u001b[39m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   1868\u001b[0m     tr_loss \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m tr_loss \u001b[39m/\u001b[39m (\u001b[39m1\u001b[39m \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mglobal_step \u001b[39m-\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_globalstep_last_logged)\n\u001b[1;32m   1869\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from transformers import GenerationConfig\n",
    "# generation_config = GenerationConfig.from_pretrained(\"openai/whisper-tiny.en\")\n",
    "# generation_config.push_to_hub('mirodil/whisper-tiny-en-us-minds14')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5cb7500ba08c4c98b821669c3207517d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "events.out.tfevents.1700719599.L67DDV9G7R.91939.0:   0%|          | 0.00/29.3k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26bce367c9974964a5e06097af5959e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/151M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d947a721dfcc44cab504adee4a2cab9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "training_args.bin:   0%|          | 0.00/4.73k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4487700a97e42188f5bf27cf538c82d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Upload 3 LFS files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'https://huggingface.co/mirodil/whisper-tiny-en-us-minds14/tree/main/'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kwargs = {\n",
    "    \"dataset_tags\": \"PolyAI/minds14\",\n",
    "    \"finetuned_from\": \"openai/whisper-tiny\",\n",
    "    \"tasks\": \"automatic-speech-recognition\",\n",
    "}\n",
    "trainer.push_to_hub(**kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.generation_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hasattr(generation_config, \"lang_to_id\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}