Speaker Identification Verification training
State: closed | Pull request: no
Labels: name: bug

name: stale

**Describe the bug**

[https://github.com/NVIDIA/NeMo/blob/main/tutorials/speaker_tasks/Speaker_Identification_Verification.ipynb]
While trying to performs the steps mentioned in the above notebook, I'm facing error in trainer.fit(speaker_model)

**Steps/Code to reproduce bug**

Error Log:

{
	"name": "KeyError",
	"message": "'65190d15daa5426985e5f52e516de631'",
	"stack": "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)\nCell \u001b[0;32mIn[27], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m trainer\u001b[39m.\u001b[39;49mfit(speaker_model)\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:582\u001b[0m, in \u001b[0;36mTrainer.fit\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m    580\u001b[0m     \u001b[39mraise\u001b[39;00m \u001b[39mTypeError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m`Trainer.fit()` requires a `LightningModule`, got: \u001b[39m\u001b[39m{\u001b[39;00mmodel\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__qualname__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m    581\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstrategy\u001b[39m.\u001b[39m_lightning_module \u001b[39m=\u001b[39m model\n\u001b[0;32m--> 582\u001b[0m call\u001b[39m.\u001b[39;49m_call_and_handle_interrupt(\n\u001b[1;32m    583\u001b[0m     \u001b[39mself\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_fit_impl, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path\n\u001b[1;32m    584\u001b[0m )\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/call.py:38\u001b[0m, in \u001b[0;36m_call_and_handle_interrupt\u001b[0;34m(trainer, trainer_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[39mreturn\u001b[39;00m trainer\u001b[39m.\u001b[39mstrategy\u001b[39m.\u001b[39mlauncher\u001b[39m.\u001b[39mlaunch(trainer_fn, \u001b[39m*\u001b[39margs, trainer\u001b[39m=\u001b[39mtrainer, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m     37\u001b[0m     \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 38\u001b[0m         \u001b[39mreturn\u001b[39;00m trainer_fn(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m     40\u001b[0m \u001b[39mexcept\u001b[39;00m _TunerExitException:\n\u001b[1;32m     41\u001b[0m     trainer\u001b[39m.\u001b[39m_call_teardown_hook()\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:624\u001b[0m, in \u001b[0;36mTrainer._fit_impl\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m    617\u001b[0m ckpt_path \u001b[39m=\u001b[39m ckpt_path \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresume_from_checkpoint\n\u001b[1;32m    618\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ckpt_path \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_checkpoint_connector\u001b[39m.\u001b[39m_set_ckpt_path(\n\u001b[1;32m    619\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mfn,\n\u001b[1;32m    620\u001b[0m     ckpt_path,  \u001b[39m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m    621\u001b[0m     model_provided\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m    622\u001b[0m     model_connected\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlightning_module \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[1;32m    623\u001b[0m )\n\u001b[0;32m--> 624\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run(model, ckpt_path\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mckpt_path)\n\u001b[1;32m    626\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mstopped\n\u001b[1;32m    627\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtraining \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1061\u001b[0m, in \u001b[0;36mTrainer._run\u001b[0;34m(self, model, ckpt_path)\u001b[0m\n\u001b[1;32m   1057\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_checkpoint_connector\u001b[39m.\u001b[39mrestore_training_state()\n\u001b[1;32m   1059\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_checkpoint_connector\u001b[39m.\u001b[39mresume_end()\n\u001b[0;32m-> 1061\u001b[0m results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run_stage()\n\u001b[1;32m   1063\u001b[0m log\u001b[39m.\u001b[39mdetail(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m: trainer tearing down\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m   1064\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_teardown()\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1140\u001b[0m, in \u001b[0;36mTrainer._run_stage\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1138\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpredicting:\n\u001b[1;32m   1139\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_run_predict()\n\u001b[0;32m-> 1140\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run_train()\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1153\u001b[0m, in \u001b[0;36mTrainer._run_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1150\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_pre_training_routine()\n\u001b[1;32m   1152\u001b[0m \u001b[39mwith\u001b[39;00m isolate_rng():\n\u001b[0;32m-> 1153\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run_sanity_check()\n\u001b[1;32m   1155\u001b[0m \u001b[39m# enable train mode\u001b[39;00m\n\u001b[1;32m   1156\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1225\u001b[0m, in \u001b[0;36mTrainer._run_sanity_check\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1223\u001b[0m \u001b[39m# run eval step\u001b[39;00m\n\u001b[1;32m   1224\u001b[0m \u001b[39mwith\u001b[39;00m torch\u001b[39m.\u001b[39mno_grad():\n\u001b[0;32m-> 1225\u001b[0m     val_loop\u001b[39m.\u001b[39;49mrun()\n\u001b[1;32m   1227\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call_callback_hooks(\u001b[39m\"\u001b[39m\u001b[39mon_sanity_check_end\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m   1229\u001b[0m \u001b[39m# reset logger connector\u001b[39;00m\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/loops/loop.py:206\u001b[0m, in \u001b[0;36mLoop.run\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    203\u001b[0m         \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m    204\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_restarting \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\u001b[0;32m--> 206\u001b[0m output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mon_run_end()\n\u001b[1;32m    207\u001b[0m \u001b[39mreturn\u001b[39;00m output\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py:198\u001b[0m, in \u001b[0;36mEvaluationLoop.on_run_end\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    195\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrainer\u001b[39m.\u001b[39m_logger_connector\u001b[39m.\u001b[39mlog_eval_end_metrics(all_logged_outputs)\n\u001b[1;32m    197\u001b[0m \u001b[39m# hook\u001b[39;00m\n\u001b[0;32m--> 198\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_on_evaluation_end()\n\u001b[1;32m    200\u001b[0m \u001b[39m# enable train mode again\u001b[39;00m\n\u001b[1;32m    201\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_on_evaluation_model_train()\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/loops/dataloader/evaluation_loop.py:262\u001b[0m, in \u001b[0;36mEvaluationLoop._on_evaluation_end\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    260\u001b[0m \u001b[39m\"\"\"Runs ``on_{validation/test}_end`` hook.\"\"\"\u001b[39;00m\n\u001b[1;32m    261\u001b[0m hook_name \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mon_test_end\u001b[39m\u001b[39m\"\u001b[39m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrainer\u001b[39m.\u001b[39mtesting \u001b[39melse\u001b[39;00m \u001b[39m\"\u001b[39m\u001b[39mon_validation_end\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m--> 262\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtrainer\u001b[39m.\u001b[39;49m_call_callback_hooks(hook_name, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    263\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrainer\u001b[39m.\u001b[39m_call_lightning_module_hook(hook_name, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    264\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtrainer\u001b[39m.\u001b[39m_call_strategy_hook(hook_name, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/trainer/trainer.py:1343\u001b[0m, in \u001b[0;36mTrainer._call_callback_hooks\u001b[0;34m(self, hook_name, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1341\u001b[0m     \u001b[39mif\u001b[39;00m callable(fn):\n\u001b[1;32m   1342\u001b[0m         \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprofiler\u001b[39m.\u001b[39mprofile(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m[Callback]\u001b[39m\u001b[39m{\u001b[39;00mcallback\u001b[39m.\u001b[39mstate_key\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m{\u001b[39;00mhook_name\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m):\n\u001b[0;32m-> 1343\u001b[0m             fn(\u001b[39mself\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlightning_module, \u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1345\u001b[0m \u001b[39mif\u001b[39;00m pl_module:\n\u001b[1;32m   1346\u001b[0m     \u001b[39m# restore current_fx when nested context\u001b[39;00m\n\u001b[1;32m   1347\u001b[0m     pl_module\u001b[39m.\u001b[39m_current_fx_name \u001b[39m=\u001b[39m prev_fx_name\n\nFile \u001b[0;32m~/.local/lib/python3.8/site-packages/pytorch_lightning/callbacks/progress/tqdm_progress.py:299\u001b[0m, in \u001b[0;36mTQDMProgressBar.on_validation_end\u001b[0;34m(self, trainer, pl_module)\u001b[0m\n\u001b[1;32m    297\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_main_progress_bar \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m trainer\u001b[39m.\u001b[39mstate\u001b[39m.\u001b[39mfn \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mfit\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m    298\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmain_progress_bar\u001b[39m.\u001b[39mset_postfix(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_metrics(trainer, pl_module))\n\u001b[0;32m--> 299\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mval_progress_bar\u001b[39m.\u001b[39;49mclose()\n\u001b[1;32m    300\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mreset_dataloader_idx_tracker()\n\nFile \u001b[0;32m/usr/local/lib/python3.8/dist-packages/tqdm/notebook.py:293\u001b[0m, in \u001b[0;36mtqdm_notebook.close\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    291\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdisp(bar_style\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39msuccess\u001b[39m\u001b[39m'\u001b[39m, check_delay\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n\u001b[1;32m    292\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 293\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdisp(close\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, check_delay\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m)\n\nFile \u001b[0;32m/usr/local/lib/python3.8/dist-packages/tqdm/notebook.py:192\u001b[0m, in \u001b[0;36mtqdm_notebook.display\u001b[0;34m(self, msg, pos, close, bar_style, check_delay)\u001b[0m\n\u001b[1;32m    190\u001b[0m \u001b[39mif\u001b[39;00m close \u001b[39mand\u001b[39;00m pbar\u001b[39m.\u001b[39mbar_style \u001b[39m!=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mdanger\u001b[39m\u001b[39m'\u001b[39m:  \u001b[39m# hide only if no error\u001b[39;00m\n\u001b[1;32m    191\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 192\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcontainer\u001b[39m.\u001b[39;49mclose()\n\u001b[1;32m    193\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mAttributeError\u001b[39;00m:\n\u001b[1;32m    194\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcontainer\u001b[39m.\u001b[39mvisible \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n\nFile \u001b[0;32m/usr/local/lib/python3.8/dist-packages/ipywidgets/widgets/widget.py:500\u001b[0m, in \u001b[0;36mWidget.close\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    498\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcomm \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    499\u001b[0m     Widget\u001b[39m.\u001b[39m_active_widgets\u001b[39m.\u001b[39mpop(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel_id, \u001b[39mNone\u001b[39;00m)\n\u001b[0;32m--> 500\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcomm\u001b[39m.\u001b[39;49mclose()\n\u001b[1;32m    501\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcomm \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m    502\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_repr_mimebundle_ \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\nFile \u001b[0;32m/usr/local/lib/python3.8/dist-packages/comm/base_comm.py:109\u001b[0m, in \u001b[0;36mBaseComm.close\u001b[0;34m(self, data, metadata, buffers, deleting)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpublish_msg(\n\u001b[1;32m    102\u001b[0m     \u001b[39m\"\u001b[39m\u001b[39mcomm_close\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m    103\u001b[0m     data\u001b[39m=\u001b[39mdata,\n\u001b[1;32m    104\u001b[0m     metadata\u001b[39m=\u001b[39mmetadata,\n\u001b[1;32m    105\u001b[0m     buffers\u001b[39m=\u001b[39mbuffers,\n\u001b[1;32m    106\u001b[0m )\n\u001b[1;32m    107\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m deleting:\n\u001b[1;32m    108\u001b[0m     \u001b[39m# If deleting, the comm can't be registered\u001b[39;00m\n\u001b[0;32m--> 109\u001b[0m     comm\u001b[39m.\u001b[39;49mget_comm_manager()\u001b[39m.\u001b[39;49munregister_comm(\u001b[39mself\u001b[39;49m)\n\nFile \u001b[0;32m/usr/local/lib/python3.8/dist-packages/comm/base_comm.py:199\u001b[0m, in \u001b[0;36mCommManager.unregister_comm\u001b[0;34m(self, comm)\u001b[0m\n\u001b[1;32m    197\u001b[0m \u001b[39m\"\"\"Unregister a comm, and close its counterpart\"\"\"\u001b[39;00m\n\u001b[1;32m    198\u001b[0m \u001b[39m# unlike get_comm, this should raise a KeyError\u001b[39;00m\n\u001b[0;32m--> 199\u001b[0m comm \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mcomms\u001b[39m.\u001b[39;49mpop(comm\u001b[39m.\u001b[39;49mcomm_id)\n\n\u001b[0;31mKeyError\u001b[0m: '65190d15daa5426985e5f52e516de631'"
}

**Environment details**

If NVIDIA docker image is used you don't need to specify these.
Otherwise, please provide:
- Ubuntu 20.04.5 LTS
- PyTorch version: 1.12.1+cu113
- Python version: Python 3.8.10