Repository: voicepaw/so-vits-svc-fork

Issue:
Error when training using multi gpu
i'm training using 2 gpu and get this error when running svc training
i don't know if this error is related to multi GPU training or not

Traceback (most recent call last):
  File "/opt/conda/envs/LoRA/bin/svc", line 8, in <module>
    sys.exit(cli())
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/click/core.py", line 1130, in __call__
    return self.main(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/click/core.py", line 1055, in main
    rv = self.invoke(ctx)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/click/core.py", line 760, in invoke
    return __callback(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/so_vits_svc_fork/__main__.py", line 129, in train
    train(
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/so_vits_svc_fork/train.py", line 82, in train
    trainer.fit(model, datamodule=datamodule)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 520, in fit
    call._call_and_handle_interrupt(
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 42, in _call_and_handle_interrupt
    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 92, in launch
    return function(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 559, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 935, in _run
    results = self._run_stage()
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 978, in _run_stage
    self.fit_loop.run()
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 201, in run
    self.advance()
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 354, in advance
    self.epoch_loop.run(self._data_fetcher)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 133, in run
    self.advance(data_fetcher)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 220, in advance
    batch_output = self.manual_optimization.run(kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/manual.py", line 90, in run
    self.advance(kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/loops/optimization/manual.py", line 109, in advance
    training_step_output = call._call_strategy_hook(trainer, "training_step", *kwargs.values())
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 288, in _call_strategy_hook
    output = fn(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/strategies/ddp.py", line 329, in training_step
    return self.model(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1156, in forward
    output = self._run_ddp_forward(*inputs, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/torch/nn/parallel/distributed.py", line 1110, in _run_ddp_forward
    return module_to_run(*inputs[0], **kwargs[0])  # type: ignore[index]
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
    return forward_call(*args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/overrides/base.py", line 90, in forward
    output = self._forward_module.training_step(*inputs, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/so_vits_svc_fork/train.py", line 417, in training_step
    self.manual_backward(loss_disc_all)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/core/module.py", line 1036, in manual_backward
    self.trainer.strategy.backward(loss, None, *args, **kwargs)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/strategies/strategy.py", line 195, in backward
    self.pre_backward(closure_loss)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/strategies/ddp.py", line 301, in pre_backward
    prepare_for_backward(self.model, closure_loss)
  File "/opt/conda/envs/LoRA/lib/python3.10/site-packages/lightning/pytorch/overrides/distributed.py", line 52, in prepare_for_backward
    reducer._rebuild_buckets()  # avoids "INTERNAL ASSERT FAILED" with `find_unused_parameters=False`
RuntimeError: It looks like your LightningModule has parameters that were not used in producing the loss returned by training_step. If this is intentional, you must enable the detection of unused parameters in DDP, either by setting the string value `strategy='ddp_find_unused_parameters_true'` or by setting the flag in the strategy with `strategy=DDPStrategy(find_unused_parameters=True)`


Discussion:
Could you try strategy='ddp_find_unused_parameters_true' and check if it works?

Resolving patch:
diff --git a/src/so_vits_svc_fork/train.py b/src/so_vits_svc_fork/train.py
--- a/src/so_vits_svc_fork/train.py
+++ b/src/so_vits_svc_fork/train.py
@@ -66,6 +66,10 @@ def train(
     utils.ensure_pretrained_model(model_path, hparams.model.get("type_", "hifi-gan"))
 
     datamodule = VCDataModule(hparams)
+    strategy = (
+        "ddp_find_unused_parameters_true" if torch.cuda.device_count() > 1 else "auto"
+    )
+    LOG.info(f"Using strategy: {strategy}")
     trainer = pl.Trainer(
         logger=TensorBoardLogger(model_path),
         # profiler="simple",
@@ -77,6 +81,7 @@ def train(
         else "bf16-mixed"
         if hparams.train.get("bf16_run", False)
         else 32,
+        strategy=strategy,
     )
     model = VitsLightning(reset_optimizer=reset_optimizer, **hparams)
     trainer.fit(model, datamodule=datamodule)
@@ -326,7 +331,6 @@ def training_step(self, batch: dict[str, torch.Tensor], batch_idx: int) -> None:
         )
 
         # generator loss
-        LOG.debug("Calculating generator loss")
         y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = self.net_d(y, y_hat)
 
         with autocast(enabled=False):


Test patch: