Repository: voicepaw/so-vits-svc-fork

Issue:
Error in training when the code tries deleting older saved models
**Describe the bug**
I am trying to train a new model, but when the code tries to delete older models, the models do get deleted but I also get an error that the file no longer exists (which is true since it was deleted) and the training stops. I then have to manually quit the training and restart it.


**To Reproduce**
1- Run `svc train`
2- Wait for several hundred epochs
3- Error is raised by the code when attempting to delete an older model

Here is the error I receive:
```
 INFO     [07:48:09] Removing logs/44k/G_320.pth          utils.py:327
Traceback (most recent call last):
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 42, in _call_and_handle_interrupt
    return trainer.strategy.launcher.launch(trainer_fn, *args, trainer=trainer, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/strategies/launchers/subprocess_script.py", line 92, in launch
    return function(*args, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 559, in _fit_impl
    self._run(model, ckpt_path=ckpt_path)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 935, in _run
    results = self._run_stage()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 978, in _run_stage
    self.fit_loop.run()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 201, in run
    self.advance()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/fit_loop.py", line 354, in advance
    self.epoch_loop.run(self._data_fetcher)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 134, in run
    self.on_advance_end()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/training_epoch_loop.py", line 248, in on_advance_end
    self.val_loop.run()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/utilities.py", line 177, in _decorator
    return loop_run(self, *args, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 122, in run
    return self.on_run_end()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 258, in on_run_end
    self._on_evaluation_end()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/loops/evaluation_loop.py", line 304, in _on_evaluation_end
    call._call_lightning_module_hook(trainer, hook_name, *args, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 142, in _call_lightning_module_hook
    output = fn(*args, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/so_vits_svc_fork/train.py", line 551, in on_validation_end
    self.save_checkpoints()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/so_vits_svc_fork/train.py", line 283, in save_checkpoints
    utils.clean_checkpoints(
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/so_vits_svc_fork/utils.py", line 328, in clean_checkpoints
    to_delete.unlink()
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/pathlib.py", line 1206, in unlink
    self._accessor.unlink(self)
FileNotFoundError: [Errno 2] No such file or directory: 'logs/44k/G_320.pth'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/bin/svc", line 8, in <module>
    sys.exit(cli())
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/click/core.py", line 1130, in __call__
    return self.main(*args, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/click/core.py", line 1055, in main
    rv = self.invoke(ctx)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/click/core.py", line 1657, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/click/core.py", line 760, in invoke
    return __callback(*args, **kwargs)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/so_vits_svc_fork/__main__.py", line 129, in train
    train(
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/so_vits_svc_fork/train.py", line 138, in train
    trainer.fit(model, datamodule=datamodule)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/trainer.py", line 520, in fit
    call._call_and_handle_interrupt(
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/trainer/call.py", line 65, in _call_and_handle_interrupt
    trainer.strategy.on_exception(exception)
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/strategies/ddp.py", line 395, in on_exception
    _augment_message(
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/utilities/exceptions.py", line 40, in _augment_message
    exception.args = tuple(
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/site-packages/lightning/pytorch/utilities/exceptions.py", line 41, in <genexpr>
    new_message if re.match(pattern, message, re.DOTALL) else message for message in exception.args
  File "/home/ma5679/anaconda3/envs/so-vits-svc-fork/lib/python3.10/re.py", line 190, in match
    return _compile(pattern, flags).match(string)
TypeError: expected string or bytes-like object
```





Discussion:
Do you have more than one gpus connected to your PC?
Yes, I am using 2 GPUs
workaround by changing so_vits_svc_fork/train.py line 283 with a
try:
except:
pass
block.
Also, you get an error when training on more than 1 GPU and use -t flag (Tensorboard). It throws an error that it can't bind to port (Cause GPU 1 bound to it) and hangs.

Resolving patch:
diff --git a/src/so_vits_svc_fork/train.py b/src/so_vits_svc_fork/train.py
--- a/src/so_vits_svc_fork/train.py
+++ b/src/so_vits_svc_fork/train.py
@@ -254,10 +254,20 @@ def stft(
             torch.stft = stft
 
     def on_train_end(self) -> None:
-        if not self.tuning:
-            self.save_checkpoints(adjust=0)
+        self.save_checkpoints(adjust=0)
 
     def save_checkpoints(self, adjust=1):
+        if self.tuning or self.trainer.sanity_checking:
+            return
+
+        # only save checkpoints if we are on the main device
+        if (
+            hasattr(self.device, "index")
+            and self.device.index != None
+            and self.device.index != 0
+        ):
+            return
+
         # `on_train_end` will be the actual epoch, not a -1, so we have to call it with `adjust = 0`
         current_epoch = self.current_epoch + adjust
         total_batch_idx = self.total_batch_idx - 1 + adjust
@@ -547,5 +557,4 @@ def validation_step(self, batch, batch_idx):
             )
 
     def on_validation_end(self) -> None:
-        if not self.trainer.sanity_checking and not self.tuning:
-            self.save_checkpoints()
+        self.save_checkpoints()
diff --git a/src/so_vits_svc_fork/utils.py b/src/so_vits_svc_fork/utils.py
--- a/src/so_vits_svc_fork/utils.py
+++ b/src/so_vits_svc_fork/utils.py
@@ -324,8 +324,9 @@ def clean_checkpoints(
         to_delete_list = list(group_items)[:-n_ckpts_to_keep]
 
         for to_delete in to_delete_list:
-            LOG.info(f"Removing {to_delete}")
-            to_delete.unlink()
+            if to_delete.exists():
+                LOG.info(f"Removing {to_delete}")
+                to_delete.unlink()
 
 
 def latest_checkpoint_path(dir_path: Path | str, regex: str = "G_*.pth") -> Path | None:


Test patch: