Repository: jolibrain/joliGEN

Issue:
BUG: AttributeError: 'float' object has no attribute 'clone'
On the current master I am getting the following error consistently:

Traceback (most recent call last):
  File "/workspace/joliGEN/train.py", line 445, in <module>
    launch_training(opt)
  File "/workspace/joliGEN/train.py", line 419, in launch_training
    mp.spawn(
  File "/usr/local/lib/python3.10/dist-packages/torch/multiprocessing/spawn.py", line 246, in spawn
    return start_processes(fn, args, nprocs, join, daemon, start_method="spawn")
  File "/usr/local/lib/python3.10/dist-packages/torch/multiprocessing/spawn.py", line 202, in start_processes
    while not context.join():
  File "/usr/local/lib/python3.10/dist-packages/torch/multiprocessing/spawn.py", line 163, in join
    raise ProcessRaisedException(msg, error_index, failed_process.pid)
torch.multiprocessing.spawn.ProcessRaisedException:

-- Process 0 terminated with the following error:
Traceback (most recent call last):
  File "/usr/local/lib/python3.10/dist-packages/torch/multiprocessing/spawn.py", line 74, in _wrap
    fn(i, *args)
  File "/workspace/joliGEN/train.py", line 197, in train_gpu
    model.optimize_parameters()  # calculate loss functions, get gradients, update network weights
  File "/workspace/joliGEN/models/base_model.py", line 1138, in optimize_parameters
    self.compute_step(group.optimizer, loss_names)
  File "/workspace/joliGEN/models/base_model.py", line 1029, in compute_step
    getattr(self, "loss_" + loss_name).clone()
AttributeError: 'float' object has no attribute 'clone'

root@3c5c0934a4de:/workspace/joliGEN#

AttributeError: 'float' object has no attribute 'clone'

Here is my config:
{
    "D": {
        "dropout": false,
        "n_layers": 3,
        "ndf": 64,
        "netDs": [
            "projected_d",
            "basic"
        ],
        "norm": "instance",
        "proj_interp": 1024,
        "proj_network_type": "efficientnet"
    },
    "G": {
        "attn_nb_mask_attn": 10,
        "attn_nb_mask_input": 1,
        "dropout": false,
        "nblocks": 9,
        "netG": "mobile_resnet_attn",
        "ngf": 64,
        "norm": "instance",
        "padding_type": "reflect"
    },
    "alg": {
        "gan": {
            "lambda": 1.0
        },
        "cut": {
            "HDCE_gamma": 1.0,
            "HDCE_gamma_min": 1.0,
            "MSE_idt": false,
            "flip_equivariance": false,
            "lambda_MSE_idt": 1.0,
            "lambda_NCE": 1.0,
            "lambda_SRC": 0.0,
            "nce_T": 0.07,
            "nce_idt": true,
            "nce_includes_all_negatives_from_minibatch": false,
            "nce_layers": "0,4,8,12,16",
            "nce_loss": "monce",
            "netF": "mlp_sample",
            "netF_dropout": false,
            "netF_nc": 256,
            "netF_norm": "instance",
            "num_patches": 256
        }
    },
    "data": {
        "crop_size": 256,
        "dataset_mode": "unaligned",
        "direction": "AtoB",
        "load_size": 256,
        "max_dataset_size": 1000000000,
        "num_threads": 4,
        "preprocess": "resize_and_crop"
    },
    "output": {
        "display": {
            "freq": 400,
            "id": 1,
            "ncols": 0,
            "type": [
                "visdom"
            ],
            "visdom_port": 8097,
            "visdom_server": "http://localhost",
            "winsize": 256
        },
        "no_html": false,
        "print_freq": 100,
        "update_html_freq": 1000,
        "verbose": false
    },
    "model": {
        "init_gain": 0.02,
        "init_type": "normal",
        "input_nc": 3,
        "multimodal": false,
        "output_nc": 3
    },
    "train": {
        "D_lr": 0.0001,
        "G_ema": false,
        "G_ema_beta": 0.999,
        "G_lr": 0.0002,
        "batch_size": 4,
        "beta1": 0.9,
        "beta2": 0.999,
        "continue": false,
        "epoch": "latest",
        "epoch_count": 1,
        "export_jit": false,
        "gan_mode": "lsgan",
        "iter_size": 8,
        "load_iter": 0,
        "metrics_every": 1000,
        "n_epochs": 200,
        "n_epochs_decay": 100,
        "nb_img_max_fid": 1000000000,
        "optim": "adam",
        "pool_size": 50,
        "save_by_iter": false,
        "save_epoch_freq": 1,
        "save_latest_freq": 5000
    },
    "dataaug": {
        "APA": false,
        "APA_every": 4,
        "APA_nimg": 50,
        "APA_p": 0,
        "APA_target": 0.6,
        "D_diffusion": false,
        "D_diffusion_every": 4,
        "D_label_smooth": false,
        "D_noise": 0.0,
        "affine": 0.0,
        "affine_scale_max": 1.2,
        "affine_scale_min": 0.8,
        "affine_shear": 45,
        "affine_translate": 0.2,
        "diff_aug_policy": "",
        "diff_aug_proba": 0.5,
        "imgaug": false,
        "no_flip": false,
        "no_rotate": true
    },
    "checkpoints_dir": "/root/joliGEN/checkpoints",
    "dataroot": "/root/joliGEN/datasets/ecommerce",
    "ddp_port": "12355",
    "gpu_ids": "0",
    "model_type": "cut",
    "name": "ecommerce",
    "phase": "train",
    "test_batch_size": 1,
    "warning_mode": false,
    "with_amp": false,
    "with_tf32": false,
    "with_torch_compile": false
}

I have tested in multiple environments with both an rtx 3090 and an rtx 4090 and the error is consistent. If I roll back to this commit: 811ba3d, then I can train just fine.



Discussion:
Hi, can you print the `loss_name` value in `models/base_model.py:1028` by any chance ? I doubt this is related to amp since you do not appear to have activated it. This is due to a loss not being a tensor, for some unknown reason that I've never encountered, nor does it trigger the tests...

On the side, you´d want `D_proj_interp` to be 256, there should be no benefit to have it set to a higher value.
Hi, yes I will get this printed for you with results asap. I increased as I would like to train on higher resolutions. Your tips training [here ](https://www.joligen.com/doc/tips.html) says that I should set projected D to the anticipated value, is that not correct?

![image](https://github.com/jolibrain/joliGEN/assets/104535803/c5461f4d-2b69-4a32-8358-2edae0dff73c)

Here are the loss names before the script errors:

Loss name:  G_tot
Loss name:  G_NCE
Loss name:  G_supervised
> I increased as I would like to train on higher resolutions

You are correct then :) 

Resolving patch:
diff --git a/models/cut_model.py b/models/cut_model.py
--- a/models/cut_model.py
+++ b/models/cut_model.py
@@ -366,7 +366,7 @@ def __init__(self, opt, rank):
 
         # Losses names
         losses_G = ["G_NCE"]
-        if opt.alg_cut_supervised_loss != "":
+        if opt.alg_cut_supervised_loss != [""]:
             losses_G += ["G_supervised"]
         losses_D = []
         if opt.alg_cut_nce_idt and self.isTrain:
@@ -664,11 +664,15 @@ def compute_G_loss_cut(self):
             )
         else:
             self.loss_G_supervised_lpips = 0
-        self.loss_G_supervised = self.opt.alg_cut_lambda_supervised * (
-            self.loss_G_supervised_norm + self.loss_G_supervised_lpips
-        )
 
-        self.loss_G_tot += loss_NCE_both + self.loss_G_MSE_idt + self.loss_G_supervised
+        self.loss_G_tot += loss_NCE_both + self.loss_G_MSE_idt
+
+        if self.loss_G_supervised_norm > 0 or self.loss_G_supervised_lpips > 0:
+            self.loss_G_supervised = self.opt.alg_cut_lambda_supervised * (
+                self.loss_G_supervised_norm + self.loss_G_supervised_lpips
+            )
+            self.loss_G_tot += self.loss_G_supervised
+
         self.compute_E_loss()
         self.loss_G_tot += self.loss_G_z
 


Test patch:
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
--- a/scripts/run_tests.sh
+++ b/scripts/run_tests.sh
@@ -278,7 +278,7 @@ if [ $OUT != 0 ]; then
 fi
 
 ####### pix2pix test
-echo "Running pix2pix tests"
+echo "Running pix2pix diffusion tests"
 URL=https://joligen.com/datasets/SEN2VEN_mini.zip
 ZIP_FILE=$DIR/SEN2VEN_mini.zip
 TARGET_PIX2PIX_DIR=$DIR/SEN2VEN_mini
@@ -287,6 +287,15 @@ mkdir $TARGET_PIX2PIX_DIR
 unzip $ZIP_FILE -d $DIR
 rm $ZIP_FILE
 
+echo "Running pix2pix GAN tests"
+python3 -m pytest --rootdir ${current_dir} -p no:cacheprovider -s "${current_dir}/../tests/test_run_pix2pix_gan.py" --dataroot "$TARGET_PIX2PIX_DIR"
+OUT=$?
+
+if [ $OUT != 0 ]; then
+    exit 1
+fi
+
+
 python3 -m pytest --rootdir ${current_dir} -p no:cacheprovider -s "${current_dir}/../tests/test_run_pix2pix_diffusion.py" --dataroot "$TARGET_PIX2PIX_DIR"
 OUT=$?
 
diff --git a/tests/test_run_pix2pix_gan.py b/tests/test_run_pix2pix_gan.py
new file mode 100644
--- /dev/null
+++ b/tests/test_run_pix2pix_gan.py
@@ -0,0 +1,56 @@
+import pytest
+import torch.multiprocessing as mp
+import sys
+from itertools import product
+
+sys.path.append(sys.path[0] + "/..")
+import train
+from options.train_options import TrainOptions
+from data import create_dataset
+
+json_like_dict = {
+    "name": "joligen_utest",
+    "output_display_env": "joligen_utest",
+    "output_display_id": 0,
+    "gpu_ids": "0",
+    "data_dataset_mode": "aligned",
+    "data_load_size": 256,
+    "data_crop_size": 128,
+    "train_batch_size": 1,
+    "train_n_epochs": 1,
+    "train_n_epochs_decay": 0,
+    "data_max_dataset_size": 10,
+    "data_relative_paths": True,
+    "train_G_ema": True,
+    "dataaug_no_rotate": True,
+    "G_unet_mha_num_head_channels": 16,
+    "G_unet_mha_channel_mults": [1, 2],
+    "G_nblocks": 1,
+    "G_padding_type": "reflect",
+    "dataaug_flip": "both",
+}
+
+
+models_diffusion = ["cut"]
+G_netG = ["unet_mha"]
+G_efficient = [True]
+G_unet_mha_norm_layer = [
+    "groupnorm",
+]
+product_list = product(models_diffusion, G_netG, G_efficient, G_unet_mha_norm_layer)
+
+
+def test_diffusion(dataroot):
+    json_like_dict["dataroot"] = dataroot
+    json_like_dict["checkpoints_dir"] = "/".join(dataroot.split("/")[:-1])
+    for model, Gtype, G_efficient, G_norm in product_list:
+        json_like_dict_c = json_like_dict.copy()
+        json_like_dict_c["model_type"] = model
+        json_like_dict_c["name"] += "_" + model
+        json_like_dict_c["G_netG"] = Gtype
+        json_like_dict_c["G_unet_mha_norm_layer"] = G_norm
+        json_like_dict_c["G_unet_mha_vit_efficient"] = G_efficient
+        json_like_dict_c["alg_cut_supervised_loss"] = ["L1"]
+
+        opt = TrainOptions().parse_json(json_like_dict_c, save_config=True)
+        train.launch_training(opt)