Upload 320 files

note.txt

@@ -0,0 +1,10 @@
+File Utils.ipynb bao gồm:
+- Di chuyển ảnh: Copy ngẫu nhiên 5k ảnh trong 10k ảnh được sample (cho việc tính FID 10k) từ các mô hình để tính IS 5k.
+- Tính FID 10k và IS 5k bằng thư viện torch-fidelity (https://github.com/toshas/torch-fidelity)
+
+Trong mỗi thư mục (ví dụ resnet) gồm:
+- Một thư mục "model" chứa checkpoint của mô hình cùng tên với thư mục gốc (ở đây là resnet) tại epoch thứ 30.
+- Một thư mục "log" chứa log và ảnh sample sau mỗi 1000 iter. Số lượng ảnh sample có thể không bằng nhau do ban đầu để max_epoch là 50.
+- Một tệp "DDPM_ResNet.ipynb", ở đây, ResNet chỉ là 1 ví dụ, với các mô hình khác sẽ có tên là "DDPM_ResNet_wo_t.ipynb" (mô hình Res-Net không sử dụng thời gian t), "DDPM_UNet.ipynb" (mô hình U-Net), "DDPM_UNet_wo_t.ipynb" (mô hình U-Net không có thời gian t). Trong đây sẽ tách rõ các phần của mô hình, code dùng để train, ... Mục đích chính của tệp này là dùng để huấn luyện mô hình.
+- Một tệp "DDPM_ResNet.py", tên thay đổi theo mô hình như trên. Đây chỉ là bản convert từ một tệp ".ipynb" sang ".py" do treo máy nhà qua đêm, chạy trên tệp ".py" bằng terminal sẽ nhẹ nhàng hơn.
+- Một tệp "DDPM_ResNet_sample.py", tên thay đổi theo mô hình như trên. Đây là bản chỉnh sửa từ tệp ".py", xoá hết tất cả các code về gọi data, huấn luyện, save log, ... và thay thế bằng code dùng để sample và lưu ảnh.

resnet/DDPM_ResNet.py

@@ -0,0 +1,952 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Library
+
+# In[1]:
+
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from torch.cuda.amp import autocast
+
+import torchvision
+from torchvision.transforms import transforms
+from torch.utils.data import DataLoader
+
+from torch.optim import Adam
+
+from einops import rearrange, reduce, repeat
+import math
+from random import random
+
+from collections import namedtuple
+from functools import partial
+from tqdm.auto import tqdm
+import logging
+import os
+
+from PIL import Image
+from torchvision import utils
+
+
+# # Helper
+
+# ### Constant
+
+# In[2]:
+
+
+ModelPrediction =  namedtuple('ModelPrediction', ['pred_noise', 'pred_x_start'])
+
+
+# ### Functions
+
+# In[3]:
+
+
+def exists(x):
+    return x is not None
+
+def default(val, d):
+    if exists(val):
+        return val
+    return d() if callable(d) else d
+
+
+# In[4]:
+
+
+def cast_tuple(t, length = 1):
+    if isinstance(t, tuple):
+        return t
+    return ((t,) * length)
+
+
+# In[5]:
+
+
+def divisible_by(numer, denom):
+    return (numer % denom) == 0
+
+
+# In[6]:
+
+
+def identity(t, *args, **kwargs):
+    return t
+
+
+# In[7]:
+
+
+def cycle(dl):
+    while True:
+        for data in dl:
+            yield data
+
+
+# In[8]:
+
+
+def has_int_squareroot(num):
+    return (math.sqrt(num) ** 2) == num
+
+
+# In[9]:
+
+
+def num_to_groups(num, divisor):
+    groups = num // divisor
+    remainder = num % divisor
+    arr = [divisor] * groups
+    if remainder > 0:
+        arr.append(remainder)
+    return arr
+
+
+# In[10]:
+
+
+def convert_image_to_fn(img_type, image):
+    if image.mode != img_type:
+        return image.convert(img_type)
+    return image
+
+
+# In[11]:
+
+
+def extract(a, t, x_shape):
+    b, *_ = t.shape
+    out = a.gather(-1, t)
+    return out.reshape(b, *((1,) * (len(x_shape) - 1)))
+
+
+# ### Normalization Functions
+
+# In[12]:
+
+
+def normalize_to_neg_one_to_one(img):
+    return img * 2 - 1
+
+def unnormalize_to_zero_to_one(t):
+    return (t + 1) * 0.5
+
+
+# ### Sinusoidal positional embeds
+
+# In[13]:
+
+
+class SinusoidalPosEmb(nn.Module):
+    def __init__(self, dim, theta = 10000):
+        super().__init__()
+        self.dim = dim
+        self.theta = theta
+
+    def forward(self, x):
+        device = x.device
+        half_dim = self.dim // 2
+        emb = math.log(self.theta) / (half_dim - 1)
+        emb = torch.exp(torch.arange(half_dim, device=device) * -emb)
+        emb = x[:, None] * emb[None, :]
+        emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
+        return emb
+
+
+# In[14]:
+
+
+class RandomOrLearnedSinusoidalPosEmb(nn.Module):
+    """ following @crowsonkb 's lead with random (learned optional) sinusoidal pos emb """
+    """ https://github.com/crowsonkb/v-diffusion-jax/blob/master/diffusion/models/danbooru_128.py#L8 """
+
+    def __init__(self, dim, is_random = False):
+        super().__init__()
+        assert divisible_by(dim, 2)
+        half_dim = dim // 2
+        self.weights = nn.Parameter(torch.randn(half_dim), requires_grad = not is_random)
+
+    def forward(self, x):
+        x = rearrange(x, 'b -> b 1')
+        freqs = x * rearrange(self.weights, 'd -> 1 d') * 2 * math.pi
+        fouriered = torch.cat((freqs.sin(), freqs.cos()), dim = -1)
+        fouriered = torch.cat((x, fouriered), dim = -1)
+        return fouriered
+
+
+# ### Schedule
+
+# In[15]:
+
+
+def linear_beta_schedule(timesteps):
+    """
+    linear schedule, proposed in original ddpm paper
+    """
+    scale = 1000 / timesteps
+    beta_start = scale * 0.0001
+    beta_end = scale * 0.02
+    return torch.linspace(beta_start, beta_end, timesteps, dtype = torch.float64)
+
+
+# In[16]:
+
+
+def cosine_beta_schedule(timesteps, s = 0.008):
+    """
+    cosine schedule
+    as proposed in https://openreview.net/forum?id=-NEXDKk8gZ
+    """
+    steps = timesteps + 1
+    t = torch.linspace(0, timesteps, steps, dtype = torch.float64) / timesteps
+    alphas_cumprod = torch.cos((t + s) / (1 + s) * math.pi * 0.5) ** 2
+    alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
+    betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
+    return torch.clip(betas, 0, 0.999)
+
+
+# In[17]:
+
+
+def sigmoid_beta_schedule(timesteps, start = -3, end = 3, tau = 1, clamp_min = 1e-5):
+    """
+    sigmoid schedule
+    proposed in https://arxiv.org/abs/2212.11972 - Figure 8
+    better for images > 64x64, when used during training
+    """
+    steps = timesteps + 1
+    t = torch.linspace(0, timesteps, steps, dtype = torch.float64) / timesteps
+    v_start = torch.tensor(start / tau).sigmoid()
+    v_end = torch.tensor(end / tau).sigmoid()
+    alphas_cumprod = (-((t * (end - start) + start) / tau).sigmoid() + v_end) / (v_end - v_start)
+    alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
+    betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
+    return torch.clip(betas, 0, 0.999)
+
+
+# # Diffusion model
+
+# In[18]:
+
+
+class GaussianDiffusion(nn.Module):
+    # Copy from https://github.com/lucidrains/denoising-diffusion-pytorch/blob/main/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py#L163
+
+    def __init__(
+        self,
+        model,
+        *,
+        image_size,
+        timesteps = 1000,
+        sampling_timesteps = None,
+        objective = 'pred_noise',
+        beta_schedule = 'linear',
+        schedule_fn_kwargs = dict(),
+        ddim_sampling_eta = 0.,
+        auto_normalize = True,
+        offset_noise_strength = 0.,  # https://www.crosslabs.org/blog/diffusion-with-offset-noise
+        min_snr_loss_weight = False, # https://arxiv.org/abs/2303.09556
+        min_snr_gamma = 5
+    ):
+        super().__init__()
+        assert not (type(self) == GaussianDiffusion and model.channels != model.out_dim)
+        assert not hasattr(model, 'random_or_learned_sinusoidal_cond') or not model.random_or_learned_sinusoidal_cond
+
+        self.model = model
+
+        self.channels = self.model.channels
+        self.self_condition = self.model.self_condition
+
+        self.image_size = image_size
+
+        self.objective = objective
+
+        assert objective in {'pred_noise', 'pred_x0', 'pred_v'}, 'objective must be either pred_noise (predict noise) or pred_x0 (predict image start) or pred_v (predict v [v-parameterization as defined in appendix D of progressive distillation paper, used in imagen-video successfully])'
+
+        if beta_schedule == 'linear':
+            beta_schedule_fn = linear_beta_schedule
+        elif beta_schedule == 'cosine':
+            beta_schedule_fn = cosine_beta_schedule
+        elif beta_schedule == 'sigmoid':
+            beta_schedule_fn = sigmoid_beta_schedule
+        else:
+            raise ValueError(f'unknown beta schedule {beta_schedule}')
+
+        betas = beta_schedule_fn(timesteps, **schedule_fn_kwargs)
+
+        alphas = 1. - betas
+        alphas_cumprod = torch.cumprod(alphas, dim=0)
+        alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value = 1.)
+
+        timesteps, = betas.shape
+        self.num_timesteps = int(timesteps)
+
+        # sampling related parameters
+
+        self.sampling_timesteps = default(sampling_timesteps, timesteps) # default num sampling timesteps to number of timesteps at training
+
+        assert self.sampling_timesteps <= timesteps
+        self.is_ddim_sampling = self.sampling_timesteps < timesteps
+        self.ddim_sampling_eta = ddim_sampling_eta
+
+        # helper function to register buffer from float64 to float32
+
+        register_buffer = lambda name, val: self.register_buffer(name, val.to(torch.float32))
+
+        register_buffer('betas', betas)
+        register_buffer('alphas_cumprod', alphas_cumprod)
+        register_buffer('alphas_cumprod_prev', alphas_cumprod_prev)
+
+        # calculations for diffusion q(x_t | x_{t-1}) and others
+
+        register_buffer('sqrt_alphas_cumprod', torch.sqrt(alphas_cumprod))
+        register_buffer('sqrt_one_minus_alphas_cumprod', torch.sqrt(1. - alphas_cumprod))
+        register_buffer('log_one_minus_alphas_cumprod', torch.log(1. - alphas_cumprod))
+        register_buffer('sqrt_recip_alphas_cumprod', torch.sqrt(1. / alphas_cumprod))
+        register_buffer('sqrt_recipm1_alphas_cumprod', torch.sqrt(1. / alphas_cumprod - 1))
+
+        # calculations for posterior q(x_{t-1} | x_t, x_0)
+
+        posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod)
+
+        # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t)
+
+        register_buffer('posterior_variance', posterior_variance)
+
+        # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
+
+        register_buffer('posterior_log_variance_clipped', torch.log(posterior_variance.clamp(min =1e-20)))
+        register_buffer('posterior_mean_coef1', betas * torch.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod))
+        register_buffer('posterior_mean_coef2', (1. - alphas_cumprod_prev) * torch.sqrt(alphas) / (1. - alphas_cumprod))
+
+        # offset noise strength - in blogpost, they claimed 0.1 was ideal
+
+        self.offset_noise_strength = offset_noise_strength
+
+        # derive loss weight
+        # snr - signal noise ratio
+
+        snr = alphas_cumprod / (1 - alphas_cumprod)
+
+        # https://arxiv.org/abs/2303.09556
+
+        maybe_clipped_snr = snr.clone()
+        if min_snr_loss_weight:
+            maybe_clipped_snr.clamp_(max = min_snr_gamma)
+
+        if objective == 'pred_noise':
+            register_buffer('loss_weight', maybe_clipped_snr / snr)
+        elif objective == 'pred_x0':
+            register_buffer('loss_weight', maybe_clipped_snr)
+        elif objective == 'pred_v':
+            register_buffer('loss_weight', maybe_clipped_snr / (snr + 1))
+
+        # auto-normalization of data [0, 1] -> [-1, 1] - can turn off by setting it to be False
+
+        self.normalize = normalize_to_neg_one_to_one if auto_normalize else identity
+        self.unnormalize = unnormalize_to_zero_to_one if auto_normalize else identity
+
+    @property
+    def device(self):
+        return self.betas.device
+
+    def predict_start_from_noise(self, x_t, t, noise):
+        return (
+            extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t -
+            extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise
+        )
+
+    def predict_noise_from_start(self, x_t, t, x0):
+        return (
+            (extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / \
+            extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)
+        )
+
+    def predict_v(self, x_start, t, noise):
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_start.shape) * noise -
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * x_start
+        )
+
+    def predict_start_from_v(self, x_t, t, v):
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_t.shape) * x_t -
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_t.shape) * v
+        )
+
+    def q_posterior(self, x_start, x_t, t):
+        posterior_mean = (
+            extract(self.posterior_mean_coef1, t, x_t.shape) * x_start +
+            extract(self.posterior_mean_coef2, t, x_t.shape) * x_t
+        )
+        posterior_variance = extract(self.posterior_variance, t, x_t.shape)
+        posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape)
+        return posterior_mean, posterior_variance, posterior_log_variance_clipped
+
+    def model_predictions(self, x, t, x_self_cond = None, clip_x_start = False, rederive_pred_noise = False):
+        model_output = self.model(x, t, x_self_cond)
+        maybe_clip = partial(torch.clamp, min = -1., max = 1.) if clip_x_start else identity
+
+        if self.objective == 'pred_noise':
+            pred_noise = model_output
+            x_start = self.predict_start_from_noise(x, t, pred_noise)
+            x_start = maybe_clip(x_start)
+
+            if clip_x_start and rederive_pred_noise:
+                pred_noise = self.predict_noise_from_start(x, t, x_start)
+
+        elif self.objective == 'pred_x0':
+            x_start = model_output
+            x_start = maybe_clip(x_start)
+            pred_noise = self.predict_noise_from_start(x, t, x_start)
+
+        elif self.objective == 'pred_v':
+            v = model_output
+            x_start = self.predict_start_from_v(x, t, v)
+            x_start = maybe_clip(x_start)
+            pred_noise = self.predict_noise_from_start(x, t, x_start)
+
+        return ModelPrediction(pred_noise, x_start)
+
+    def p_mean_variance(self, x, t, x_self_cond = None, clip_denoised = True):
+        preds = self.model_predictions(x, t, x_self_cond)
+        x_start = preds.pred_x_start
+
+        if clip_denoised:
+            x_start.clamp_(-1., 1.)
+
+        model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start = x_start, x_t = x, t = t)
+        return model_mean, posterior_variance, posterior_log_variance, x_start
+
+    @torch.inference_mode()
+    def p_sample(self, x, t: int, x_self_cond = None):
+        b, *_, device = *x.shape, self.device
+        batched_times = torch.full((b,), t, device = device, dtype = torch.long)
+        model_mean, _, model_log_variance, x_start = self.p_mean_variance(x = x, t = batched_times, x_self_cond = x_self_cond, clip_denoised = True)
+        noise = torch.randn_like(x) if t > 0 else 0. # no noise if t == 0
+        pred_img = model_mean + (0.5 * model_log_variance).exp() * noise
+        return pred_img, x_start
+
+    @torch.inference_mode()
+    def p_sample_loop(self, shape, return_all_timesteps = False):
+        batch, device = shape[0], self.device
+
+        img = torch.randn(shape, device = device)
+        imgs = [img]
+
+        x_start = None
+
+        for t in tqdm(reversed(range(0, self.num_timesteps)), desc = 'sampling loop time step', total = self.num_timesteps):
+            self_cond = x_start if self.self_condition else None
+            img, x_start = self.p_sample(img, t, self_cond)
+            imgs.append(img)
+
+        ret = img if not return_all_timesteps else torch.stack(imgs, dim = 1)
+
+        ret = self.unnormalize(ret)
+        return ret
+
+    @torch.inference_mode()
+    def ddim_sample(self, shape, return_all_timesteps = False):
+        batch, device, total_timesteps, sampling_timesteps, eta, objective = shape[0], self.device, self.num_timesteps, self.sampling_timesteps, self.ddim_sampling_eta, self.objective
+
+        times = torch.linspace(-1, total_timesteps - 1, steps = sampling_timesteps + 1)   # [-1, 0, 1, 2, ..., T-1] when sampling_timesteps == total_timesteps
+        times = list(reversed(times.int().tolist()))
+        time_pairs = list(zip(times[:-1], times[1:])) # [(T-1, T-2), (T-2, T-3), ..., (1, 0), (0, -1)]
+
+        img = torch.randn(shape, device = device)
+        imgs = [img]
+
+        x_start = None
+
+        for time, time_next in tqdm(time_pairs, desc = 'sampling loop time step'):
+            time_cond = torch.full((batch,), time, device = device, dtype = torch.long)
+            self_cond = x_start if self.self_condition else None
+            pred_noise, x_start, *_ = self.model_predictions(img, time_cond, self_cond, clip_x_start = True, rederive_pred_noise = True)
+
+            if time_next < 0:
+                img = x_start
+                imgs.append(img)
+                continue
+
+            alpha = self.alphas_cumprod[time]
+            alpha_next = self.alphas_cumprod[time_next]
+
+            sigma = eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt()
+            c = (1 - alpha_next - sigma ** 2).sqrt()
+
+            noise = torch.randn_like(img)
+
+            img = x_start * alpha_next.sqrt() + \
+                  c * pred_noise + \
+                  sigma * noise
+
+            imgs.append(img)
+
+        ret = img if not return_all_timesteps else torch.stack(imgs, dim = 1)
+
+        ret = self.unnormalize(ret)
+        return ret
+
+    @torch.inference_mode()
+    def sample(self, batch_size = 16, return_all_timesteps = False):
+        image_size, channels = self.image_size, self.channels
+        sample_fn = self.p_sample_loop if not self.is_ddim_sampling else self.ddim_sample
+        return sample_fn((batch_size, channels, image_size, image_size), return_all_timesteps = return_all_timesteps)
+
+    @torch.inference_mode()
+    def interpolate(self, x1, x2, t = None, lam = 0.5):
+        b, *_, device = *x1.shape, x1.device
+        t = default(t, self.num_timesteps - 1)
+
+        assert x1.shape == x2.shape
+
+        t_batched = torch.full((b,), t, device = device)
+        xt1, xt2 = map(lambda x: self.q_sample(x, t = t_batched), (x1, x2))
+
+        img = (1 - lam) * xt1 + lam * xt2
+
+        x_start = None
+
+        for i in tqdm(reversed(range(0, t)), desc = 'interpolation sample time step', total = t):
+            self_cond = x_start if self.self_condition else None
+            img, x_start = self.p_sample(img, i, self_cond)
+
+        return img
+
+    @autocast(enabled = False)
+    def q_sample(self, x_start, t, noise = None):
+        noise = default(noise, lambda: torch.randn_like(x_start))
+
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise
+        )
+
+    def p_losses(self, x_start, t, noise = None, offset_noise_strength = None):
+        b, c, h, w = x_start.shape
+
+        noise = default(noise, lambda: torch.randn_like(x_start))
+
+        # offset noise - https://www.crosslabs.org/blog/diffusion-with-offset-noise
+
+        offset_noise_strength = default(offset_noise_strength, self.offset_noise_strength)
+
+        if offset_noise_strength > 0.:
+            offset_noise = torch.randn(x_start.shape[:2], device = self.device)
+            noise += offset_noise_strength * rearrange(offset_noise, 'b c -> b c 1 1')
+
+        # noise sample
+
+        x = self.q_sample(x_start = x_start, t = t, noise = noise)
+
+        # if doing self-conditioning, 50% of the time, predict x_start from current set of times
+        # and condition with unet with that
+        # this technique will slow down training by 25%, but seems to lower FID significantly
+
+        x_self_cond = None
+        if self.self_condition and random() < 0.5:
+            with torch.no_grad():
+                x_self_cond = self.model_predictions(x, t).pred_x_start
+                x_self_cond.detach_()
+
+        # predict and take gradient step
+
+        model_out = self.model(x, t, x_self_cond)
+
+        if self.objective == 'pred_noise':
+            target = noise
+        elif self.objective == 'pred_x0':
+            target = x_start
+        elif self.objective == 'pred_v':
+            v = self.predict_v(x_start, t, noise)
+            target = v
+        else:
+            raise ValueError(f'unknown objective {self.objective}')
+
+        loss = F.mse_loss(model_out, target, reduction = 'none')
+        loss = reduce(loss, 'b ... -> b', 'mean')
+
+        loss = loss * extract(self.loss_weight, t, loss.shape)
+        return loss.mean()
+
+    def forward(self, img, *args, **kwargs):
+        b, c, h, w, device, img_size, = *img.shape, img.device, self.image_size
+        assert h == img_size and w == img_size, f'height and width of image must be {img_size}'
+        t = torch.randint(0, self.num_timesteps, (b,), device=device).long()
+
+        img = self.normalize(img)
+        return self.p_losses(img, t, *args, **kwargs)
+
+
+# # Resnet Model
+
+# In[19]:
+
+
+def default_conv(in_channels, out_channels, kernel_size, bias=True):
+    return nn.Conv2d(
+        in_channels, out_channels, kernel_size,
+        padding=(kernel_size//2), bias=bias)
+
+
+# In[20]:
+
+
+class Swish(nn.Module):
+    def forward(self, x):
+        return x * torch.sigmoid(x)
+
+
+# In[21]:
+
+
+class AttnBlock(nn.Module):
+    def __init__(self, in_ch):
+        super().__init__()
+        self.group_norm = nn.GroupNorm(32, in_ch)
+        self.proj_q = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+        self.proj_k = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+        self.proj_v = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+        self.proj = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        h = self.group_norm(x)
+        q = self.proj_q(h)
+        k = self.proj_k(h)
+        v = self.proj_v(h)
+
+        q = q.permute(0, 2, 3, 1).view(B, H * W, C)
+        k = k.view(B, C, H * W)
+        w = torch.bmm(q, k) * (int(C) ** (-0.5))
+        assert list(w.shape) == [B, H * W, H * W]
+        w = F.softmax(w, dim=-1)
+
+        v = v.permute(0, 2, 3, 1).view(B, H * W, C)
+        h = torch.bmm(w, v)
+        assert list(h.shape) == [B, H * W, C]
+        h = h.view(B, H, W, C).permute(0, 3, 1, 2)
+        h = self.proj(h)
+
+        return x + h
+
+
+# In[22]:
+
+
+class ResBlock(nn.Module):
+    def __init__(self, in_ch, out_ch, tdim, dropout, attn=False):
+        super().__init__()
+        self.block1 = nn.Sequential(
+            nn.GroupNorm(32, in_ch),
+            Swish(),
+            nn.Conv2d(in_ch, out_ch, 3, stride=1, padding=1),
+        )
+        self.temb_proj = nn.Sequential(
+            Swish(),
+            nn.Linear(tdim, out_ch),
+        )
+        self.block2 = nn.Sequential(
+            nn.GroupNorm(32, out_ch),
+            Swish(),
+            nn.Dropout(dropout),
+            nn.Conv2d(out_ch, out_ch, 3, stride=1, padding=1),
+        )
+        if in_ch != out_ch:
+            self.shortcut = nn.Conv2d(in_ch, out_ch, 1, stride=1, padding=0)
+        else:
+            self.shortcut = nn.Identity()
+        if attn:
+            self.attn = AttnBlock(out_ch)
+        else:
+            self.attn = nn.Identity()
+
+    def forward(self, x, temb):
+        h = self.block1(x)
+        h += self.temb_proj(temb)[:, :, None, None]
+        h = self.block2(h)
+
+        h = h + self.shortcut(x)
+        h = self.attn(h)
+        return h
+
+
+# In[23]:
+
+
+class EDSR(nn.Module):
+    # Modified from https://github.com/sanghyun-son/EDSR-PyTorch/blob/master/src/model/edsr.py#L31
+
+    def __init__(self,
+                 resblocks=['ResBlock', 'ResBlock', 'ResBlock', 'AttnBlock', 'AttnBlock', 'ResBlock', 'ResBlock', 'ResBlock'],
+                 n_feats=128,
+                 t_dim=256,
+                 dropout=0.1,
+                 channels=1,
+                 out_dim=1,
+                 self_condition = False,
+                 learned_sinusoidal_cond=False,
+                 random_fourier_features=False,
+                 learned_sinusoidal_dim=16,
+                 sinusoidal_pos_emb_theta=10000,
+                 conv=default_conv):
+        super(EDSR, self).__init__()
+
+        self.resblocks = resblocks
+        self.n_feats = n_feats
+        self.t_dim = t_dim
+        self.dropout = dropout
+        self.channels = channels
+        self.out_dim = out_dim
+        self.self_condition = self_condition
+        self.kernel_size = 3
+
+        # define time embedding
+        if learned_sinusoidal_cond:
+            sinu_pos_emb = RandomOrLearnedSinusoidalPosEmb(learned_sinusoidal_dim, random_fourier_features)
+            fourier_dim = learned_sinusoidal_dim + 1
+        else:
+            sinu_pos_emb = SinusoidalPosEmb(dim=self.n_feats, theta=sinusoidal_pos_emb_theta)
+            fourier_dim = self.n_feats
+
+        self.time_mlp = nn.Sequential(
+            sinu_pos_emb,
+            nn.Linear(fourier_dim, self.t_dim),
+            nn.GELU(),
+            nn.Linear(self.t_dim, self.t_dim)
+        )
+
+        # define head module
+        self.head = conv(self.channels, self.n_feats, self.kernel_size)
+
+        # define body module
+        self.body = nn.ModuleList()
+        for block in resblocks:
+            if block == "ResBlock":
+                self.body.append(
+                    ResBlock(in_ch=self.n_feats,
+                             out_ch=self.n_feats,
+                             tdim=self.t_dim,
+                             dropout=self.dropout,
+                             attn=False))
+            elif block == "AttnBlock":
+                self.body.append(
+                    ResBlock(in_ch=self.n_feats,
+                             out_ch=self.n_feats,
+                             tdim=self.t_dim,
+                             dropout=self.dropout,
+                             attn=True))
+            else:
+                raise NotImplementedError("Model currently doesn't support this kind of block!")
+        self.body.append(conv(self.n_feats, self.n_feats, self.kernel_size))
+
+        # define tail module
+        self.tail = conv(self.n_feats, self.out_dim, self.kernel_size)
+
+
+    def forward(self, x, t, cond=None):
+        t = self.time_mlp(t)
+
+        x = self.head(x)
+
+        res = x
+        for block in self.body:
+            if isinstance(block, ResBlock):
+                res = block(res, t)
+            else:
+                res = block(res)
+                res += x
+
+        x = self.tail(res)
+
+        return x
+
+
+# # Train
+
+# In[24]:
+
+
+# output dir
+save_path = 'resnet/model'
+log_path = 'resnet/log'
+
+if not os.path.exists(log_path):
+    os.mkdir(log_path)
+if not os.path.exists(save_path):
+    os.mkdir(save_path)
+
+
+# In[25]:
+
+
+# setup logging
+
+# Setup logging to file
+logging.basicConfig(
+     filename=os.path.join(log_path, 'info.log'),
+     filemode="w",
+     level=logging.DEBUG,
+     format= '[%(asctime)s] %(levelname)s - %(message)s',
+     datefmt='%H:%M:%S',
+     force=True
+ )
+
+
+# Stop PIL from printing to file
+pil_logger = logging.getLogger('PIL')
+pil_logger.setLevel(logging.INFO)
+
+# write and print at the same time
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+logging.getLogger().addHandler(console)
+
+logger = logging.getLogger('Diffusion_Resnet')
+
+
+# In[26]:
+
+
+# define model
+model = EDSR(
+    resblocks=['ResBlock', 'ResBlock', 'ResBlock', 'AttnBlock', 'AttnBlock', 
+               'AttnBlock', 'AttnBlock', 'ResBlock', 'ResBlock', 'ResBlock',],
+    n_feats=256,
+    t_dim=512,
+    dropout=0.1,
+    channels=1, # MNIST
+    out_dim=1, # MNIST
+    learned_sinusoidal_cond=False,
+    random_fourier_features=False,
+    learned_sinusoidal_dim=16,
+    sinusoidal_pos_emb_theta=10000,)
+
+diffusion_model = GaussianDiffusion(
+        model,
+        image_size=28, # MNIST
+        timesteps=1000,
+        sampling_timesteps=None,
+        objective ='pred_noise',
+        beta_schedule ='linear',
+        schedule_fn_kwargs=dict(),
+        ddim_sampling_eta= 0.,
+        auto_normalize = True,
+        offset_noise_strength = 0.,  # https://www.crosslabs.org/blog/diffusion-with-offset-noise
+        min_snr_loss_weight = False, # https://arxiv.org/abs/2303.09556
+        min_snr_gamma = 5)
+
+
+# In[27]:
+
+
+# define dataset
+transform = transforms.Compose([
+    transforms.ToTensor(),
+    # v2.Normalize((0.1307,), (0.3081,)), # https://stackoverflow.com/questions/70892017/normalize-mnist-in-pytorch
+])
+
+train_dataset = torchvision.datasets.MNIST(root='.', train=True,
+                                     download=True, transform=transform)
+# test_dataset = torchvision.datasets.MNIST(root='.', train=True,
+#                                      download=True, transform=transform)
+
+train_dataloader = DataLoader(train_dataset, batch_size=32, shuffle=True)
+# test_dataloader = DataLoader(test_dataset, batch_size=32, shuffle=True)
+
+
+# In[28]:
+
+
+# define optimizer
+train_lr = 1e-4
+adam_betas = (0.9, 0.99)
+
+optimizer = Adam(diffusion_model.parameters(),
+                 lr=train_lr,
+                 betas=adam_betas)
+
+
+# In[29]:
+
+
+# device
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+# In[30]:
+
+
+# trainer
+max_epoches = 50
+iter_print = 100
+iter_sample = 1000
+save_each = 1
+
+diffusion_model = diffusion_model.to(device)
+
+last_trained_path = None
+if last_trained_path:
+    data = torch.load(os.path.join(last_trained_path))
+    diffusion_model.load_state_dict(data['model'])
+    optimizer.load_state_dict(data['opt'])
+    count = data['step']
+    start_epoch = data['epoch']
+    log_loss = data['loss']
+else:
+    count = 0
+    start_epoch = 1
+    log_loss = []
+
+for epoch in range(start_epoch, max_epoches+1):
+    diffusion_model.train()
+    for img, _ in train_dataloader:
+        img = img.to(device)
+
+        loss = diffusion_model(img)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        if count % iter_print == 0 or count == 0:
+            logger.info('Epoch {}/{}, Iter {}: Loss = {}, lr = {}'.format(
+                epoch,
+                max_epoches,
+                count,
+                loss.mean().item(),
+                train_lr,
+            ))
+
+        log_loss.append(loss.mean().item())
+
+        loss = None
+
+        count += 1
+
+        if count % iter_sample == 0:
+            diffusion_model.eval()
+
+            sample_imgs = diffusion_model.sample(batch_size=16)
+
+            utils.save_image(sample_imgs,
+                            os.path.join(log_path, f"iter_{count}.png"),
+                            nrow = int(math.sqrt(16)))
+
+
+    if epoch % save_each == 0:
+        data = {
+            'model': diffusion_model.state_dict(),
+            'opt': optimizer.state_dict(),
+            'step': count,
+            'epoch': epoch,
+            'loss': log_loss,
+        }
+
+        torch.save(data, os.path.join(save_path, f"epoch_{epoch}.pth"))
+

resnet/DDPM_ResNet_sample.py

@@ -0,0 +1,856 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Library
+
+# In[1]:
+
+
+import torch
+import torch.nn.functional as F
+from torch import nn
+from torch.cuda.amp import autocast
+
+import torchvision
+from torchvision.transforms import transforms
+from torch.utils.data import DataLoader
+
+from torch.optim import Adam
+
+from einops import rearrange, reduce, repeat
+import math
+from random import random
+
+from collections import namedtuple
+from functools import partial
+from tqdm.auto import tqdm
+import logging
+import os
+
+from PIL import Image
+from torchvision import utils
+
+
+# # Helper
+
+# ### Constant
+
+# In[2]:
+
+
+ModelPrediction =  namedtuple('ModelPrediction', ['pred_noise', 'pred_x_start'])
+
+
+# ### Functions
+
+# In[3]:
+
+
+def exists(x):
+    return x is not None
+
+def default(val, d):
+    if exists(val):
+        return val
+    return d() if callable(d) else d
+
+
+# In[4]:
+
+
+def cast_tuple(t, length = 1):
+    if isinstance(t, tuple):
+        return t
+    return ((t,) * length)
+
+
+# In[5]:
+
+
+def divisible_by(numer, denom):
+    return (numer % denom) == 0
+
+
+# In[6]:
+
+
+def identity(t, *args, **kwargs):
+    return t
+
+
+# In[7]:
+
+
+def cycle(dl):
+    while True:
+        for data in dl:
+            yield data
+
+
+# In[8]:
+
+
+def has_int_squareroot(num):
+    return (math.sqrt(num) ** 2) == num
+
+
+# In[9]:
+
+
+def num_to_groups(num, divisor):
+    groups = num // divisor
+    remainder = num % divisor
+    arr = [divisor] * groups
+    if remainder > 0:
+        arr.append(remainder)
+    return arr
+
+
+# In[10]:
+
+
+def convert_image_to_fn(img_type, image):
+    if image.mode != img_type:
+        return image.convert(img_type)
+    return image
+
+
+# In[11]:
+
+
+def extract(a, t, x_shape):
+    b, *_ = t.shape
+    out = a.gather(-1, t)
+    return out.reshape(b, *((1,) * (len(x_shape) - 1)))
+
+
+# ### Normalization Functions
+
+# In[12]:
+
+
+def normalize_to_neg_one_to_one(img):
+    return img * 2 - 1
+
+def unnormalize_to_zero_to_one(t):
+    return (t + 1) * 0.5
+
+
+# ### Sinusoidal positional embeds
+
+# In[13]:
+
+
+class SinusoidalPosEmb(nn.Module):
+    def __init__(self, dim, theta = 10000):
+        super().__init__()
+        self.dim = dim
+        self.theta = theta
+
+    def forward(self, x):
+        device = x.device
+        half_dim = self.dim // 2
+        emb = math.log(self.theta) / (half_dim - 1)
+        emb = torch.exp(torch.arange(half_dim, device=device) * -emb)
+        emb = x[:, None] * emb[None, :]
+        emb = torch.cat((emb.sin(), emb.cos()), dim=-1)
+        return emb
+
+
+# In[14]:
+
+
+class RandomOrLearnedSinusoidalPosEmb(nn.Module):
+    """ following @crowsonkb 's lead with random (learned optional) sinusoidal pos emb """
+    """ https://github.com/crowsonkb/v-diffusion-jax/blob/master/diffusion/models/danbooru_128.py#L8 """
+
+    def __init__(self, dim, is_random = False):
+        super().__init__()
+        assert divisible_by(dim, 2)
+        half_dim = dim // 2
+        self.weights = nn.Parameter(torch.randn(half_dim), requires_grad = not is_random)
+
+    def forward(self, x):
+        x = rearrange(x, 'b -> b 1')
+        freqs = x * rearrange(self.weights, 'd -> 1 d') * 2 * math.pi
+        fouriered = torch.cat((freqs.sin(), freqs.cos()), dim = -1)
+        fouriered = torch.cat((x, fouriered), dim = -1)
+        return fouriered
+
+
+# ### Schedule
+
+# In[15]:
+
+
+def linear_beta_schedule(timesteps):
+    """
+    linear schedule, proposed in original ddpm paper
+    """
+    scale = 1000 / timesteps
+    beta_start = scale * 0.0001
+    beta_end = scale * 0.02
+    return torch.linspace(beta_start, beta_end, timesteps, dtype = torch.float64)
+
+
+# In[16]:
+
+
+def cosine_beta_schedule(timesteps, s = 0.008):
+    """
+    cosine schedule
+    as proposed in https://openreview.net/forum?id=-NEXDKk8gZ
+    """
+    steps = timesteps + 1
+    t = torch.linspace(0, timesteps, steps, dtype = torch.float64) / timesteps
+    alphas_cumprod = torch.cos((t + s) / (1 + s) * math.pi * 0.5) ** 2
+    alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
+    betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
+    return torch.clip(betas, 0, 0.999)
+
+
+# In[17]:
+
+
+def sigmoid_beta_schedule(timesteps, start = -3, end = 3, tau = 1, clamp_min = 1e-5):
+    """
+    sigmoid schedule
+    proposed in https://arxiv.org/abs/2212.11972 - Figure 8
+    better for images > 64x64, when used during training
+    """
+    steps = timesteps + 1
+    t = torch.linspace(0, timesteps, steps, dtype = torch.float64) / timesteps
+    v_start = torch.tensor(start / tau).sigmoid()
+    v_end = torch.tensor(end / tau).sigmoid()
+    alphas_cumprod = (-((t * (end - start) + start) / tau).sigmoid() + v_end) / (v_end - v_start)
+    alphas_cumprod = alphas_cumprod / alphas_cumprod[0]
+    betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1])
+    return torch.clip(betas, 0, 0.999)
+
+
+# # Diffusion model
+
+# In[18]:
+
+
+class GaussianDiffusion(nn.Module):
+    # Copy from https://github.com/lucidrains/denoising-diffusion-pytorch/blob/main/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py#L163
+
+    def __init__(
+        self,
+        model,
+        *,
+        image_size,
+        timesteps = 1000,
+        sampling_timesteps = None,
+        objective = 'pred_noise',
+        beta_schedule = 'linear',
+        schedule_fn_kwargs = dict(),
+        ddim_sampling_eta = 0.,
+        auto_normalize = True,
+        offset_noise_strength = 0.,  # https://www.crosslabs.org/blog/diffusion-with-offset-noise
+        min_snr_loss_weight = False, # https://arxiv.org/abs/2303.09556
+        min_snr_gamma = 5
+    ):
+        super().__init__()
+        assert not (type(self) == GaussianDiffusion and model.channels != model.out_dim)
+        assert not hasattr(model, 'random_or_learned_sinusoidal_cond') or not model.random_or_learned_sinusoidal_cond
+
+        self.model = model
+
+        self.channels = self.model.channels
+        self.self_condition = self.model.self_condition
+
+        self.image_size = image_size
+
+        self.objective = objective
+
+        assert objective in {'pred_noise', 'pred_x0', 'pred_v'}, 'objective must be either pred_noise (predict noise) or pred_x0 (predict image start) or pred_v (predict v [v-parameterization as defined in appendix D of progressive distillation paper, used in imagen-video successfully])'
+
+        if beta_schedule == 'linear':
+            beta_schedule_fn = linear_beta_schedule
+        elif beta_schedule == 'cosine':
+            beta_schedule_fn = cosine_beta_schedule
+        elif beta_schedule == 'sigmoid':
+            beta_schedule_fn = sigmoid_beta_schedule
+        else:
+            raise ValueError(f'unknown beta schedule {beta_schedule}')
+
+        betas = beta_schedule_fn(timesteps, **schedule_fn_kwargs)
+
+        alphas = 1. - betas
+        alphas_cumprod = torch.cumprod(alphas, dim=0)
+        alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value = 1.)
+
+        timesteps, = betas.shape
+        self.num_timesteps = int(timesteps)
+
+        # sampling related parameters
+
+        self.sampling_timesteps = default(sampling_timesteps, timesteps) # default num sampling timesteps to number of timesteps at training
+
+        assert self.sampling_timesteps <= timesteps
+        self.is_ddim_sampling = self.sampling_timesteps < timesteps
+        self.ddim_sampling_eta = ddim_sampling_eta
+
+        # helper function to register buffer from float64 to float32
+
+        register_buffer = lambda name, val: self.register_buffer(name, val.to(torch.float32))
+
+        register_buffer('betas', betas)
+        register_buffer('alphas_cumprod', alphas_cumprod)
+        register_buffer('alphas_cumprod_prev', alphas_cumprod_prev)
+
+        # calculations for diffusion q(x_t | x_{t-1}) and others
+
+        register_buffer('sqrt_alphas_cumprod', torch.sqrt(alphas_cumprod))
+        register_buffer('sqrt_one_minus_alphas_cumprod', torch.sqrt(1. - alphas_cumprod))
+        register_buffer('log_one_minus_alphas_cumprod', torch.log(1. - alphas_cumprod))
+        register_buffer('sqrt_recip_alphas_cumprod', torch.sqrt(1. / alphas_cumprod))
+        register_buffer('sqrt_recipm1_alphas_cumprod', torch.sqrt(1. / alphas_cumprod - 1))
+
+        # calculations for posterior q(x_{t-1} | x_t, x_0)
+
+        posterior_variance = betas * (1. - alphas_cumprod_prev) / (1. - alphas_cumprod)
+
+        # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t)
+
+        register_buffer('posterior_variance', posterior_variance)
+
+        # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain
+
+        register_buffer('posterior_log_variance_clipped', torch.log(posterior_variance.clamp(min =1e-20)))
+        register_buffer('posterior_mean_coef1', betas * torch.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod))
+        register_buffer('posterior_mean_coef2', (1. - alphas_cumprod_prev) * torch.sqrt(alphas) / (1. - alphas_cumprod))
+
+        # offset noise strength - in blogpost, they claimed 0.1 was ideal
+
+        self.offset_noise_strength = offset_noise_strength
+
+        # derive loss weight
+        # snr - signal noise ratio
+
+        snr = alphas_cumprod / (1 - alphas_cumprod)
+
+        # https://arxiv.org/abs/2303.09556
+
+        maybe_clipped_snr = snr.clone()
+        if min_snr_loss_weight:
+            maybe_clipped_snr.clamp_(max = min_snr_gamma)
+
+        if objective == 'pred_noise':
+            register_buffer('loss_weight', maybe_clipped_snr / snr)
+        elif objective == 'pred_x0':
+            register_buffer('loss_weight', maybe_clipped_snr)
+        elif objective == 'pred_v':
+            register_buffer('loss_weight', maybe_clipped_snr / (snr + 1))
+
+        # auto-normalization of data [0, 1] -> [-1, 1] - can turn off by setting it to be False
+
+        self.normalize = normalize_to_neg_one_to_one if auto_normalize else identity
+        self.unnormalize = unnormalize_to_zero_to_one if auto_normalize else identity
+
+    @property
+    def device(self):
+        return self.betas.device
+
+    def predict_start_from_noise(self, x_t, t, noise):
+        return (
+            extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t -
+            extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * noise
+        )
+
+    def predict_noise_from_start(self, x_t, t, x0):
+        return (
+            (extract(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / \
+            extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)
+        )
+
+    def predict_v(self, x_start, t, noise):
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_start.shape) * noise -
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * x_start
+        )
+
+    def predict_start_from_v(self, x_t, t, v):
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_t.shape) * x_t -
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_t.shape) * v
+        )
+
+    def q_posterior(self, x_start, x_t, t):
+        posterior_mean = (
+            extract(self.posterior_mean_coef1, t, x_t.shape) * x_start +
+            extract(self.posterior_mean_coef2, t, x_t.shape) * x_t
+        )
+        posterior_variance = extract(self.posterior_variance, t, x_t.shape)
+        posterior_log_variance_clipped = extract(self.posterior_log_variance_clipped, t, x_t.shape)
+        return posterior_mean, posterior_variance, posterior_log_variance_clipped
+
+    def model_predictions(self, x, t, x_self_cond = None, clip_x_start = False, rederive_pred_noise = False):
+        model_output = self.model(x, t, x_self_cond)
+        maybe_clip = partial(torch.clamp, min = -1., max = 1.) if clip_x_start else identity
+
+        if self.objective == 'pred_noise':
+            pred_noise = model_output
+            x_start = self.predict_start_from_noise(x, t, pred_noise)
+            x_start = maybe_clip(x_start)
+
+            if clip_x_start and rederive_pred_noise:
+                pred_noise = self.predict_noise_from_start(x, t, x_start)
+
+        elif self.objective == 'pred_x0':
+            x_start = model_output
+            x_start = maybe_clip(x_start)
+            pred_noise = self.predict_noise_from_start(x, t, x_start)
+
+        elif self.objective == 'pred_v':
+            v = model_output
+            x_start = self.predict_start_from_v(x, t, v)
+            x_start = maybe_clip(x_start)
+            pred_noise = self.predict_noise_from_start(x, t, x_start)
+
+        return ModelPrediction(pred_noise, x_start)
+
+    def p_mean_variance(self, x, t, x_self_cond = None, clip_denoised = True):
+        preds = self.model_predictions(x, t, x_self_cond)
+        x_start = preds.pred_x_start
+
+        if clip_denoised:
+            x_start.clamp_(-1., 1.)
+
+        model_mean, posterior_variance, posterior_log_variance = self.q_posterior(x_start = x_start, x_t = x, t = t)
+        return model_mean, posterior_variance, posterior_log_variance, x_start
+
+    @torch.inference_mode()
+    def p_sample(self, x, t: int, x_self_cond = None):
+        b, *_, device = *x.shape, self.device
+        batched_times = torch.full((b,), t, device = device, dtype = torch.long)
+        model_mean, _, model_log_variance, x_start = self.p_mean_variance(x = x, t = batched_times, x_self_cond = x_self_cond, clip_denoised = True)
+        noise = torch.randn_like(x) if t > 0 else 0. # no noise if t == 0
+        pred_img = model_mean + (0.5 * model_log_variance).exp() * noise
+        return pred_img, x_start
+
+    @torch.inference_mode()
+    def p_sample_loop(self, shape, return_all_timesteps = False):
+        batch, device = shape[0], self.device
+
+        img = torch.randn(shape, device = device)
+        imgs = [img]
+
+        x_start = None
+
+        for t in tqdm(reversed(range(0, self.num_timesteps)), desc = 'sampling loop time step', total = self.num_timesteps):
+            self_cond = x_start if self.self_condition else None
+            img, x_start = self.p_sample(img, t, self_cond)
+            imgs.append(img)
+
+        ret = img if not return_all_timesteps else torch.stack(imgs, dim = 1)
+
+        ret = self.unnormalize(ret)
+        return ret
+
+    @torch.inference_mode()
+    def ddim_sample(self, shape, return_all_timesteps = False):
+        batch, device, total_timesteps, sampling_timesteps, eta, objective = shape[0], self.device, self.num_timesteps, self.sampling_timesteps, self.ddim_sampling_eta, self.objective
+
+        times = torch.linspace(-1, total_timesteps - 1, steps = sampling_timesteps + 1)   # [-1, 0, 1, 2, ..., T-1] when sampling_timesteps == total_timesteps
+        times = list(reversed(times.int().tolist()))
+        time_pairs = list(zip(times[:-1], times[1:])) # [(T-1, T-2), (T-2, T-3), ..., (1, 0), (0, -1)]
+
+        img = torch.randn(shape, device = device)
+        imgs = [img]
+
+        x_start = None
+
+        for time, time_next in tqdm(time_pairs, desc = 'sampling loop time step'):
+            time_cond = torch.full((batch,), time, device = device, dtype = torch.long)
+            self_cond = x_start if self.self_condition else None
+            pred_noise, x_start, *_ = self.model_predictions(img, time_cond, self_cond, clip_x_start = True, rederive_pred_noise = True)
+
+            if time_next < 0:
+                img = x_start
+                imgs.append(img)
+                continue
+
+            alpha = self.alphas_cumprod[time]
+            alpha_next = self.alphas_cumprod[time_next]
+
+            sigma = eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt()
+            c = (1 - alpha_next - sigma ** 2).sqrt()
+
+            noise = torch.randn_like(img)
+
+            img = x_start * alpha_next.sqrt() + \
+                  c * pred_noise + \
+                  sigma * noise
+
+            imgs.append(img)
+
+        ret = img if not return_all_timesteps else torch.stack(imgs, dim = 1)
+
+        ret = self.unnormalize(ret)
+        return ret
+
+    @torch.inference_mode()
+    def sample(self, batch_size = 16, return_all_timesteps = False):
+        image_size, channels = self.image_size, self.channels
+        sample_fn = self.p_sample_loop if not self.is_ddim_sampling else self.ddim_sample
+        return sample_fn((batch_size, channels, image_size, image_size), return_all_timesteps = return_all_timesteps)
+
+    @torch.inference_mode()
+    def interpolate(self, x1, x2, t = None, lam = 0.5):
+        b, *_, device = *x1.shape, x1.device
+        t = default(t, self.num_timesteps - 1)
+
+        assert x1.shape == x2.shape
+
+        t_batched = torch.full((b,), t, device = device)
+        xt1, xt2 = map(lambda x: self.q_sample(x, t = t_batched), (x1, x2))
+
+        img = (1 - lam) * xt1 + lam * xt2
+
+        x_start = None
+
+        for i in tqdm(reversed(range(0, t)), desc = 'interpolation sample time step', total = t):
+            self_cond = x_start if self.self_condition else None
+            img, x_start = self.p_sample(img, i, self_cond)
+
+        return img
+
+    @autocast(enabled = False)
+    def q_sample(self, x_start, t, noise = None):
+        noise = default(noise, lambda: torch.randn_like(x_start))
+
+        return (
+            extract(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start +
+            extract(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) * noise
+        )
+
+    def p_losses(self, x_start, t, noise = None, offset_noise_strength = None):
+        b, c, h, w = x_start.shape
+
+        noise = default(noise, lambda: torch.randn_like(x_start))
+
+        # offset noise - https://www.crosslabs.org/blog/diffusion-with-offset-noise
+
+        offset_noise_strength = default(offset_noise_strength, self.offset_noise_strength)
+
+        if offset_noise_strength > 0.:
+            offset_noise = torch.randn(x_start.shape[:2], device = self.device)
+            noise += offset_noise_strength * rearrange(offset_noise, 'b c -> b c 1 1')
+
+        # noise sample
+
+        x = self.q_sample(x_start = x_start, t = t, noise = noise)
+
+        # if doing self-conditioning, 50% of the time, predict x_start from current set of times
+        # and condition with unet with that
+        # this technique will slow down training by 25%, but seems to lower FID significantly
+
+        x_self_cond = None
+        if self.self_condition and random() < 0.5:
+            with torch.no_grad():
+                x_self_cond = self.model_predictions(x, t).pred_x_start
+                x_self_cond.detach_()
+
+        # predict and take gradient step
+
+        model_out = self.model(x, t, x_self_cond)
+
+        if self.objective == 'pred_noise':
+            target = noise
+        elif self.objective == 'pred_x0':
+            target = x_start
+        elif self.objective == 'pred_v':
+            v = self.predict_v(x_start, t, noise)
+            target = v
+        else:
+            raise ValueError(f'unknown objective {self.objective}')
+
+        loss = F.mse_loss(model_out, target, reduction = 'none')
+        loss = reduce(loss, 'b ... -> b', 'mean')
+
+        loss = loss * extract(self.loss_weight, t, loss.shape)
+        return loss.mean()
+
+    def forward(self, img, *args, **kwargs):
+        b, c, h, w, device, img_size, = *img.shape, img.device, self.image_size
+        assert h == img_size and w == img_size, f'height and width of image must be {img_size}'
+        t = torch.randint(0, self.num_timesteps, (b,), device=device).long()
+
+        img = self.normalize(img)
+        return self.p_losses(img, t, *args, **kwargs)
+
+
+# # Resnet Model
+
+# In[19]:
+
+
+def default_conv(in_channels, out_channels, kernel_size, bias=True):
+    return nn.Conv2d(
+        in_channels, out_channels, kernel_size,
+        padding=(kernel_size//2), bias=bias)
+
+
+# In[20]:
+
+
+class Swish(nn.Module):
+    def forward(self, x):
+        return x * torch.sigmoid(x)
+
+
+# In[21]:
+
+
+class AttnBlock(nn.Module):
+    def __init__(self, in_ch):
+        super().__init__()
+        self.group_norm = nn.GroupNorm(32, in_ch)
+        self.proj_q = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+        self.proj_k = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+        self.proj_v = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+        self.proj = nn.Conv2d(in_ch, in_ch, 1, stride=1, padding=0)
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        h = self.group_norm(x)
+        q = self.proj_q(h)
+        k = self.proj_k(h)
+        v = self.proj_v(h)
+
+        q = q.permute(0, 2, 3, 1).view(B, H * W, C)
+        k = k.view(B, C, H * W)
+        w = torch.bmm(q, k) * (int(C) ** (-0.5))
+        assert list(w.shape) == [B, H * W, H * W]
+        w = F.softmax(w, dim=-1)
+
+        v = v.permute(0, 2, 3, 1).view(B, H * W, C)
+        h = torch.bmm(w, v)
+        assert list(h.shape) == [B, H * W, C]
+        h = h.view(B, H, W, C).permute(0, 3, 1, 2)
+        h = self.proj(h)
+
+        return x + h
+
+
+# In[22]:
+
+
+class ResBlock(nn.Module):
+    def __init__(self, in_ch, out_ch, tdim, dropout, attn=False):
+        super().__init__()
+        self.block1 = nn.Sequential(
+            nn.GroupNorm(32, in_ch),
+            Swish(),
+            nn.Conv2d(in_ch, out_ch, 3, stride=1, padding=1),
+        )
+        self.temb_proj = nn.Sequential(
+            Swish(),
+            nn.Linear(tdim, out_ch),
+        )
+        self.block2 = nn.Sequential(
+            nn.GroupNorm(32, out_ch),
+            Swish(),
+            nn.Dropout(dropout),
+            nn.Conv2d(out_ch, out_ch, 3, stride=1, padding=1),
+        )
+        if in_ch != out_ch:
+            self.shortcut = nn.Conv2d(in_ch, out_ch, 1, stride=1, padding=0)
+        else:
+            self.shortcut = nn.Identity()
+        if attn:
+            self.attn = AttnBlock(out_ch)
+        else:
+            self.attn = nn.Identity()
+
+    def forward(self, x, temb):
+        h = self.block1(x)
+        h += self.temb_proj(temb)[:, :, None, None]
+        h = self.block2(h)
+
+        h = h + self.shortcut(x)
+        h = self.attn(h)
+        return h
+
+
+# In[23]:
+
+
+class EDSR(nn.Module):
+    # Modified from https://github.com/sanghyun-son/EDSR-PyTorch/blob/master/src/model/edsr.py#L31
+
+    def __init__(self,
+                 resblocks=['ResBlock', 'ResBlock', 'ResBlock', 'AttnBlock', 'AttnBlock', 'ResBlock', 'ResBlock', 'ResBlock'],
+                 n_feats=128,
+                 t_dim=256,
+                 dropout=0.1,
+                 channels=1,
+                 out_dim=1,
+                 self_condition = False,
+                 learned_sinusoidal_cond=False,
+                 random_fourier_features=False,
+                 learned_sinusoidal_dim=16,
+                 sinusoidal_pos_emb_theta=10000,
+                 conv=default_conv):
+        super(EDSR, self).__init__()
+
+        self.resblocks = resblocks
+        self.n_feats = n_feats
+        self.t_dim = t_dim
+        self.dropout = dropout
+        self.channels = channels
+        self.out_dim = out_dim
+        self.self_condition = self_condition
+        self.kernel_size = 3
+
+        # define time embedding
+        if learned_sinusoidal_cond:
+            sinu_pos_emb = RandomOrLearnedSinusoidalPosEmb(learned_sinusoidal_dim, random_fourier_features)
+            fourier_dim = learned_sinusoidal_dim + 1
+        else:
+            sinu_pos_emb = SinusoidalPosEmb(dim=self.n_feats, theta=sinusoidal_pos_emb_theta)
+            fourier_dim = self.n_feats
+
+        self.time_mlp = nn.Sequential(
+            sinu_pos_emb,
+            nn.Linear(fourier_dim, self.t_dim),
+            nn.GELU(),
+            nn.Linear(self.t_dim, self.t_dim)
+        )
+
+        # define head module
+        self.head = conv(self.channels, self.n_feats, self.kernel_size)
+
+        # define body module
+        self.body = nn.ModuleList()
+        for block in resblocks:
+            if block == "ResBlock":
+                self.body.append(
+                    ResBlock(in_ch=self.n_feats,
+                             out_ch=self.n_feats,
+                             tdim=self.t_dim,
+                             dropout=self.dropout,
+                             attn=False))
+            elif block == "AttnBlock":
+                self.body.append(
+                    ResBlock(in_ch=self.n_feats,
+                             out_ch=self.n_feats,
+                             tdim=self.t_dim,
+                             dropout=self.dropout,
+                             attn=True))
+            else:
+                raise NotImplementedError("Model currently doesn't support this kind of block!")
+        self.body.append(conv(self.n_feats, self.n_feats, self.kernel_size))
+
+        # define tail module
+        self.tail = conv(self.n_feats, self.out_dim, self.kernel_size)
+
+
+    def forward(self, x, t, cond=None):
+        t = self.time_mlp(t)
+
+        x = self.head(x)
+
+        res = x
+        for block in self.body:
+            if isinstance(block, ResBlock):
+                res = block(res, t)
+            else:
+                res = block(res)
+                res += x
+
+        x = self.tail(res)
+
+        return x
+
+
+# # Train
+
+# In[24]:
+
+
+# In[25]:
+
+
+
+# In[26]:
+
+
+# define model
+model = EDSR(
+    resblocks=['ResBlock', 'ResBlock', 'ResBlock', 'AttnBlock', 'AttnBlock', 
+               'AttnBlock', 'AttnBlock', 'ResBlock', 'ResBlock', 'ResBlock',],
+    n_feats=256,
+    t_dim=512,
+    dropout=0.1,
+    channels=1, # MNIST
+    out_dim=1, # MNIST
+    learned_sinusoidal_cond=False,
+    random_fourier_features=False,
+    learned_sinusoidal_dim=16,
+    sinusoidal_pos_emb_theta=10000,)
+
+diffusion_model = GaussianDiffusion(
+        model,
+        image_size=28, # MNIST
+        timesteps=1000,
+        sampling_timesteps=None,
+        objective ='pred_noise',
+        beta_schedule ='linear',
+        schedule_fn_kwargs=dict(),
+        ddim_sampling_eta= 0.,
+        auto_normalize = True,
+        offset_noise_strength = 0.,  # https://www.crosslabs.org/blog/diffusion-with-offset-noise
+        min_snr_loss_weight = False, # https://arxiv.org/abs/2303.09556
+        min_snr_gamma = 5)
+
+
+# In[27]:
+
+
+# In[28]:
+
+
+
+# In[29]:
+
+
+# device
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+# In[30]:
+
+
+# trainer
+max_epoches = 50
+iter_print = 100
+iter_sample = 1000
+save_each = 1
+
+diffusion_model = diffusion_model.to(device)
+
+last_trained_path = 'resnet\model\epoch_30.pth'
+diffusion_model.load_state_dict(torch.load(os.path.join(last_trained_path))['model'])
+    
+sample_path = 'resnet/sample2'
+
+if not os.path.exists(sample_path):
+    os.mkdir(sample_path)
+
+num_sample = 500
+sample_batch = 16
+count = 0
+
+if num_sample % sample_batch != 0:
+    num_sample = num_sample + (sample_batch - (num_sample % sample_batch))
+
+for batch in range(num_sample//sample_batch):
+    imgs = diffusion_model.sample(batch_size=sample_batch, return_all_timesteps=False)
+    for i in range(imgs.size(0)):
+        torchvision.utils.save_image(imgs[i, :, :, :], os.path.join(sample_path ,f'{count}.png'))
+        count += 1
+

resnet/log/info.log

@@ -0,0 +1,585 @@
+[02:18:34] INFO - Epoch 1/50, Iter 0: Loss = 1.23480224609375, lr = 0.0001
+[02:18:59] INFO - Epoch 1/50, Iter 100: Loss = 0.08826638758182526, lr = 0.0001
+[02:19:38] INFO - Epoch 1/50, Iter 200: Loss = 0.0545695461332798, lr = 0.0001
+[02:20:35] INFO - Epoch 1/50, Iter 300: Loss = 0.06626827269792557, lr = 0.0001
+[02:21:29] INFO - Epoch 1/50, Iter 400: Loss = 0.07271286845207214, lr = 0.0001
+[02:22:24] INFO - Epoch 1/50, Iter 500: Loss = 0.027932994067668915, lr = 0.0001
+[02:23:19] INFO - Epoch 1/50, Iter 600: Loss = 0.037907857447862625, lr = 0.0001
+[02:24:14] INFO - Epoch 1/50, Iter 700: Loss = 0.03283434733748436, lr = 0.0001
+[02:25:10] INFO - Epoch 1/50, Iter 800: Loss = 0.0401763841509819, lr = 0.0001
+[02:26:05] INFO - Epoch 1/50, Iter 900: Loss = 0.02380681410431862, lr = 0.0001
+[02:28:36] INFO - Epoch 1/50, Iter 1000: Loss = 0.03142669051885605, lr = 0.0001
+[02:29:30] INFO - Epoch 1/50, Iter 1100: Loss = 0.021915458142757416, lr = 0.0001
+[02:30:25] INFO - Epoch 1/50, Iter 1200: Loss = 0.03710126131772995, lr = 0.0001
+[02:31:19] INFO - Epoch 1/50, Iter 1300: Loss = 0.017894160002470016, lr = 0.0001
+[02:32:14] INFO - Epoch 1/50, Iter 1400: Loss = 0.032229095697402954, lr = 0.0001
+[02:33:08] INFO - Epoch 1/50, Iter 1500: Loss = 0.022246181964874268, lr = 0.0001
+[02:34:04] INFO - Epoch 1/50, Iter 1600: Loss = 0.02387898601591587, lr = 0.0001
+[02:34:58] INFO - Epoch 1/50, Iter 1700: Loss = 0.033216990530490875, lr = 0.0001
+[02:35:53] INFO - Epoch 1/50, Iter 1800: Loss = 0.03182423859834671, lr = 0.0001
+[02:36:48] INFO - Epoch 2/50, Iter 1900: Loss = 0.027017910033464432, lr = 0.0001
+[02:39:20] INFO - Epoch 2/50, Iter 2000: Loss = 0.03848206251859665, lr = 0.0001
+[02:40:14] INFO - Epoch 2/50, Iter 2100: Loss = 0.02826070785522461, lr = 0.0001
+[02:41:09] INFO - Epoch 2/50, Iter 2200: Loss = 0.03657548129558563, lr = 0.0001
+[02:42:04] INFO - Epoch 2/50, Iter 2300: Loss = 0.03236750513315201, lr = 0.0001
+[02:42:59] INFO - Epoch 2/50, Iter 2400: Loss = 0.02394908107817173, lr = 0.0001
+[02:43:54] INFO - Epoch 2/50, Iter 2500: Loss = 0.028264183551073074, lr = 0.0001
+[02:44:48] INFO - Epoch 2/50, Iter 2600: Loss = 0.034485459327697754, lr = 0.0001
+[02:45:44] INFO - Epoch 2/50, Iter 2700: Loss = 0.02295440435409546, lr = 0.0001
+[02:46:38] INFO - Epoch 2/50, Iter 2800: Loss = 0.03146759420633316, lr = 0.0001
+[02:47:33] INFO - Epoch 2/50, Iter 2900: Loss = 0.022224590182304382, lr = 0.0001
+[02:50:06] INFO - Epoch 2/50, Iter 3000: Loss = 0.03717297315597534, lr = 0.0001
+[02:51:00] INFO - Epoch 2/50, Iter 3100: Loss = 0.023568114265799522, lr = 0.0001
+[02:51:55] INFO - Epoch 2/50, Iter 3200: Loss = 0.01752738654613495, lr = 0.0001
+[02:52:49] INFO - Epoch 2/50, Iter 3300: Loss = 0.024697361513972282, lr = 0.0001
+[02:53:44] INFO - Epoch 2/50, Iter 3400: Loss = 0.027621649205684662, lr = 0.0001
+[02:54:39] INFO - Epoch 2/50, Iter 3500: Loss = 0.03197108209133148, lr = 0.0001
+[02:55:34] INFO - Epoch 2/50, Iter 3600: Loss = 0.034603990614414215, lr = 0.0001
+[02:56:29] INFO - Epoch 2/50, Iter 3700: Loss = 0.024781333282589912, lr = 0.0001
+[02:57:25] INFO - Epoch 3/50, Iter 3800: Loss = 0.029720211401581764, lr = 0.0001
+[02:58:20] INFO - Epoch 3/50, Iter 3900: Loss = 0.050903625786304474, lr = 0.0001
+[03:00:52] INFO - Epoch 3/50, Iter 4000: Loss = 0.022276397794485092, lr = 0.0001
+[03:01:48] INFO - Epoch 3/50, Iter 4100: Loss = 0.02051287144422531, lr = 0.0001
+[03:02:42] INFO - Epoch 3/50, Iter 4200: Loss = 0.02138718217611313, lr = 0.0001
+[03:03:37] INFO - Epoch 3/50, Iter 4300: Loss = 0.013692906126379967, lr = 0.0001
+[03:04:31] INFO - Epoch 3/50, Iter 4400: Loss = 0.026416348293423653, lr = 0.0001
+[03:05:26] INFO - Epoch 3/50, Iter 4500: Loss = 0.02263474650681019, lr = 0.0001
+[03:06:21] INFO - Epoch 3/50, Iter 4600: Loss = 0.02561156451702118, lr = 0.0001
+[03:07:15] INFO - Epoch 3/50, Iter 4700: Loss = 0.022007182240486145, lr = 0.0001
+[03:08:11] INFO - Epoch 3/50, Iter 4800: Loss = 0.024828705936670303, lr = 0.0001
+[03:09:05] INFO - Epoch 3/50, Iter 4900: Loss = 0.0277644544839859, lr = 0.0001
+[03:11:37] INFO - Epoch 3/50, Iter 5000: Loss = 0.022669199854135513, lr = 0.0001
+[03:12:31] INFO - Epoch 3/50, Iter 5100: Loss = 0.03488582372665405, lr = 0.0001
+[03:13:27] INFO - Epoch 3/50, Iter 5200: Loss = 0.033707648515701294, lr = 0.0001
+[03:14:21] INFO - Epoch 3/50, Iter 5300: Loss = 0.034617647528648376, lr = 0.0001
+[03:15:16] INFO - Epoch 3/50, Iter 5400: Loss = 0.015979502350091934, lr = 0.0001
+[03:16:11] INFO - Epoch 3/50, Iter 5500: Loss = 0.017885394394397736, lr = 0.0001
+[03:17:05] INFO - Epoch 3/50, Iter 5600: Loss = 0.013684597797691822, lr = 0.0001
+[03:18:01] INFO - Epoch 4/50, Iter 5700: Loss = 0.018592171370983124, lr = 0.0001
+[03:18:56] INFO - Epoch 4/50, Iter 5800: Loss = 0.019852526485919952, lr = 0.0001
+[03:19:52] INFO - Epoch 4/50, Iter 5900: Loss = 0.014810988679528236, lr = 0.0001
+[03:22:25] INFO - Epoch 4/50, Iter 6000: Loss = 0.022946510463953018, lr = 0.0001
+[03:23:19] INFO - Epoch 4/50, Iter 6100: Loss = 0.022477544844150543, lr = 0.0001
+[03:24:14] INFO - Epoch 4/50, Iter 6200: Loss = 0.021514300256967545, lr = 0.0001
+[03:25:09] INFO - Epoch 4/50, Iter 6300: Loss = 0.017631331458687782, lr = 0.0001
+[03:26:03] INFO - Epoch 4/50, Iter 6400: Loss = 0.02970929630100727, lr = 0.0001
+[03:26:58] INFO - Epoch 4/50, Iter 6500: Loss = 0.02417093515396118, lr = 0.0001
+[03:27:52] INFO - Epoch 4/50, Iter 6600: Loss = 0.028470398858189583, lr = 0.0001
+[03:28:48] INFO - Epoch 4/50, Iter 6700: Loss = 0.02186693623661995, lr = 0.0001
+[03:29:42] INFO - Epoch 4/50, Iter 6800: Loss = 0.021022997796535492, lr = 0.0001
+[03:30:37] INFO - Epoch 4/50, Iter 6900: Loss = 0.02663368172943592, lr = 0.0001
+[03:33:08] INFO - Epoch 4/50, Iter 7000: Loss = 0.0202815942466259, lr = 0.0001
+[03:34:04] INFO - Epoch 4/50, Iter 7100: Loss = 0.017694229260087013, lr = 0.0001
+[03:34:58] INFO - Epoch 4/50, Iter 7200: Loss = 0.03217596560716629, lr = 0.0001
+[03:35:53] INFO - Epoch 4/50, Iter 7300: Loss = 0.027110356837511063, lr = 0.0001
+[03:36:47] INFO - Epoch 4/50, Iter 7400: Loss = 0.02598414570093155, lr = 0.0001
+[03:37:42] INFO - Epoch 5/50, Iter 7500: Loss = 0.031232168897986412, lr = 0.0001
+[03:38:37] INFO - Epoch 5/50, Iter 7600: Loss = 0.0394064262509346, lr = 0.0001
+[03:39:33] INFO - Epoch 5/50, Iter 7700: Loss = 0.017326747998595238, lr = 0.0001
+[03:40:28] INFO - Epoch 5/50, Iter 7800: Loss = 0.029284335672855377, lr = 0.0001
+[03:41:24] INFO - Epoch 5/50, Iter 7900: Loss = 0.01525358110666275, lr = 0.0001
+[03:43:56] INFO - Epoch 5/50, Iter 8000: Loss = 0.019312670454382896, lr = 0.0001
+[03:44:51] INFO - Epoch 5/50, Iter 8100: Loss = 0.022943828254938126, lr = 0.0001
+[03:45:46] INFO - Epoch 5/50, Iter 8200: Loss = 0.014834869652986526, lr = 0.0001
+[03:46:40] INFO - Epoch 5/50, Iter 8300: Loss = 0.013647425919771194, lr = 0.0001
+[03:47:35] INFO - Epoch 5/50, Iter 8400: Loss = 0.012797506526112556, lr = 0.0001
+[03:48:29] INFO - Epoch 5/50, Iter 8500: Loss = 0.028487099334597588, lr = 0.0001
+[03:49:25] INFO - Epoch 5/50, Iter 8600: Loss = 0.0326717309653759, lr = 0.0001
+[03:50:20] INFO - Epoch 5/50, Iter 8700: Loss = 0.018652349710464478, lr = 0.0001
+[03:51:14] INFO - Epoch 5/50, Iter 8800: Loss = 0.026515061035752296, lr = 0.0001
+[03:52:09] INFO - Epoch 5/50, Iter 8900: Loss = 0.02715548872947693, lr = 0.0001
+[03:54:40] INFO - Epoch 5/50, Iter 9000: Loss = 0.025071512907743454, lr = 0.0001
+[03:55:34] INFO - Epoch 5/50, Iter 9100: Loss = 0.02286442741751671, lr = 0.0001
+[03:56:29] INFO - Epoch 5/50, Iter 9200: Loss = 0.024927817285060883, lr = 0.0001
+[03:57:25] INFO - Epoch 5/50, Iter 9300: Loss = 0.02016012743115425, lr = 0.0001
+[03:58:20] INFO - Epoch 6/50, Iter 9400: Loss = 0.016080211848020554, lr = 0.0001
+[03:59:16] INFO - Epoch 6/50, Iter 9500: Loss = 0.03025580570101738, lr = 0.0001
+[04:00:10] INFO - Epoch 6/50, Iter 9600: Loss = 0.034918542951345444, lr = 0.0001
+[04:01:06] INFO - Epoch 6/50, Iter 9700: Loss = 0.024010658264160156, lr = 0.0001
+[04:02:01] INFO - Epoch 6/50, Iter 9800: Loss = 0.024768657982349396, lr = 0.0001
+[04:02:57] INFO - Epoch 6/50, Iter 9900: Loss = 0.02912471443414688, lr = 0.0001
+[04:05:28] INFO - Epoch 6/50, Iter 10000: Loss = 0.013935514725744724, lr = 0.0001
+[04:06:24] INFO - Epoch 6/50, Iter 10100: Loss = 0.024383660405874252, lr = 0.0001
+[04:07:19] INFO - Epoch 6/50, Iter 10200: Loss = 0.02626352570950985, lr = 0.0001
+[04:08:13] INFO - Epoch 6/50, Iter 10300: Loss = 0.02143704704940319, lr = 0.0001
+[04:09:08] INFO - Epoch 6/50, Iter 10400: Loss = 0.022659476846456528, lr = 0.0001
+[04:10:02] INFO - Epoch 6/50, Iter 10500: Loss = 0.020370323210954666, lr = 0.0001
+[04:10:57] INFO - Epoch 6/50, Iter 10600: Loss = 0.02100287191569805, lr = 0.0001
+[04:11:52] INFO - Epoch 6/50, Iter 10700: Loss = 0.01825377717614174, lr = 0.0001
+[04:12:46] INFO - Epoch 6/50, Iter 10800: Loss = 0.026205215603113174, lr = 0.0001
+[04:13:42] INFO - Epoch 6/50, Iter 10900: Loss = 0.03552094101905823, lr = 0.0001
+[04:16:13] INFO - Epoch 6/50, Iter 11000: Loss = 0.016668759286403656, lr = 0.0001
+[04:17:07] INFO - Epoch 6/50, Iter 11100: Loss = 0.018555857241153717, lr = 0.0001
+[04:18:02] INFO - Epoch 6/50, Iter 11200: Loss = 0.01698373258113861, lr = 0.0001
+[04:18:58] INFO - Epoch 7/50, Iter 11300: Loss = 0.021595774218440056, lr = 0.0001
+[04:19:53] INFO - Epoch 7/50, Iter 11400: Loss = 0.029402505606412888, lr = 0.0001
+[04:20:49] INFO - Epoch 7/50, Iter 11500: Loss = 0.017380326986312866, lr = 0.0001
+[04:21:44] INFO - Epoch 7/50, Iter 11600: Loss = 0.022462423890829086, lr = 0.0001
+[04:22:40] INFO - Epoch 7/50, Iter 11700: Loss = 0.024359144270420074, lr = 0.0001
+[04:23:35] INFO - Epoch 7/50, Iter 11800: Loss = 0.025637302547693253, lr = 0.0001
+[04:24:31] INFO - Epoch 7/50, Iter 11900: Loss = 0.027863897383213043, lr = 0.0001
+[04:27:02] INFO - Epoch 7/50, Iter 12000: Loss = 0.025426337495446205, lr = 0.0001
+[04:27:58] INFO - Epoch 7/50, Iter 12100: Loss = 0.03268758952617645, lr = 0.0001
+[04:28:52] INFO - Epoch 7/50, Iter 12200: Loss = 0.016548998653888702, lr = 0.0001
+[04:29:47] INFO - Epoch 7/50, Iter 12300: Loss = 0.02512863650918007, lr = 0.0001
+[04:30:41] INFO - Epoch 7/50, Iter 12400: Loss = 0.0246925987303257, lr = 0.0001
+[04:31:36] INFO - Epoch 7/50, Iter 12500: Loss = 0.018600817769765854, lr = 0.0001
+[04:32:31] INFO - Epoch 7/50, Iter 12600: Loss = 0.01979782059788704, lr = 0.0001
+[04:33:25] INFO - Epoch 7/50, Iter 12700: Loss = 0.021152257919311523, lr = 0.0001
+[04:34:21] INFO - Epoch 7/50, Iter 12800: Loss = 0.02903410792350769, lr = 0.0001
+[04:35:16] INFO - Epoch 7/50, Iter 12900: Loss = 0.03196360170841217, lr = 0.0001
+[04:37:46] INFO - Epoch 7/50, Iter 13000: Loss = 0.019338594749569893, lr = 0.0001
+[04:38:41] INFO - Epoch 7/50, Iter 13100: Loss = 0.027051424607634544, lr = 0.0001
+[04:39:37] INFO - Epoch 8/50, Iter 13200: Loss = 0.0238485224545002, lr = 0.0001
+[04:40:32] INFO - Epoch 8/50, Iter 13300: Loss = 0.02585774101316929, lr = 0.0001
+[04:41:28] INFO - Epoch 8/50, Iter 13400: Loss = 0.01865781843662262, lr = 0.0001
+[04:42:23] INFO - Epoch 8/50, Iter 13500: Loss = 0.03003603406250477, lr = 0.0001
+[04:43:17] INFO - Epoch 8/50, Iter 13600: Loss = 0.02756107971072197, lr = 0.0001
+[04:44:13] INFO - Epoch 8/50, Iter 13700: Loss = 0.018252156674861908, lr = 0.0001
+[04:45:09] INFO - Epoch 8/50, Iter 13800: Loss = 0.0232943594455719, lr = 0.0001
+[04:46:04] INFO - Epoch 8/50, Iter 13900: Loss = 0.03505060076713562, lr = 0.0001
+[04:48:36] INFO - Epoch 8/50, Iter 14000: Loss = 0.015609338879585266, lr = 0.0001
+[04:49:31] INFO - Epoch 8/50, Iter 14100: Loss = 0.024727653712034225, lr = 0.0001
+[04:50:25] INFO - Epoch 8/50, Iter 14200: Loss = 0.01343458704650402, lr = 0.0001
+[04:51:20] INFO - Epoch 8/50, Iter 14300: Loss = 0.02276020497083664, lr = 0.0001
+[04:52:15] INFO - Epoch 8/50, Iter 14400: Loss = 0.030666548758745193, lr = 0.0001
+[04:53:09] INFO - Epoch 8/50, Iter 14500: Loss = 0.027710841968655586, lr = 0.0001
+[04:54:04] INFO - Epoch 8/50, Iter 14600: Loss = 0.02813234180212021, lr = 0.0001
+[04:54:58] INFO - Epoch 8/50, Iter 14700: Loss = 0.0154835544526577, lr = 0.0001
+[04:55:54] INFO - Epoch 8/50, Iter 14800: Loss = 0.0330531969666481, lr = 0.0001
+[04:56:49] INFO - Epoch 8/50, Iter 14900: Loss = 0.02566523663699627, lr = 0.0001
+[04:59:20] INFO - Epoch 9/50, Iter 15000: Loss = 0.03587709367275238, lr = 0.0001
+[05:00:16] INFO - Epoch 9/50, Iter 15100: Loss = 0.011817749589681625, lr = 0.0001
+[05:01:11] INFO - Epoch 9/50, Iter 15200: Loss = 0.019955918192863464, lr = 0.0001
+[05:02:06] INFO - Epoch 9/50, Iter 15300: Loss = 0.01926155760884285, lr = 0.0001
+[05:03:01] INFO - Epoch 9/50, Iter 15400: Loss = 0.025760915130376816, lr = 0.0001
+[05:03:57] INFO - Epoch 9/50, Iter 15500: Loss = 0.023390091955661774, lr = 0.0001
+[05:04:52] INFO - Epoch 9/50, Iter 15600: Loss = 0.03382980450987816, lr = 0.0001
+[05:05:48] INFO - Epoch 9/50, Iter 15700: Loss = 0.019686255604028702, lr = 0.0001
+[05:06:43] INFO - Epoch 9/50, Iter 15800: Loss = 0.017689798027276993, lr = 0.0001
+[05:07:39] INFO - Epoch 9/50, Iter 15900: Loss = 0.02643013373017311, lr = 0.0001
+[05:10:10] INFO - Epoch 9/50, Iter 16000: Loss = 0.01975519210100174, lr = 0.0001
+[05:11:05] INFO - Epoch 9/50, Iter 16100: Loss = 0.02566615864634514, lr = 0.0001
+[05:12:01] INFO - Epoch 9/50, Iter 16200: Loss = 0.023744797334074974, lr = 0.0001
+[05:12:54] INFO - Epoch 9/50, Iter 16300: Loss = 0.029149867594242096, lr = 0.0001
+[05:13:50] INFO - Epoch 9/50, Iter 16400: Loss = 0.024619584903120995, lr = 0.0001
+[05:14:44] INFO - Epoch 9/50, Iter 16500: Loss = 0.017802121117711067, lr = 0.0001
+[05:15:39] INFO - Epoch 9/50, Iter 16600: Loss = 0.030343685299158096, lr = 0.0001
+[05:16:34] INFO - Epoch 9/50, Iter 16700: Loss = 0.028128691017627716, lr = 0.0001
+[05:17:28] INFO - Epoch 9/50, Iter 16800: Loss = 0.013130296021699905, lr = 0.0001
+[05:18:23] INFO - Epoch 10/50, Iter 16900: Loss = 0.015325885266065598, lr = 0.0001
+[05:20:55] INFO - Epoch 10/50, Iter 17000: Loss = 0.02369626611471176, lr = 0.0001
+[05:21:50] INFO - Epoch 10/50, Iter 17100: Loss = 0.03911880403757095, lr = 0.0001
+[05:22:44] INFO - Epoch 10/50, Iter 17200: Loss = 0.019555510953068733, lr = 0.0001
+[05:23:40] INFO - Epoch 10/50, Iter 17300: Loss = 0.026994436979293823, lr = 0.0001
+[05:24:35] INFO - Epoch 10/50, Iter 17400: Loss = 0.014918794855475426, lr = 0.0001
+[05:25:29] INFO - Epoch 10/50, Iter 17500: Loss = 0.015928588807582855, lr = 0.0001
+[05:26:24] INFO - Epoch 10/50, Iter 17600: Loss = 0.026111863553524017, lr = 0.0001
+[05:27:19] INFO - Epoch 10/50, Iter 17700: Loss = 0.023383410647511482, lr = 0.0001
+[05:28:13] INFO - Epoch 10/50, Iter 17800: Loss = 0.022820118814706802, lr = 0.0001
+[05:29:08] INFO - Epoch 10/50, Iter 17900: Loss = 0.016951140016317368, lr = 0.0001
+[05:31:40] INFO - Epoch 10/50, Iter 18000: Loss = 0.021106135100126266, lr = 0.0001
+[05:32:34] INFO - Epoch 10/50, Iter 18100: Loss = 0.015148286707699299, lr = 0.0001
+[05:33:29] INFO - Epoch 10/50, Iter 18200: Loss = 0.019842375069856644, lr = 0.0001
+[05:34:24] INFO - Epoch 10/50, Iter 18300: Loss = 0.022392811253666878, lr = 0.0001
+[05:35:18] INFO - Epoch 10/50, Iter 18400: Loss = 0.02733965963125229, lr = 0.0001
+[05:36:13] INFO - Epoch 10/50, Iter 18500: Loss = 0.02087550237774849, lr = 0.0001
+[05:37:08] INFO - Epoch 10/50, Iter 18600: Loss = 0.02672572433948517, lr = 0.0001
+[05:38:03] INFO - Epoch 10/50, Iter 18700: Loss = 0.02076902985572815, lr = 0.0001
+[05:38:59] INFO - Epoch 11/50, Iter 18800: Loss = 0.0208309106528759, lr = 0.0001
+[05:39:54] INFO - Epoch 11/50, Iter 18900: Loss = 0.01603943109512329, lr = 0.0001
+[05:42:26] INFO - Epoch 11/50, Iter 19000: Loss = 0.018146460875868797, lr = 0.0001
+[05:43:20] INFO - Epoch 11/50, Iter 19100: Loss = 0.03146671503782272, lr = 0.0001
+[05:44:15] INFO - Epoch 11/50, Iter 19200: Loss = 0.017263440415263176, lr = 0.0001
+[05:45:10] INFO - Epoch 11/50, Iter 19300: Loss = 0.021944427862763405, lr = 0.0001
+[05:46:04] INFO - Epoch 11/50, Iter 19400: Loss = 0.017847534269094467, lr = 0.0001
+[05:46:59] INFO - Epoch 11/50, Iter 19500: Loss = 0.021428382024168968, lr = 0.0001
+[05:47:55] INFO - Epoch 11/50, Iter 19600: Loss = 0.020893530920147896, lr = 0.0001
+[05:48:49] INFO - Epoch 11/50, Iter 19700: Loss = 0.02261212095618248, lr = 0.0001
+[05:49:44] INFO - Epoch 11/50, Iter 19800: Loss = 0.017424296587705612, lr = 0.0001
+[05:50:39] INFO - Epoch 11/50, Iter 19900: Loss = 0.025077205151319504, lr = 0.0001
+[05:53:10] INFO - Epoch 11/50, Iter 20000: Loss = 0.029975447803735733, lr = 0.0001
+[05:54:04] INFO - Epoch 11/50, Iter 20100: Loss = 0.019458118826150894, lr = 0.0001
+[05:54:59] INFO - Epoch 11/50, Iter 20200: Loss = 0.0232146717607975, lr = 0.0001
+[05:55:53] INFO - Epoch 11/50, Iter 20300: Loss = 0.02360851876437664, lr = 0.0001
+[05:56:48] INFO - Epoch 11/50, Iter 20400: Loss = 0.024858074262738228, lr = 0.0001
+[05:57:44] INFO - Epoch 11/50, Iter 20500: Loss = 0.044195011258125305, lr = 0.0001
+[05:58:38] INFO - Epoch 11/50, Iter 20600: Loss = 0.018540263175964355, lr = 0.0001
+[05:59:33] INFO - Epoch 12/50, Iter 20700: Loss = 0.021583855152130127, lr = 0.0001
+[06:00:29] INFO - Epoch 12/50, Iter 20800: Loss = 0.02421833947300911, lr = 0.0001
+[06:01:24] INFO - Epoch 12/50, Iter 20900: Loss = 0.026535984128713608, lr = 0.0001
+[06:03:57] INFO - Epoch 12/50, Iter 21000: Loss = 0.01781940832734108, lr = 0.0001
+[06:04:51] INFO - Epoch 12/50, Iter 21100: Loss = 0.023128725588321686, lr = 0.0001
+[06:05:46] INFO - Epoch 12/50, Iter 21200: Loss = 0.02317957766354084, lr = 0.0001
+[06:06:40] INFO - Epoch 12/50, Iter 21300: Loss = 0.016345253214240074, lr = 0.0001
+[06:07:36] INFO - Epoch 12/50, Iter 21400: Loss = 0.02558373659849167, lr = 0.0001
+[06:08:31] INFO - Epoch 12/50, Iter 21500: Loss = 0.026121504604816437, lr = 0.0001
+[06:09:25] INFO - Epoch 12/50, Iter 21600: Loss = 0.022759977728128433, lr = 0.0001
+[06:10:20] INFO - Epoch 12/50, Iter 21700: Loss = 0.026271792128682137, lr = 0.0001
+[06:11:14] INFO - Epoch 12/50, Iter 21800: Loss = 0.027187272906303406, lr = 0.0001
+[06:12:09] INFO - Epoch 12/50, Iter 21900: Loss = 0.023094702512025833, lr = 0.0001
+[06:14:40] INFO - Epoch 12/50, Iter 22000: Loss = 0.016669970005750656, lr = 0.0001
+[06:15:36] INFO - Epoch 12/50, Iter 22100: Loss = 0.026704635471105576, lr = 0.0001
+[06:16:30] INFO - Epoch 12/50, Iter 22200: Loss = 0.02754068374633789, lr = 0.0001
+[06:17:25] INFO - Epoch 12/50, Iter 22300: Loss = 0.025661129504442215, lr = 0.0001
+[06:18:19] INFO - Epoch 12/50, Iter 22400: Loss = 0.025509830564260483, lr = 0.0001
+[06:19:14] INFO - Epoch 13/50, Iter 22500: Loss = 0.025348283350467682, lr = 0.0001
+[06:20:10] INFO - Epoch 13/50, Iter 22600: Loss = 0.026772376149892807, lr = 0.0001
+[06:21:05] INFO - Epoch 13/50, Iter 22700: Loss = 0.01741105318069458, lr = 0.0001
+[06:22:01] INFO - Epoch 13/50, Iter 22800: Loss = 0.02285039983689785, lr = 0.0001
+[06:22:56] INFO - Epoch 13/50, Iter 22900: Loss = 0.027282923460006714, lr = 0.0001
+[06:25:28] INFO - Epoch 13/50, Iter 23000: Loss = 0.012414131313562393, lr = 0.0001
+[06:26:23] INFO - Epoch 13/50, Iter 23100: Loss = 0.019650613889098167, lr = 0.0001
+[06:27:18] INFO - Epoch 13/50, Iter 23200: Loss = 0.02651660516858101, lr = 0.0001
+[06:28:12] INFO - Epoch 13/50, Iter 23300: Loss = 0.026138421148061752, lr = 0.0001
+[06:29:07] INFO - Epoch 13/50, Iter 23400: Loss = 0.018627706915140152, lr = 0.0001
+[06:30:01] INFO - Epoch 13/50, Iter 23500: Loss = 0.028943434357643127, lr = 0.0001
+[06:30:57] INFO - Epoch 13/50, Iter 23600: Loss = 0.01649133488535881, lr = 0.0001
+[06:31:51] INFO - Epoch 13/50, Iter 23700: Loss = 0.01378883421421051, lr = 0.0001
+[06:32:46] INFO - Epoch 13/50, Iter 23800: Loss = 0.02124626189470291, lr = 0.0001
+[06:33:41] INFO - Epoch 13/50, Iter 23900: Loss = 0.017396021634340286, lr = 0.0001
+[06:36:12] INFO - Epoch 13/50, Iter 24000: Loss = 0.01732352189719677, lr = 0.0001
+[06:37:06] INFO - Epoch 13/50, Iter 24100: Loss = 0.014166954904794693, lr = 0.0001
+[06:38:02] INFO - Epoch 13/50, Iter 24200: Loss = 0.02176068350672722, lr = 0.0001
+[06:38:57] INFO - Epoch 13/50, Iter 24300: Loss = 0.019656777381896973, lr = 0.0001
+[06:39:51] INFO - Epoch 14/50, Iter 24400: Loss = 0.02193061262369156, lr = 0.0001
+[06:40:46] INFO - Epoch 14/50, Iter 24500: Loss = 0.018643012270331383, lr = 0.0001
+[06:41:42] INFO - Epoch 14/50, Iter 24600: Loss = 0.012337702326476574, lr = 0.0001
+[06:42:37] INFO - Epoch 14/50, Iter 24700: Loss = 0.016973398625850677, lr = 0.0001
+[06:43:33] INFO - Epoch 14/50, Iter 24800: Loss = 0.025368668138980865, lr = 0.0001
+[06:44:28] INFO - Epoch 14/50, Iter 24900: Loss = 0.02520618960261345, lr = 0.0001
+[06:47:00] INFO - Epoch 14/50, Iter 25000: Loss = 0.01767529547214508, lr = 0.0001
+[06:47:55] INFO - Epoch 14/50, Iter 25100: Loss = 0.021381141617894173, lr = 0.0001
+[06:48:49] INFO - Epoch 14/50, Iter 25200: Loss = 0.021116536110639572, lr = 0.0001
+[06:49:44] INFO - Epoch 14/50, Iter 25300: Loss = 0.017928242683410645, lr = 0.0001
+[06:50:39] INFO - Epoch 14/50, Iter 25400: Loss = 0.021284624934196472, lr = 0.0001
+[06:51:33] INFO - Epoch 14/50, Iter 25500: Loss = 0.013009730726480484, lr = 0.0001
+[06:52:28] INFO - Epoch 14/50, Iter 25600: Loss = 0.018284976482391357, lr = 0.0001
+[06:53:22] INFO - Epoch 14/50, Iter 25700: Loss = 0.019000139087438583, lr = 0.0001
+[06:54:18] INFO - Epoch 14/50, Iter 25800: Loss = 0.01757623441517353, lr = 0.0001
+[06:55:12] INFO - Epoch 14/50, Iter 25900: Loss = 0.019956346601247787, lr = 0.0001
+[06:57:43] INFO - Epoch 14/50, Iter 26000: Loss = 0.025380369275808334, lr = 0.0001
+[06:58:38] INFO - Epoch 14/50, Iter 26100: Loss = 0.02575628086924553, lr = 0.0001
+[06:59:32] INFO - Epoch 14/50, Iter 26200: Loss = 0.02441999688744545, lr = 0.0001
+[07:00:28] INFO - Epoch 15/50, Iter 26300: Loss = 0.015507195144891739, lr = 0.0001
+[07:01:23] INFO - Epoch 15/50, Iter 26400: Loss = 0.018518857657909393, lr = 0.0001
+[07:02:18] INFO - Epoch 15/50, Iter 26500: Loss = 0.0218639075756073, lr = 0.0001
+[07:03:14] INFO - Epoch 15/50, Iter 26600: Loss = 0.01484048180282116, lr = 0.0001
+[07:04:09] INFO - Epoch 15/50, Iter 26700: Loss = 0.020309407263994217, lr = 0.0001
+[07:05:05] INFO - Epoch 15/50, Iter 26800: Loss = 0.02281174622476101, lr = 0.0001
+[07:06:00] INFO - Epoch 15/50, Iter 26900: Loss = 0.022504042834043503, lr = 0.0001
+[07:08:32] INFO - Epoch 15/50, Iter 27000: Loss = 0.016440019011497498, lr = 0.0001
+[07:09:27] INFO - Epoch 15/50, Iter 27100: Loss = 0.015486285090446472, lr = 0.0001
+[07:10:21] INFO - Epoch 15/50, Iter 27200: Loss = 0.01972173899412155, lr = 0.0001
+[07:11:16] INFO - Epoch 15/50, Iter 27300: Loss = 0.018617577850818634, lr = 0.0001
+[07:12:11] INFO - Epoch 15/50, Iter 27400: Loss = 0.02082516998052597, lr = 0.0001
+[07:13:05] INFO - Epoch 15/50, Iter 27500: Loss = 0.01791219785809517, lr = 0.0001
+[07:14:00] INFO - Epoch 15/50, Iter 27600: Loss = 0.02241137996315956, lr = 0.0001
+[07:14:54] INFO - Epoch 15/50, Iter 27700: Loss = 0.020293384790420532, lr = 0.0001
+[07:15:49] INFO - Epoch 15/50, Iter 27800: Loss = 0.029861796647310257, lr = 0.0001
+[07:16:44] INFO - Epoch 15/50, Iter 27900: Loss = 0.02275857701897621, lr = 0.0001
+[07:19:16] INFO - Epoch 15/50, Iter 28000: Loss = 0.015355780720710754, lr = 0.0001
+[07:20:10] INFO - Epoch 15/50, Iter 28100: Loss = 0.019503731280565262, lr = 0.0001
+[07:21:05] INFO - Epoch 16/50, Iter 28200: Loss = 0.024656936526298523, lr = 0.0001
+[07:22:01] INFO - Epoch 16/50, Iter 28300: Loss = 0.016661042347550392, lr = 0.0001
+[07:22:56] INFO - Epoch 16/50, Iter 28400: Loss = 0.017921866849064827, lr = 0.0001
+[07:23:52] INFO - Epoch 16/50, Iter 28500: Loss = 0.020502446219325066, lr = 0.0001
+[07:24:47] INFO - Epoch 16/50, Iter 28600: Loss = 0.012834666296839714, lr = 0.0001
+[07:25:42] INFO - Epoch 16/50, Iter 28700: Loss = 0.017596762627363205, lr = 0.0001
+[07:26:37] INFO - Epoch 16/50, Iter 28800: Loss = 0.02352038025856018, lr = 0.0001
+[07:27:32] INFO - Epoch 16/50, Iter 28900: Loss = 0.022114895284175873, lr = 0.0001
+[07:30:05] INFO - Epoch 16/50, Iter 29000: Loss = 0.018584776669740677, lr = 0.0001
+[07:30:59] INFO - Epoch 16/50, Iter 29100: Loss = 0.021322712302207947, lr = 0.0001
+[07:31:54] INFO - Epoch 16/50, Iter 29200: Loss = 0.01889413595199585, lr = 0.0001
+[07:32:48] INFO - Epoch 16/50, Iter 29300: Loss = 0.027229465544223785, lr = 0.0001
+[07:33:43] INFO - Epoch 16/50, Iter 29400: Loss = 0.026700954884290695, lr = 0.0001
+[07:34:37] INFO - Epoch 16/50, Iter 29500: Loss = 0.026901915669441223, lr = 0.0001
+[07:35:32] INFO - Epoch 16/50, Iter 29600: Loss = 0.0257167499512434, lr = 0.0001
+[07:36:27] INFO - Epoch 16/50, Iter 29700: Loss = 0.023790445178747177, lr = 0.0001
+[07:37:21] INFO - Epoch 16/50, Iter 29800: Loss = 0.010275682434439659, lr = 0.0001
+[07:38:17] INFO - Epoch 16/50, Iter 29900: Loss = 0.024285804480314255, lr = 0.0001
+[07:40:48] INFO - Epoch 17/50, Iter 30000: Loss = 0.01686658337712288, lr = 0.0001
+[07:41:44] INFO - Epoch 17/50, Iter 30100: Loss = 0.019942965358495712, lr = 0.0001
+[07:42:39] INFO - Epoch 17/50, Iter 30200: Loss = 0.032290853559970856, lr = 0.0001
+[07:43:35] INFO - Epoch 17/50, Iter 30300: Loss = 0.02391435205936432, lr = 0.0001
+[07:44:29] INFO - Epoch 17/50, Iter 30400: Loss = 0.022961270064115524, lr = 0.0001
+[07:45:24] INFO - Epoch 17/50, Iter 30500: Loss = 0.02686147764325142, lr = 0.0001
+[07:46:20] INFO - Epoch 17/50, Iter 30600: Loss = 0.021469425410032272, lr = 0.0001
+[07:47:15] INFO - Epoch 17/50, Iter 30700: Loss = 0.019237644970417023, lr = 0.0001
+[07:48:11] INFO - Epoch 17/50, Iter 30800: Loss = 0.01243587676435709, lr = 0.0001
+[07:49:06] INFO - Epoch 17/50, Iter 30900: Loss = 0.019927412271499634, lr = 0.0001
+[07:51:38] INFO - Epoch 17/50, Iter 31000: Loss = 0.021345121785998344, lr = 0.0001
+[07:52:33] INFO - Epoch 17/50, Iter 31100: Loss = 0.0189402773976326, lr = 0.0001
+[07:53:28] INFO - Epoch 17/50, Iter 31200: Loss = 0.022389506921172142, lr = 0.0001
+[07:54:22] INFO - Epoch 17/50, Iter 31300: Loss = 0.019248703494668007, lr = 0.0001
+[07:55:18] INFO - Epoch 17/50, Iter 31400: Loss = 0.020908750593662262, lr = 0.0001
+[07:56:12] INFO - Epoch 17/50, Iter 31500: Loss = 0.029640033841133118, lr = 0.0001
+[07:57:07] INFO - Epoch 17/50, Iter 31600: Loss = 0.026583340018987656, lr = 0.0001
+[07:58:02] INFO - Epoch 17/50, Iter 31700: Loss = 0.01729031279683113, lr = 0.0001
+[07:58:56] INFO - Epoch 17/50, Iter 31800: Loss = 0.026669491082429886, lr = 0.0001
+[07:59:51] INFO - Epoch 18/50, Iter 31900: Loss = 0.015399916097521782, lr = 0.0001
+[08:02:23] INFO - Epoch 18/50, Iter 32000: Loss = 0.027698248624801636, lr = 0.0001
+[08:03:18] INFO - Epoch 18/50, Iter 32100: Loss = 0.020098572596907616, lr = 0.0001
+[08:04:12] INFO - Epoch 18/50, Iter 32200: Loss = 0.023418741300702095, lr = 0.0001
+[08:05:07] INFO - Epoch 18/50, Iter 32300: Loss = 0.015688564628362656, lr = 0.0001
+[08:06:02] INFO - Epoch 18/50, Iter 32400: Loss = 0.013760192319750786, lr = 0.0001
+[08:06:56] INFO - Epoch 18/50, Iter 32500: Loss = 0.018602928146719933, lr = 0.0001
+[08:07:52] INFO - Epoch 18/50, Iter 32600: Loss = 0.0171047393232584, lr = 0.0001
+[08:08:46] INFO - Epoch 18/50, Iter 32700: Loss = 0.02287128195166588, lr = 0.0001
+[08:09:41] INFO - Epoch 18/50, Iter 32800: Loss = 0.01747080124914646, lr = 0.0001
+[08:10:35] INFO - Epoch 18/50, Iter 32900: Loss = 0.032003749161958694, lr = 0.0001
+[08:13:06] INFO - Epoch 18/50, Iter 33000: Loss = 0.021088197827339172, lr = 0.0001
+[08:14:01] INFO - Epoch 18/50, Iter 33100: Loss = 0.0243061576038599, lr = 0.0001
+[08:14:55] INFO - Epoch 18/50, Iter 33200: Loss = 0.017390495166182518, lr = 0.0001
+[08:15:50] INFO - Epoch 18/50, Iter 33300: Loss = 0.027531778439879417, lr = 0.0001
+[08:16:45] INFO - Epoch 18/50, Iter 33400: Loss = 0.01495380699634552, lr = 0.0001
+[08:17:39] INFO - Epoch 18/50, Iter 33500: Loss = 0.02041369117796421, lr = 0.0001
+[08:18:35] INFO - Epoch 18/50, Iter 33600: Loss = 0.016778916120529175, lr = 0.0001
+[08:19:29] INFO - Epoch 18/50, Iter 33700: Loss = 0.0185483880341053, lr = 0.0001
+[08:20:24] INFO - Epoch 19/50, Iter 33800: Loss = 0.017258750274777412, lr = 0.0001
+[08:21:20] INFO - Epoch 19/50, Iter 33900: Loss = 0.013514120131731033, lr = 0.0001
+[08:23:52] INFO - Epoch 19/50, Iter 34000: Loss = 0.017329292371869087, lr = 0.0001
+[08:24:46] INFO - Epoch 19/50, Iter 34100: Loss = 0.03175392746925354, lr = 0.0001
+[08:25:42] INFO - Epoch 19/50, Iter 34200: Loss = 0.024144772440195084, lr = 0.0001
+[08:26:36] INFO - Epoch 19/50, Iter 34300: Loss = 0.025116432458162308, lr = 0.0001
+[08:27:31] INFO - Epoch 19/50, Iter 34400: Loss = 0.023968493565917015, lr = 0.0001
+[08:28:26] INFO - Epoch 19/50, Iter 34500: Loss = 0.023263823240995407, lr = 0.0001
+[08:29:20] INFO - Epoch 19/50, Iter 34600: Loss = 0.015572518110275269, lr = 0.0001
+[08:30:15] INFO - Epoch 19/50, Iter 34700: Loss = 0.011077907867729664, lr = 0.0001
+[08:31:10] INFO - Epoch 19/50, Iter 34800: Loss = 0.019685542210936546, lr = 0.0001
+[08:32:04] INFO - Epoch 19/50, Iter 34900: Loss = 0.026246516034007072, lr = 0.0001
+[08:34:35] INFO - Epoch 19/50, Iter 35000: Loss = 0.0264703631401062, lr = 0.0001
+[08:35:31] INFO - Epoch 19/50, Iter 35100: Loss = 0.018090050667524338, lr = 0.0001
+[08:36:25] INFO - Epoch 19/50, Iter 35200: Loss = 0.014332180842757225, lr = 0.0001
+[08:37:20] INFO - Epoch 19/50, Iter 35300: Loss = 0.03227975219488144, lr = 0.0001
+[08:38:15] INFO - Epoch 19/50, Iter 35400: Loss = 0.017180195078253746, lr = 0.0001
+[08:39:09] INFO - Epoch 19/50, Iter 35500: Loss = 0.01773938722908497, lr = 0.0001
+[08:40:04] INFO - Epoch 19/50, Iter 35600: Loss = 0.02321586385369301, lr = 0.0001
+[08:41:00] INFO - Epoch 20/50, Iter 35700: Loss = 0.018052995204925537, lr = 0.0001
+[08:41:55] INFO - Epoch 20/50, Iter 35800: Loss = 0.02333519607782364, lr = 0.0001
+[08:42:51] INFO - Epoch 20/50, Iter 35900: Loss = 0.023782718926668167, lr = 0.0001
+[08:45:22] INFO - Epoch 20/50, Iter 36000: Loss = 0.021948453038930893, lr = 0.0001
+[08:46:17] INFO - Epoch 20/50, Iter 36100: Loss = 0.01616925373673439, lr = 0.0001
+[08:47:11] INFO - Epoch 20/50, Iter 36200: Loss = 0.0195147804915905, lr = 0.0001
+[08:48:07] INFO - Epoch 20/50, Iter 36300: Loss = 0.02167724072933197, lr = 0.0001
+[08:49:02] INFO - Epoch 20/50, Iter 36400: Loss = 0.017993919551372528, lr = 0.0001
+[08:49:56] INFO - Epoch 20/50, Iter 36500: Loss = 0.024179894477128983, lr = 0.0001
+[08:50:51] INFO - Epoch 20/50, Iter 36600: Loss = 0.029972080141305923, lr = 0.0001
+[08:51:45] INFO - Epoch 20/50, Iter 36700: Loss = 0.02250525914132595, lr = 0.0001
+[08:52:40] INFO - Epoch 20/50, Iter 36800: Loss = 0.016068585216999054, lr = 0.0001
+[08:53:35] INFO - Epoch 20/50, Iter 36900: Loss = 0.02062491700053215, lr = 0.0001
+[08:56:07] INFO - Epoch 20/50, Iter 37000: Loss = 0.026054339483380318, lr = 0.0001
+[08:57:01] INFO - Epoch 20/50, Iter 37100: Loss = 0.01617574132978916, lr = 0.0001
+[08:57:56] INFO - Epoch 20/50, Iter 37200: Loss = 0.01841990277171135, lr = 0.0001
+[08:58:51] INFO - Epoch 20/50, Iter 37300: Loss = 0.016723550856113434, lr = 0.0001
+[08:59:45] INFO - Epoch 20/50, Iter 37400: Loss = 0.015482468530535698, lr = 0.0001
+[09:00:41] INFO - Epoch 21/50, Iter 37500: Loss = 0.028426745906472206, lr = 0.0001
+[09:01:36] INFO - Epoch 21/50, Iter 37600: Loss = 0.026276376098394394, lr = 0.0001
+[09:02:32] INFO - Epoch 21/50, Iter 37700: Loss = 0.026483114808797836, lr = 0.0001
+[09:03:27] INFO - Epoch 21/50, Iter 37800: Loss = 0.021477442234754562, lr = 0.0001
+[09:04:21] INFO - Epoch 21/50, Iter 37900: Loss = 0.015382439829409122, lr = 0.0001
+[09:06:54] INFO - Epoch 21/50, Iter 38000: Loss = 0.013858610764145851, lr = 0.0001
+[09:07:48] INFO - Epoch 21/50, Iter 38100: Loss = 0.022090336307883263, lr = 0.0001
+[09:08:44] INFO - Epoch 21/50, Iter 38200: Loss = 0.025041067972779274, lr = 0.0001
+[09:09:39] INFO - Epoch 21/50, Iter 38300: Loss = 0.01404337864369154, lr = 0.0001
+[09:10:33] INFO - Epoch 21/50, Iter 38400: Loss = 0.022372154518961906, lr = 0.0001
+[09:11:28] INFO - Epoch 21/50, Iter 38500: Loss = 0.022488964721560478, lr = 0.0001
+[09:12:22] INFO - Epoch 21/50, Iter 38600: Loss = 0.018394947052001953, lr = 0.0001
+[09:13:17] INFO - Epoch 21/50, Iter 38700: Loss = 0.019345279783010483, lr = 0.0001
+[09:14:12] INFO - Epoch 21/50, Iter 38800: Loss = 0.013524915091693401, lr = 0.0001
+[09:15:06] INFO - Epoch 21/50, Iter 38900: Loss = 0.023479681462049484, lr = 0.0001
+[09:17:38] INFO - Epoch 21/50, Iter 39000: Loss = 0.018239330500364304, lr = 0.0001
+[09:18:33] INFO - Epoch 21/50, Iter 39100: Loss = 0.014270618557929993, lr = 0.0001
+[09:19:27] INFO - Epoch 21/50, Iter 39200: Loss = 0.012470152229070663, lr = 0.0001
+[09:20:22] INFO - Epoch 21/50, Iter 39300: Loss = 0.024510135874152184, lr = 0.0001
+[09:21:18] INFO - Epoch 22/50, Iter 39400: Loss = 0.01967580057680607, lr = 0.0001
+[09:22:13] INFO - Epoch 22/50, Iter 39500: Loss = 0.02651473507285118, lr = 0.0001
+[09:23:09] INFO - Epoch 22/50, Iter 39600: Loss = 0.014456840232014656, lr = 0.0001
+[09:24:03] INFO - Epoch 22/50, Iter 39700: Loss = 0.013815360143780708, lr = 0.0001
+[09:24:58] INFO - Epoch 22/50, Iter 39800: Loss = 0.026865314692258835, lr = 0.0001
+[09:25:54] INFO - Epoch 22/50, Iter 39900: Loss = 0.022365324199199677, lr = 0.0001
+[09:28:27] INFO - Epoch 22/50, Iter 40000: Loss = 0.02029530331492424, lr = 0.0001
+[09:29:21] INFO - Epoch 22/50, Iter 40100: Loss = 0.021116379648447037, lr = 0.0001
+[09:30:16] INFO - Epoch 22/50, Iter 40200: Loss = 0.02509278617799282, lr = 0.0001
+[09:31:11] INFO - Epoch 22/50, Iter 40300: Loss = 0.02551993355154991, lr = 0.0001
+[09:32:05] INFO - Epoch 22/50, Iter 40400: Loss = 0.020986683666706085, lr = 0.0001
+[09:33:00] INFO - Epoch 22/50, Iter 40500: Loss = 0.020868226885795593, lr = 0.0001
+[09:33:54] INFO - Epoch 22/50, Iter 40600: Loss = 0.017478734254837036, lr = 0.0001
+[09:34:49] INFO - Epoch 22/50, Iter 40700: Loss = 0.027790624648332596, lr = 0.0001
+[09:35:45] INFO - Epoch 22/50, Iter 40800: Loss = 0.022644832730293274, lr = 0.0001
+[09:36:39] INFO - Epoch 22/50, Iter 40900: Loss = 0.024670612066984177, lr = 0.0001
+[09:39:10] INFO - Epoch 22/50, Iter 41000: Loss = 0.026195334270596504, lr = 0.0001
+[09:40:05] INFO - Epoch 22/50, Iter 41100: Loss = 0.021374046802520752, lr = 0.0001
+[09:41:00] INFO - Epoch 22/50, Iter 41200: Loss = 0.02115592733025551, lr = 0.0001
+[09:41:56] INFO - Epoch 23/50, Iter 41300: Loss = 0.01633710041642189, lr = 0.0001
+[09:42:52] INFO - Epoch 23/50, Iter 41400: Loss = 0.02131003886461258, lr = 0.0001
+[09:43:46] INFO - Epoch 23/50, Iter 41500: Loss = 0.022764872759580612, lr = 0.0001
+[09:44:41] INFO - Epoch 23/50, Iter 41600: Loss = 0.01728042960166931, lr = 0.0001
+[09:45:37] INFO - Epoch 23/50, Iter 41700: Loss = 0.0162839163094759, lr = 0.0001
+[09:46:32] INFO - Epoch 23/50, Iter 41800: Loss = 0.014318926259875298, lr = 0.0001
+[09:47:28] INFO - Epoch 23/50, Iter 41900: Loss = 0.018346164375543594, lr = 0.0001
+[09:49:59] INFO - Epoch 23/50, Iter 42000: Loss = 0.027812600135803223, lr = 0.0001
+[09:50:55] INFO - Epoch 23/50, Iter 42100: Loss = 0.026753295212984085, lr = 0.0001
+[09:51:50] INFO - Epoch 23/50, Iter 42200: Loss = 0.018069680780172348, lr = 0.0001
+[09:52:44] INFO - Epoch 23/50, Iter 42300: Loss = 0.03101518750190735, lr = 0.0001
+[09:53:39] INFO - Epoch 23/50, Iter 42400: Loss = 0.025507837533950806, lr = 0.0001
+[09:54:34] INFO - Epoch 23/50, Iter 42500: Loss = 0.017935875803232193, lr = 0.0001
+[09:55:28] INFO - Epoch 23/50, Iter 42600: Loss = 0.022867443040013313, lr = 0.0001
+[09:56:23] INFO - Epoch 23/50, Iter 42700: Loss = 0.02030709572136402, lr = 0.0001
+[09:57:18] INFO - Epoch 23/50, Iter 42800: Loss = 0.013310606591403484, lr = 0.0001
+[09:58:13] INFO - Epoch 23/50, Iter 42900: Loss = 0.014713610522449017, lr = 0.0001
+[10:00:44] INFO - Epoch 23/50, Iter 43000: Loss = 0.02300114557147026, lr = 0.0001
+[10:01:39] INFO - Epoch 23/50, Iter 43100: Loss = 0.02343389019370079, lr = 0.0001
+[10:02:35] INFO - Epoch 24/50, Iter 43200: Loss = 0.019669387489557266, lr = 0.0001
+[10:03:30] INFO - Epoch 24/50, Iter 43300: Loss = 0.025514639914035797, lr = 0.0001
+[10:04:25] INFO - Epoch 24/50, Iter 43400: Loss = 0.027034897357225418, lr = 0.0001
+[10:05:20] INFO - Epoch 24/50, Iter 43500: Loss = 0.026066435500979424, lr = 0.0001
+[10:06:16] INFO - Epoch 24/50, Iter 43600: Loss = 0.022791586816310883, lr = 0.0001
+[10:07:11] INFO - Epoch 24/50, Iter 43700: Loss = 0.01600833050906658, lr = 0.0001
+[10:08:07] INFO - Epoch 24/50, Iter 43800: Loss = 0.01834738627076149, lr = 0.0001
+[10:09:02] INFO - Epoch 24/50, Iter 43900: Loss = 0.026411669328808784, lr = 0.0001
+[10:11:34] INFO - Epoch 24/50, Iter 44000: Loss = 0.01697351410984993, lr = 0.0001
+[10:12:29] INFO - Epoch 24/50, Iter 44100: Loss = 0.025164766237139702, lr = 0.0001
+[10:13:24] INFO - Epoch 24/50, Iter 44200: Loss = 0.023120088502764702, lr = 0.0001
+[10:14:18] INFO - Epoch 24/50, Iter 44300: Loss = 0.016470227390527725, lr = 0.0001
+[10:15:13] INFO - Epoch 24/50, Iter 44400: Loss = 0.02092874050140381, lr = 0.0001
+[10:16:09] INFO - Epoch 24/50, Iter 44500: Loss = 0.017084982246160507, lr = 0.0001
+[10:17:02] INFO - Epoch 24/50, Iter 44600: Loss = 0.01771422289311886, lr = 0.0001
+[10:17:58] INFO - Epoch 24/50, Iter 44700: Loss = 0.01557396911084652, lr = 0.0001
+[10:18:52] INFO - Epoch 24/50, Iter 44800: Loss = 0.01830480992794037, lr = 0.0001
+[10:19:47] INFO - Epoch 24/50, Iter 44900: Loss = 0.03161770850419998, lr = 0.0001
+[10:22:19] INFO - Epoch 25/50, Iter 45000: Loss = 0.013423663564026356, lr = 0.0001
+[10:23:14] INFO - Epoch 25/50, Iter 45100: Loss = 0.0297955684363842, lr = 0.0001
+[10:24:10] INFO - Epoch 25/50, Iter 45200: Loss = 0.02846469357609749, lr = 0.0001
+[10:25:06] INFO - Epoch 25/50, Iter 45300: Loss = 0.015436829067766666, lr = 0.0001
+[10:26:01] INFO - Epoch 25/50, Iter 45400: Loss = 0.024918153882026672, lr = 0.0001
+[10:26:57] INFO - Epoch 25/50, Iter 45500: Loss = 0.02270306646823883, lr = 0.0001
+[10:27:52] INFO - Epoch 25/50, Iter 45600: Loss = 0.015784474089741707, lr = 0.0001
+[10:28:46] INFO - Epoch 25/50, Iter 45700: Loss = 0.011514103971421719, lr = 0.0001
+[10:29:42] INFO - Epoch 25/50, Iter 45800: Loss = 0.024075977504253387, lr = 0.0001
+[10:30:37] INFO - Epoch 25/50, Iter 45900: Loss = 0.018384993076324463, lr = 0.0001
+[10:33:10] INFO - Epoch 25/50, Iter 46000: Loss = 0.024563699960708618, lr = 0.0001
+[10:34:04] INFO - Epoch 25/50, Iter 46100: Loss = 0.015144889242947102, lr = 0.0001
+[10:34:59] INFO - Epoch 25/50, Iter 46200: Loss = 0.022055502980947495, lr = 0.0001
+[10:35:55] INFO - Epoch 25/50, Iter 46300: Loss = 0.013236483559012413, lr = 0.0001
+[10:36:49] INFO - Epoch 25/50, Iter 46400: Loss = 0.016789842396974564, lr = 0.0001
+[10:37:44] INFO - Epoch 25/50, Iter 46500: Loss = 0.018810316920280457, lr = 0.0001
+[10:38:38] INFO - Epoch 25/50, Iter 46600: Loss = 0.01891239359974861, lr = 0.0001
+[10:39:33] INFO - Epoch 25/50, Iter 46700: Loss = 0.03200780227780342, lr = 0.0001
+[10:40:28] INFO - Epoch 25/50, Iter 46800: Loss = 0.025489578023552895, lr = 0.0001
+[10:41:24] INFO - Epoch 26/50, Iter 46900: Loss = 0.02214771881699562, lr = 0.0001
+[10:43:56] INFO - Epoch 26/50, Iter 47000: Loss = 0.01889549382030964, lr = 0.0001
+[10:44:51] INFO - Epoch 26/50, Iter 47100: Loss = 0.015227919444441795, lr = 0.0001
+[10:45:45] INFO - Epoch 26/50, Iter 47200: Loss = 0.01975785568356514, lr = 0.0001
+[10:46:40] INFO - Epoch 26/50, Iter 47300: Loss = 0.021548938006162643, lr = 0.0001
+[10:47:35] INFO - Epoch 26/50, Iter 47400: Loss = 0.018300775438547134, lr = 0.0001
+[10:48:29] INFO - Epoch 26/50, Iter 47500: Loss = 0.02168145403265953, lr = 0.0001
+[10:49:24] INFO - Epoch 26/50, Iter 47600: Loss = 0.02841881290078163, lr = 0.0001
+[10:50:18] INFO - Epoch 26/50, Iter 47700: Loss = 0.01804378256201744, lr = 0.0001
+[10:51:13] INFO - Epoch 26/50, Iter 47800: Loss = 0.026898138225078583, lr = 0.0001
+[10:52:09] INFO - Epoch 26/50, Iter 47900: Loss = 0.018523452803492546, lr = 0.0001
+[10:54:40] INFO - Epoch 26/50, Iter 48000: Loss = 0.016216814517974854, lr = 0.0001
+[10:55:34] INFO - Epoch 26/50, Iter 48100: Loss = 0.02262328565120697, lr = 0.0001
+[10:56:29] INFO - Epoch 26/50, Iter 48200: Loss = 0.015000266954302788, lr = 0.0001
+[10:57:25] INFO - Epoch 26/50, Iter 48300: Loss = 0.02180442586541176, lr = 0.0001
+[10:58:20] INFO - Epoch 26/50, Iter 48400: Loss = 0.025278791785240173, lr = 0.0001
+[10:59:14] INFO - Epoch 26/50, Iter 48500: Loss = 0.03473420441150665, lr = 0.0001
+[11:00:09] INFO - Epoch 26/50, Iter 48600: Loss = 0.017245961353182793, lr = 0.0001
+[11:01:03] INFO - Epoch 26/50, Iter 48700: Loss = 0.03179230913519859, lr = 0.0001
+[11:01:59] INFO - Epoch 27/50, Iter 48800: Loss = 0.015805833041667938, lr = 0.0001
+[11:02:54] INFO - Epoch 27/50, Iter 48900: Loss = 0.02080763876438141, lr = 0.0001
+[11:05:26] INFO - Epoch 27/50, Iter 49000: Loss = 0.020735610276460648, lr = 0.0001
+[11:06:21] INFO - Epoch 27/50, Iter 49100: Loss = 0.024737179279327393, lr = 0.0001
+[11:07:16] INFO - Epoch 27/50, Iter 49200: Loss = 0.026094382628798485, lr = 0.0001
+[11:08:10] INFO - Epoch 27/50, Iter 49300: Loss = 0.021053478121757507, lr = 0.0001
+[11:09:05] INFO - Epoch 27/50, Iter 49400: Loss = 0.014476573094725609, lr = 0.0001
+[11:10:01] INFO - Epoch 27/50, Iter 49500: Loss = 0.030272990465164185, lr = 0.0001
+[11:10:55] INFO - Epoch 27/50, Iter 49600: Loss = 0.022585971280932426, lr = 0.0001
+[11:11:50] INFO - Epoch 27/50, Iter 49700: Loss = 0.01895831525325775, lr = 0.0001
+[11:12:44] INFO - Epoch 27/50, Iter 49800: Loss = 0.018344363197684288, lr = 0.0001
+[11:13:39] INFO - Epoch 27/50, Iter 49900: Loss = 0.022272832691669464, lr = 0.0001
+[11:16:10] INFO - Epoch 27/50, Iter 50000: Loss = 0.022018130868673325, lr = 0.0001
+[11:17:06] INFO - Epoch 27/50, Iter 50100: Loss = 0.027774281799793243, lr = 0.0001
+[11:18:00] INFO - Epoch 27/50, Iter 50200: Loss = 0.014724764972925186, lr = 0.0001
+[11:18:55] INFO - Epoch 27/50, Iter 50300: Loss = 0.018815312534570694, lr = 0.0001
+[11:19:50] INFO - Epoch 27/50, Iter 50400: Loss = 0.019056078046560287, lr = 0.0001
+[11:20:44] INFO - Epoch 27/50, Iter 50500: Loss = 0.01948639005422592, lr = 0.0001
+[11:21:39] INFO - Epoch 27/50, Iter 50600: Loss = 0.02332192286849022, lr = 0.0001
+[11:22:35] INFO - Epoch 28/50, Iter 50700: Loss = 0.02340688183903694, lr = 0.0001
+[11:23:31] INFO - Epoch 28/50, Iter 50800: Loss = 0.02822597697377205, lr = 0.0001
+[11:24:26] INFO - Epoch 28/50, Iter 50900: Loss = 0.02604568563401699, lr = 0.0001
+[11:26:58] INFO - Epoch 28/50, Iter 51000: Loss = 0.015130102634429932, lr = 0.0001
+[11:27:53] INFO - Epoch 28/50, Iter 51100: Loss = 0.020247958600521088, lr = 0.0001
+[11:28:47] INFO - Epoch 28/50, Iter 51200: Loss = 0.021361518651247025, lr = 0.0001
+[11:29:42] INFO - Epoch 28/50, Iter 51300: Loss = 0.0154896704480052, lr = 0.0001
+[11:30:36] INFO - Epoch 28/50, Iter 51400: Loss = 0.020418627187609673, lr = 0.0001
+[11:31:31] INFO - Epoch 28/50, Iter 51500: Loss = 0.016209501773118973, lr = 0.0001
+[11:32:26] INFO - Epoch 28/50, Iter 51600: Loss = 0.021547267213463783, lr = 0.0001
+[11:33:20] INFO - Epoch 28/50, Iter 51700: Loss = 0.03097592294216156, lr = 0.0001
+[11:34:16] INFO - Epoch 28/50, Iter 51800: Loss = 0.01853656955063343, lr = 0.0001
+[11:35:11] INFO - Epoch 28/50, Iter 51900: Loss = 0.025320153683423996, lr = 0.0001
+[11:37:42] INFO - Epoch 28/50, Iter 52000: Loss = 0.01918005384504795, lr = 0.0001
+[11:38:36] INFO - Epoch 28/50, Iter 52100: Loss = 0.02268061600625515, lr = 0.0001
+[11:39:32] INFO - Epoch 28/50, Iter 52200: Loss = 0.024810226634144783, lr = 0.0001
+[11:40:27] INFO - Epoch 28/50, Iter 52300: Loss = 0.02219560742378235, lr = 0.0001
+[11:41:21] INFO - Epoch 28/50, Iter 52400: Loss = 0.027511518448591232, lr = 0.0001
+[11:42:16] INFO - Epoch 29/50, Iter 52500: Loss = 0.016894716769456863, lr = 0.0001
+[11:43:12] INFO - Epoch 29/50, Iter 52600: Loss = 0.01918671280145645, lr = 0.0001
+[11:44:07] INFO - Epoch 29/50, Iter 52700: Loss = 0.021322811022400856, lr = 0.0001
+[11:45:03] INFO - Epoch 29/50, Iter 52800: Loss = 0.01693873107433319, lr = 0.0001
+[11:45:58] INFO - Epoch 29/50, Iter 52900: Loss = 0.028586234897375107, lr = 0.0001
+[11:48:30] INFO - Epoch 29/50, Iter 53000: Loss = 0.02094537392258644, lr = 0.0001
+[11:49:25] INFO - Epoch 29/50, Iter 53100: Loss = 0.025890830904245377, lr = 0.0001
+[11:50:20] INFO - Epoch 29/50, Iter 53200: Loss = 0.019293418154120445, lr = 0.0001
+[11:51:14] INFO - Epoch 29/50, Iter 53300: Loss = 0.013301231898367405, lr = 0.0001
+[11:52:10] INFO - Epoch 29/50, Iter 53400: Loss = 0.024367133155465126, lr = 0.0001
+[11:53:04] INFO - Epoch 29/50, Iter 53500: Loss = 0.013333385810256004, lr = 0.0001
+[11:53:59] INFO - Epoch 29/50, Iter 53600: Loss = 0.021088868379592896, lr = 0.0001
+[11:54:53] INFO - Epoch 29/50, Iter 53700: Loss = 0.014782575890421867, lr = 0.0001
+[11:55:48] INFO - Epoch 29/50, Iter 53800: Loss = 0.019235175102949142, lr = 0.0001
+[11:56:43] INFO - Epoch 29/50, Iter 53900: Loss = 0.02775110863149166, lr = 0.0001
+[11:59:15] INFO - Epoch 29/50, Iter 54000: Loss = 0.014202380552887917, lr = 0.0001
+[12:00:10] INFO - Epoch 29/50, Iter 54100: Loss = 0.021274959668517113, lr = 0.0001
+[12:01:04] INFO - Epoch 29/50, Iter 54200: Loss = 0.028708720579743385, lr = 0.0001
+[12:01:59] INFO - Epoch 29/50, Iter 54300: Loss = 0.024009495973587036, lr = 0.0001
+[12:02:55] INFO - Epoch 30/50, Iter 54400: Loss = 0.018383020535111427, lr = 0.0001
+[12:03:50] INFO - Epoch 30/50, Iter 54500: Loss = 0.012869146652519703, lr = 0.0001
+[12:04:46] INFO - Epoch 30/50, Iter 54600: Loss = 0.015052242204546928, lr = 0.0001
+[12:05:41] INFO - Epoch 30/50, Iter 54700: Loss = 0.021794060245156288, lr = 0.0001
+[12:06:37] INFO - Epoch 30/50, Iter 54800: Loss = 0.021674180403351784, lr = 0.0001
+[12:07:31] INFO - Epoch 30/50, Iter 54900: Loss = 0.0307894479483366, lr = 0.0001
+[12:10:04] INFO - Epoch 30/50, Iter 55000: Loss = 0.023494703695178032, lr = 0.0001
+[12:10:59] INFO - Epoch 30/50, Iter 55100: Loss = 0.025401834398508072, lr = 0.0001
+[12:11:54] INFO - Epoch 30/50, Iter 55200: Loss = 0.021761178970336914, lr = 0.0001
+[12:12:49] INFO - Epoch 30/50, Iter 55300: Loss = 0.02898026630282402, lr = 0.0001
+[12:13:44] INFO - Epoch 30/50, Iter 55400: Loss = 0.02216275781393051, lr = 0.0001
+[12:14:38] INFO - Epoch 30/50, Iter 55500: Loss = 0.00930317398160696, lr = 0.0001
+[12:15:33] INFO - Epoch 30/50, Iter 55600: Loss = 0.024549826979637146, lr = 0.0001
+[12:16:29] INFO - Epoch 30/50, Iter 55700: Loss = 0.016341213136911392, lr = 0.0001
+[12:17:23] INFO - Epoch 30/50, Iter 55800: Loss = 0.015864314511418343, lr = 0.0001
+[12:18:18] INFO - Epoch 30/50, Iter 55900: Loss = 0.034297745674848557, lr = 0.0001
+[12:20:49] INFO - Epoch 30/50, Iter 56000: Loss = 0.02956249937415123, lr = 0.0001
+[12:21:44] INFO - Epoch 30/50, Iter 56100: Loss = 0.02114814706146717, lr = 0.0001
+[12:22:40] INFO - Epoch 30/50, Iter 56200: Loss = 0.0200330913066864, lr = 0.0001
+[12:23:35] INFO - Epoch 31/50, Iter 56300: Loss = 0.026903297752141953, lr = 0.0001
+[12:24:31] INFO - Epoch 31/50, Iter 56400: Loss = 0.02994358167052269, lr = 0.0001
+[12:25:26] INFO - Epoch 31/50, Iter 56500: Loss = 0.016208231449127197, lr = 0.0001
+[12:26:22] INFO - Epoch 31/50, Iter 56600: Loss = 0.029720913618803024, lr = 0.0001
+[12:27:16] INFO - Epoch 31/50, Iter 56700: Loss = 0.021973680704832077, lr = 0.0001
+[12:28:11] INFO - Epoch 31/50, Iter 56800: Loss = 0.017940720543265343, lr = 0.0001
+[12:29:07] INFO - Epoch 31/50, Iter 56900: Loss = 0.022731531411409378, lr = 0.0001
+[12:31:40] INFO - Epoch 31/50, Iter 57000: Loss = 0.016729535534977913, lr = 0.0001
+[12:32:35] INFO - Epoch 31/50, Iter 57100: Loss = 0.026968562975525856, lr = 0.0001
+[12:33:29] INFO - Epoch 31/50, Iter 57200: Loss = 0.015602253377437592, lr = 0.0001
+[12:34:24] INFO - Epoch 31/50, Iter 57300: Loss = 0.028429606929421425, lr = 0.0001
+[12:35:20] INFO - Epoch 31/50, Iter 57400: Loss = 0.021183405071496964, lr = 0.0001
+[12:36:14] INFO - Epoch 31/50, Iter 57500: Loss = 0.024300210177898407, lr = 0.0001
+[12:37:09] INFO - Epoch 31/50, Iter 57600: Loss = 0.017051223665475845, lr = 0.0001
+[12:38:03] INFO - Epoch 31/50, Iter 57700: Loss = 0.016109324991703033, lr = 0.0001
+[12:38:58] INFO - Epoch 31/50, Iter 57800: Loss = 0.019427603110671043, lr = 0.0001
+[12:39:53] INFO - Epoch 31/50, Iter 57900: Loss = 0.030664775520563126, lr = 0.0001
+[12:42:25] INFO - Epoch 31/50, Iter 58000: Loss = 0.021199747920036316, lr = 0.0001
+[12:43:20] INFO - Epoch 31/50, Iter 58100: Loss = 0.01854831352829933, lr = 0.0001
+[12:44:16] INFO - Epoch 32/50, Iter 58200: Loss = 0.01928992196917534, lr = 0.0001
+[12:45:11] INFO - Epoch 32/50, Iter 58300: Loss = 0.018576214089989662, lr = 0.0001
+[12:46:00] INFO - Epoch 32/50, Iter 58400: Loss = 0.019123028963804245, lr = 0.0001