Kaggle competition: Generative Dog Images
sub = pd.read_csv(os.path.join('.. /input/jigsaw-multilingual-toxic-comment-classification/','sample_submission.csv'))
test_ids = np.load(".. /input/dataencoding/test_ids.npy", allow_pickle=True)
 sub['toxic'] = xr_model.predict(test_ids, verbose=1)
 sub.to_csv('submission.csv', index=False )<load_from_csv>
In this kernel I wrote pytorch Dataset loader that loads and preprocess all images once and stores them into RAM.
 def tic() :
start_time = time.time()
return start_time
def toc(start_time = 0):
elapsed_time = time.time() - start_time
return elapsed_time
class EasyDict(dict):
def __init__(self, d=None, **kwargs):
if d is None:
d = {}
if kwargs:
d.update(**kwargs)
 for k, v in d.items() :
setattr(self, k, v)
 for k in self.__class__.__dict__.keys() :
if not(k.startswith('__')and k.endswith('__')) and not k in('update', 'pop'):
setattr(self, k, getattr(self, k))
def __setattr__(self, name, value):
if isinstance(value,(list, tuple)) :
value = [self.__class__(x)
 if isinstance(x, dict)else x for x in value]
elif isinstance(value, dict)and not isinstance(value, self.__class__):
value = self.__class__(value)
 super(EasyDict, self ).__setattr__(name, value)
 super(EasyDict, self ).__setitem__(name, value)
 __setitem__ = __setattr__
def update(self, e=None, **f):
d = e or dict()
d.update(f)
 for k in d:
setattr(self, k, d[k])
 def pop(self, k, d=None):
delattr(self, k)
 return super(EasyDict, self ).pop(k, d)
 def _get_default_config() :
c = EasyDict()
c.data = EasyDict()
c.data.n_images = -1
c.data.sbox = True
c.data.real_label = [0.6, 0.8]
c.data.fake_label = [0.0, 0.2]
c.modelG = EasyDict()
c.modelG.name = 'generator_nroyUS'
c.modelG.params = EasyDict()
c.modelG.params.winit = 'default'
c.modelG.params.nz = 128
c.modelG.params.gchs = [1024, 512, 256, 128, 64]
c.modelG.params.upsample = 'nearest'
c.modelG.params.kernel = 3
c.modelD = EasyDict()
c.modelD.name = 'discriminator_PFN'
c.modelD.params = EasyDict()
c.modelD.params.winit = 'default'
c.modelD.params.dchs = [32, 64, 128, 256]
c.modelD.params.bns = [False, False, True, True, True]
c.modelD.params.acti = True
c.modelD.params.lrelu_slope = 0.2
c.modelD.params.bn = False
c.modelD.params.mbstd = True
c.modelD.params.nfilt = 32
c.modelD.params.kernel = 5
c.train = EasyDict()
c.train.batch_size = 32
c.train.epoch = 2000
c.train.log_step = 100
c.train.params = EasyDict()
c.train.params.use_averaging_model = True
c.train.params.ema_alpha = 0.9999
c.train.params.ma_start = 100
c.train.params.ema = True
c.eval = EasyDict()
c.eval.batch_size = 50
c.optimizerG = EasyDict()
c.optimizerG.name = 'adam'
c.optimizerG.params = EasyDict()
c.optimizerG.params.lr = 0.0005
c.optimizerG.params.beta1 = 0.5
c.optimizerG.params.beta2 = 0.999
c.optimizerD = EasyDict()
c.optimizerD.name = 'adam'
c.optimizerD.params = EasyDict()
c.optimizerD.params.lr = 0.0005
c.optimizerD.params.beta1 = 0.5
c.optimizerD.params.beta2 = 0.999
c.loss = EasyDict()
c.loss.name = 'sGAN'
c.scheduler = EasyDict()
c.scheduler.name = 'none'
c.scheduler.params = EasyDict()
return c
config = _get_default_config()
pprint.PrettyPrinter(indent=2 ).pprint(config)
DATA_PATH = "/kaggle/input/jigsaw-multilingual-toxic-comment-classification"
val_df = pd.read_csv(os.path.join(DATA_PATH, "validation.csv"))
test_df = pd.read_csv(os.path.join(DATA_PATH, "test.csv"))
test_en_df = pd.read_csv(".. /input/test-en-df/test_en.csv")
 val_ids_df = pd.read_csv(os.path.join(DATA_PATH, "validation-processed-seqlen128.csv"))
test_ids_df = pd.read_csv(os.path.join(DATA_PATH, "test-processed-seqlen128.csv"))
test_ids_df = pd.merge(test_ids_df, test_df[["id", "lang"]], on="id" )<filter>
if True:
root_dir = '.. /input/generative-dog-images'
else:
root_dir = '.. /input'
eval_do = False
use_scheduler = False
if config.data.sbox:
exrw = 1
exrh = 1
else:
exrw = 0
exrh = 0
n_images = config.data.n_images
g_model = config.modelG.name
d_model = config.modelD.name
g_params = config.modelG.params
d_params = config.modelD.params
nz = g_params['nz']
print(g_params)
 print(d_params)
 epochs = config.train.epoch
batch_size = config.train.batch_size
lrG = config.optimizerG.params.lr
G_beta1 = config.optimizerG.params.beta1
G_beta2 = config.optimizerD.params.beta2
lrD = config.optimizerD.params.lr
D_beta1 = config.optimizerD.params.beta1
D_beta2 = config.optimizerD.params.beta2
use_averaging_model = config.train.params.use_averaging_model
real_label = config.data.real_label
fake_label = config.data.fake_label
num_classes = 120
sz = 64
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
target_column = "comment_text"
test_ids_df.loc[test_ids_df[target_column].isin(val_ids_df[target_column]), ]<count_unique_values>
transform1 = transforms.Compose([transforms.Resize(sz),
transforms.CenterCrop(sz)])
 random_transforms = [transforms.RandomRotation(degrees=5)]
transform2 = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomApply(random_transforms, p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_dataset = DogDataset(img_dir=root_dir + '/all-dogs/all-dogs/',
anno_dir=root_dir + '/annotation/Annotation',
transform1=transform1,
transform2=transform2,
exrw=exrw,
exrh=exrh,
n_images=n_images)
 train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=4 )
target_column = "input_word_ids"
overlapped_test_df = test_ids_df.loc[test_ids_df[target_column].isin(val_ids_df[target_column]), ]
overlapped_val_df = val_ids_df.loc[val_ids_df[target_column].isin(test_ids_df[target_column]), ]
overlapped_test_df.nunique()<feature_engineering>
def to_img(x):
x = 0.5 *(x + 1)
 x = x.clamp(0, 1)
 x = x.view(x.size(0), 3, 64, 64)
 return x
def show_generated_img(epoch=0):
if not eval_do:
return
if epoch % 10 == 0 or epoch==epochs-1:
noise = fixed_noise
labels = fixed_labels
gen_tensors = netG(noise, labels ).to("cpu" ).clone().detach().squeeze(0)
 gen_tensors = gen_tensors/2+0.5
gen_tensors = gen_tensors.clamp(0, 1)
 gen_images = gen_tensors.numpy().transpose(0, 2, 3, 1)
 plt.subplot(1,2,1)
 plt.imshow(gen_images[0])
 plt.subplot(1,2,2)
 plt.imshow(gen_images[1])
 plt.pause(0.1)
 print('gen_image:(%f, %f)' %(gen_images.min() , gen_images.max()))
def generate_samples(outimg_dir, netG, n_images=10000):
if not os.path.exists(outimg_dir):
os.mkdir(outimg_dir)
 im_batch_size = 50
for i_batch in range(0, n_images, im_batch_size):
gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)
 gen_labels = torch.randint(num_classes,(im_batch_size,), device=device)
 gen_images =(netG(gen_z, gen_labels)+ 1)/2
images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join(outimg_dir, f'image_{i_batch+i_image:05d}.png'))
def truncated_noise_sample(batch_size=1, dim_z=128, truncation=1., seed=None):

 state = None if seed is None else np.random.RandomState(seed)
 values = truncnorm.rvs(-2, 2, size=(batch_size, dim_z), random_state=state ).astype(np.float32)
 return truncation * values
def generate_samples_truncated(outimg_dir, netG, th=1, n_images=10000):
if not os.path.exists(outimg_dir):
os.mkdir(outimg_dir)
 im_batch_size = 50
for i_batch in range(0, n_images, im_batch_size):
if th>0:
tns = truncated_noise_sample(im_batch_size, nz, th)
 tns = np.expand_dims(tns, -1)
 tns = np.expand_dims(tns, -1)
 gen_z = torch.from_numpy(tns ).to(device)
 else:
gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)
 gen_labels = torch.randint(num_classes,(im_batch_size,), device=device)
 gen_images =(netG(gen_z, gen_labels)+ 1)/2
images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join(outimg_dir, f'image_{i_batch+i_image:05d}.png'))
test_ids_df["token_hash"] = test_ids_df["input_word_ids"].apply(lambda x: crc32(x.encode())& 0xffffffff)
 val_ids_df["token_hash"] = val_ids_df["input_word_ids"].apply(lambda x: crc32(x.encode())& 0xffffffff)
 target_column = "token_hash"
overlapped_hash = test_ids_df.loc[test_ids_df[target_column].isin(val_ids_df[target_column]), target_column]
overlapped_hash = np.unique(overlapped_hash.values)
 overlapped_hash.sort()
overlapped_hash.shape<feature_engineering>
def init_weights_normal(m):
w_mean = 0.0
w_std = 0.02
if type(m)== nn.ConvTranspose2d:
print('gaussian initialization')
 torch.nn.init.normal_(m.weight, w_mean, w_std)
 elif type(m)== nn.Conv2d:
print('gaussian initialization')
 torch.nn.init.normal_(m.weight, w_mean, w_std)
 def init_weights_xavier(m):
if type(m)== nn.Linear or type(m)== nn.Conv2d:
print('xavier_uniform initialization')
 torch.nn.init.xavier_uniform_(m.weight)
 if m.bias is not None:
m.bias.data.fill_(0.)
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
self.alpha = 1e-8
def forward(self, x):

 if False:
y = x.pow(2.).mean(dim=1, keepdim=True ).add(self.alpha ).sqrt()
y = x / y
else:
mean = torch.mean(x * x, 1, keepdim=True)
 dom = torch.rsqrt(mean + self.alpha)
 y = x * dom
return y
class MinibatchStdDev(nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
self.alpha = 1e-8
def forward(self, x):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ self.alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y
class UpsamplingNN(nn.Module):
def __init__(self, n_input, n_output, k_size=3, stride=1, padding=1, bias=False):
super().__init__()
layers = [
nn.Upsample(scale_factor=2, mode='nearest'),
nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=bias),
]
self.convlayer = nn.Sequential(*layers)
 self.weight = self.convlayer[1].weight
def forward(self, x):
y = self.convlayer(x)
 return y
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels, upsample='nearest', kernel=3):
super().__init__()
if upsample=='pixelshuffle_conv':
upsample = ConvPS
elif upsample=='conv_pixelshuffle':
upsample = ConvPS_convfirst
elif upsample=='nearest':
upsample = UpsamplingNN
elif upsample=='bilinear':
upsample = UpsamplingBL
else:
print('unknown upsample: ', upsample)
 sys.exit()
print(self.__class__)
 print('upsample: ', upsample)
 print('nfeats: ', nfeats)
 print('kernel: ', kernel)
 pad = kernel//2
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(upsample(nfeats * 8, nfeats * 8, kernel, 1, pad, bias=False))
self.conv3 = spectral_norm(upsample(nfeats * 8, nfeats * 4, kernel, 1, pad, bias=False))
self.conv4 = spectral_norm(upsample(nfeats * 4, nfeats * 2, kernel, 1, pad, bias=False))
self.conv5 = spectral_norm(upsample(nfeats * 2, nfeats, kernel, 1, pad, bias=False))
self.conv6 = spectral_norm(nn.Conv2d(nfeats, nchannels, 5, 1, 2, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x, labels=None):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
def get_generator(nz, channels=3, nfilt=32, winit=None, upsample='nearest', kernel=3, **_):
model = Generator(nz, nfeats=nfilt, nchannels=channels, upsample=upsample, kernel=kernel)
 if winit=='normal':
model.apply(init_weights_normal)
 elif winit=='xavier':
model.apply(init_weights_xavier)
 return model
def display_test_val_comment(test_ids_df, val_ids_df, target_column="token_hash", target_value=0, test_en_df=None, ind_=0):
test_sample = test_ids_df.query(f"{target_column} == {target_value}")
 val_sample = val_ids_df.query(f"{target_column} == {target_value}")
 test_comment = test_sample.comment_text.values[ind_]
val_comment = val_sample.comment_text.values[0]
print("{}".format("-"*80))
print(f"{target_column}:{target_hash}")
 print(f">> TEST
ID:{test_sample.id.values[ind_]}, LANG:{test_sample.lang.values[ind_]}, Duplicated Num:{len(test_sample)}")
 print(f"COMMET_TEXT:{test_comment}
")
 print(f">> VALIDATION
ID:{val_sample.id.values[0]}, LANG:{val_sample.lang.values[0]}, Duplicated Num:{len(val_sample)}, TOXIC:{val_sample.toxic.values[0]}")
 print(f"COMMET_TEXT:{val_comment}
")
 " ).split() , val_comment.replace(" ", "
" ).split() , "TEST", "VAL", lineterm='
')
 diff = difflib.context_diff(test_comment.replace(" ", "
" ).split() , val_comment.replace(" ", "
" ).split() , "TEST", "VAL", lineterm='
')
 print(">> diff result")
 sys.stdout.writelines(diff)
 if test_en_df is not None:
test_en_comment = test_en_df.query(f"id == {test_sample.id.values[ind_]}" ).content_en.values[0]
print("
>> TEST ENGLISH Translation")
 print(f"{test_en_comment}
" )<compute_train_metric>
def snconv(chin, chout, kernel, stride, pad, bias=False, bn=False):
layers = []
layers.append(spectral_norm(nn.Conv2d(chin, chout, kernel, stride, pad, bias=bias)))
 if bn:
layers.append(nn.BatchNorm2d(chout))
return nn.Sequential(*layers)
 class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats, acti=False, lrelu_slope=0.2, bn=False, mbstd=False, kernel=3):
super().__init__()
bias = True
pad = kernel//2
self.snconv1_1 = snconv(nchannels, nfeats//2, kernel, 1, pad, bias=bias, bn=bn)
 self.snconv1_2 = snconv(nfeats//2, nfeats*1, 4, 2, 1, bias=bias, bn=bn)
 self.snconv2_1 = snconv(nfeats*1, nfeats*1, kernel, 1, pad, bias=bias, bn=bn)
 self.snconv2_2 = snconv(nfeats*1, nfeats*2, 4, 2, 1, bias=bias, bn=bn)
 self.snconv3_1 = snconv(nfeats*2, nfeats*2, kernel, 1, pad, bias=bias, bn=bn)
 self.snconv3_2 = snconv(nfeats*2, nfeats*4, 4, 2, 1, bias=bias, bn=bn)
 self.snconv4_1 = snconv(nfeats*4, nfeats*4, kernel, 1, pad, bias=bias, bn=bn)
 self.snconv4_2 = snconv(nfeats*4, nfeats*8, 4, 2, 1, bias=bias, bn=bn)
 self.batch_discriminator = MinibatchStdDev()
self.pixnorm = PixelwiseNorm()
self.snconv5 = snconv(nfeats*8+mbstd, 1, 4, 1, 0, bias=True, bn=False)
 self.conv5 = nn.Conv2d(nfeats*8+mbstd, 1, 4, 1, 0, bias=True)
 self.acti = acti
self.lrelu_slope = lrelu_slope
self.bn = bn
self.mbstd = mbstd
print('D: last sigmoid: ', self.acti)
 print(' lrelu_slope: ', self.lrelu_slope)
 print(' batch_norm: ', self.bn)
 print(' kernel_size: ', kernel)
 def forward(self, x, labels=None):
x = F.leaky_relu(self.snconv1_1(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv1_2(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv2_1(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv2_2(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv3_1(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv3_2(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv4_1(x), self.lrelu_slope)
 x = F.leaky_relu(self.snconv4_2(x), self.lrelu_slope)
 if self.mbstd:
x = self.batch_discriminator(x)
 x= self.snconv5(x)
 if self.acti:
x = torch.sigmoid(x)
 return x.view(-1, 1)
 def get_discriminator(channels=3, nfilt=32, winit=None, acti=True, lrelu_slope=0.2, bn=False, mbstd=False, kernel=3, **_):
model = Discriminator(nchannels=channels, nfeats=nfilt, acti=acti, lrelu_slope=lrelu_slope, bn=bn, mbstd=mbstd, kernel=kernel)
 if winit=='normal':
model.apply(init_weights_normal)
 elif winit=='xavier':
model.apply(init_weights_xavier)
 return model
for i, target_hash in enumerate(overlapped_hash):
display_test_val_comment(test_ids_df, val_ids_df, target_column="token_hash", target_value=target_hash, test_en_df=test_en_df)
 if i >= 2:
break<load_from_csv>
netG = get_generator(**g_params ).to(device)
 netD = get_discriminator(**d_params ).to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lrD, betas=(D_beta1, D_beta2))
optimizerG = optim.Adam(netG.parameters() , lr=lrG, betas=(G_beta1, G_beta2))
lr_schedulerD = torch.optim.lr_scheduler.StepLR(optimizerD, step_size=100, gamma=0.5)
 lr_schedulerG = torch.optim.lr_scheduler.StepLR(optimizerG, step_size=100, gamma=0.5)
 fixed_noise = torch.randn(25, nz, 1, 1, device=device)
 fixed_labels = torch.randint(1,(25,), device=device)
sub_df = pd.read_csv(".. /input/ensemble/submission.csv")
 sub_df = pd.merge(sub_df, test_ids_df, on="id")
 sub_df.head()<sort_values>
def ema_model(target_model, source_model, alpha):
target_params = target_model.state_dict()
source_params = source_model.state_dict()
for param_name, param in source_params.items() :
target_params[param_name] *= alpha
target_params[param_name] +=(1-alpha)* param
def ma_model(target_model, source_model, n):
target_params = target_model.state_dict()
source_params = source_model.state_dict()
for param_name, param in source_params.items() :
target_params[param_name] *=(n /(n+1))
target_params[param_name] +=(1/(n+1)) * param
def copy_model_params(target_model, source_model):
target_model.load_state_dict(source_model.state_dict())
 if use_averaging_model:
ma_count = 0
netG_ma = copy.deepcopy(netG)
sub_df.loc[sub_df.token_hash.isin(non_toxic_hash), :].sort_values("toxic" ).tail()<filter>
def get_lossD_sGAN() :
criterion = nn.BCELoss()
def lossD_real(outputR, real_labels):
return criterion(outputR, real_labels)
 def lossD_fake(outputF, fake_labels):
return criterion(outputF, fake_labels)
 return lossD_real, lossD_fake
def get_lossG_sGAN() :
criterion = nn.BCELoss()
def lossG(outputR, outputF, real_labels, fake_labels):
return criterion(outputF, real_labels)
 return lossG
def get_lossD_RaLSGAN() :
def lossD_rerative(outputR, outputF, real_labels, fake_labels):
errD =(torch.mean(( outputR - torch.mean(outputF)- real_labels)** 2)+
torch.mean(( outputF - torch.mean(outputR)- fake_labels)** 2)) /2
return errD
return lossD_rerative
def get_lossG_RaLSGAN() :
def lossG_rerative(outputR, outputF, real_labels, fake_labels):
errG =(torch.mean(( outputR - torch.mean(outputF)+ real_labels)** 2)+
torch.mean(( outputF - torch.mean(outputR)+ fake_labels)** 2)) /2
return errG
return lossG_rerative
loss_type = config.loss.name
if loss_type=='sGAN':
print('loss type: sGAN')
 criterionD_real, criterionD_fake = get_lossD_sGAN()
criterionG = get_lossG_sGAN()
relative_loss = False
else:
print('loss type: RaLSGAN')
 criterionD = get_lossD_RaLSGAN()
criterionG = get_lossG_RaLSGAN()
relative_loss = True
non_toxic_hash = non_toxic_hash[non_toxic_hash != 3880127965]<feature_engineering>
output_dir = '.. /tmp'
outimg_dir = '.. /tmp/output_images'
if not os.path.exists(output_dir):
os.mkdir(output_dir)
 if not os.path.exists(outimg_dir):
os.mkdir(outimg_dir)
 start_time = tic()
lossD = []
lossG = []
for epoch in range(epochs):
elasped_time = toc(start_time)
 print('elasped_time[h]: ', elasped_time/3600)
 if elasped_time/3600 > 8.5:
break
lossD_epoch = 0
lossG_epoch = 0
for ii,(real_images, bleed_labels)in tqdm(enumerate(train_loader), total=len(train_loader)) :
real_images = real_images.to(device)
 bleed_labels = bleed_labels.to(device)
 bleed_labels[:] = 0
batch_size = real_images.size(0)
 if isinstance(real_label, list):
real_labels = torch.empty(( batch_size, 1), device=device ).uniform_(real_label[0], real_label[1])
 fake_labels = torch.empty(( batch_size, 1), device=device ).uniform_(fake_label[0], fake_label[1])
 else:
real_labels = torch.full(( batch_size, 1), real_label, device=device)
 fake_labels = torch.full(( batch_size, 1), fake_label, device=device)
 netD.zero_grad()
outputR = netD(real_images, bleed_labels)
 if relative_loss:
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise, bleed_labels)
 outputF = netD(fake.detach() , bleed_labels)
 errD = criterionD(outputR, outputF, real_labels, fake_labels)
 errD.backward(retain_graph=True)
 else:
errD_real = criterionD_real(outputR, real_labels)
 errD_real.backward()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise, bleed_labels)
 outputF = netD(fake.detach() , bleed_labels)
 errD_fake = criterionD_fake(outputF, fake_labels)
 errD_fake.backward()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
outputF = netD(fake, bleed_labels)
 errG = criterionG(outputR, outputF, real_labels, fake_labels)
 errG.backward()
optimizerG.step()
lossD_epoch += errD.item()
lossG_epoch += errG.item()
if(ii+1)%(len(train_loader)//2)== 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f'
%(epoch + 1, epochs, ii+1, len(train_loader),
errD.item() , errG.item()))
if use_averaging_model:
if epoch >= config.train.params.ma_start:
if ma_count==0:
copy_model_params(netG_ma, netG)
 else:
if config.train.params.ema:
ema_model(netG_ma, netG, config.train.params.ema_alpha)
 else:
ma_model(netG_ma, netG, ma_count)
 ma_count += 1
if use_scheduler:
lr_schedulerD.step()
lr_schedulerG.step()
lossD.append(lossD_epoch)
 lossG.append(lossG_epoch)
 show_generated_img(epoch )
sub_df.loc[sub_df.token_hash.isin(non_toxic_hash), "toxic"] = 0.0
sub_df.loc[sub_df.token_hash.isin(toxic_hash), "toxic"]= 1.0<save_to_csv>
th = 1.0
if use_averaging_model:
generate_samples_truncated(outimg_dir, netG_ma)
 else:
generate_samples_truncated(outimg_dir, netG)
 shutil.make_archive('images', 'zip', outimg_dir)
 print('last epoch: ', epoch)
 elapsed_time = toc(start_time)
 print('elapsed time[h] :', elapsed_time/3600 )