Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_pretrained>
RobertaConfig, RobertaModel, RobertaTokenizer)
class DogsDataset(Dataset):
def __init__(self, root, transform1=None, transform2=None):
self.root = root
self.transform1 = transform1
self.transform2 = transform2
self.samples, self.crops = self._load_subfolders_images(self.root)
 if len(self.samples)== 0:
raise RuntimeError("Found 0 files in subfolders of: {}".format(self.root))
def _load_subfolders_images(self, root):
IMG_EXTENSIONS =('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp')
 def is_valid_file(x):
return torchvision.datasets.folder.has_file_allowed_extension(x, IMG_EXTENSIONS)
 imgs = []
img_paths = []
bboxes = []
paths = []
for root, _, fnames in sorted(os.walk(root)) :
for fname in sorted(fnames):
path = os.path.join(root, fname)
 paths.append(path)
 pbar = tqdm(paths, desc='Loading cropped images')
 for path in pbar:
if is_valid_file(path):
img = torchvision.datasets.folder.default_loader(path)
 annotation_basename = os.path.splitext(os.path.basename(path)) [0]
annotation_dirname = next(dirname for dirname in os.listdir(ANNOTATION_PATH)if dirname.startswith(annotation_basename.split('_')[0]))
annotation_filename = os.path.join(ANNOTATION_PATH, annotation_dirname, annotation_basename)
 tree = ET.parse(annotation_filename)
 root = tree.getroot()
size = root.find('size')
 width = int(size.find('width' ).text)
 height = int(size.find('height' ).text)
 objects = root.findall('object')
 objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 xmin = max(0, xmin - 4)
 xmax = min(width, xmax + 4)
 ymin = max(0, ymin - 4)
 ymax = min(height, ymax + 4)
 w = np.min(( xmax - xmin, ymax - ymin))
w = min(w, width, height)
 if w > xmax - xmin:
xmin = min(max(0, xmin - int(( w -(xmax - xmin)) /2)) , width - w)
 xmax = xmin + w
if w > ymax - ymin:
ymin = min(max(0, ymin - int(( w -(ymax - ymin)) /2)) , height - w)
 ymax = ymin + w
img_ = img.crop(( xmin, ymin, xmax, ymax))
if np.mean(img_)!= 0:
img = img_
img_paths.append(path)
 bboxes.append(( xmin, ymin, xmax, ymax))
pbar.set_postfix_str("{} cropped images loaded".format(len(img_paths)))
 return img_paths, bboxes
def __getitem__(self, index):
path = self.samples[index]
sample = torchvision.datasets.folder.default_loader(path)
 sample = sample.crop(self.crops[index])
 if self.transform2 is not None:
sample = self.transform2(sample)
 return sample
def __len__(self):
return len(self.samples )<normalization>
def set_seed(seed,n_gpu=0):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 if n_gpu > 0:
torch.cuda.manual_seed_all(seed)
 set_seed(0 )
transform1 = None
transform2 = transforms.Compose([transforms.RandomResizedCrop(64,(0.85, 1.0)) ,
transforms.ColorJitter() ,
transforms.RandomHorizontalFlip(p=0.5),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_data = DogsDataset(
 PATH,
transform1,
transform2 
)
 train_loader = torch.utils.data.DataLoader(train_data, shuffle=True,
batch_size=BATCH_SIZE, num_workers=4)
<normalization>
class Config:
max_length=64
lr=1e-6
batch_size=64
train_epochs = 3
save_epoch = 3
max_grad_norm = 1
train_ratio=0.6
valid_ratio=0.3
device=torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 config = Config()
def slerp(val, low, high):
low_norm = low/torch.norm(low, dim=1, keepdim=True)
 high_norm = high/torch.norm(high, dim=1, keepdim=True)
 omega = torch.acos(( low_norm*high_norm ).sum(1))
so = torch.sin(omega)
 res =(torch.sin(( 1.0-val)*omega)/so ).unsqueeze(1)*low +(torch.sin(val*omega)/so ).unsqueeze(1)* high
return res
class BatchNormModulate2d(nn.Module):

 def __init__(self, num_features, dim_in, eps=2e-5, momentum=0.1, affine=True,
track_running_stats=True, use_sn=True):
super().__init__()
self.num_features = num_features
self.dim_in = dim_in
self.bn = nn.BatchNorm2d(num_features, affine=False)
 self.gamma = nn.Sequential(
 nn.Linear(dim_in, num_features, bias=True),
nn.ReLU() ,
nn.Linear(num_features, num_features, bias=False)
)
 self.beta = nn.Sequential(
 nn.Linear(dim_in, num_features, bias=True),
nn.ReLU() ,
nn.Linear(num_features, num_features, bias=False)
)
 def forward(self, x, z):
out = self.bn(x)
 gamma = self.gamma(z)
 beta = self.beta(z)
 out = gamma.view(-1, self.num_features, 1, 1)* out + beta.view(-1, self.num_features, 1, 1)
 return out
class PixelNorm(nn.Module):
def __init__(self, epsilon=1e-8):

 super(PixelNorm, self ).__init__()
self.epsilon = epsilon
def forward(self, x):
tmp = torch.mul(x, x)
 tmp1 = torch.rsqrt(torch.mean(tmp, dim=1, keepdim=True)+ self.epsilon)
 return x * tmp1
class GaussianNoise(nn.Module):
def __init__(self, channels):
super().__init__()
self.weight = nn.Parameter(torch.zeros(1, channels, 1, 1))
def forward(self, x, noise=None):
if noise is None:
noise = torch.randn(x.shape[0], 1, x.shape[2], x.shape[3], device=x.device, dtype=x.dtype)
 return x + self.weight.view(1, -1, 1, 1)* noise.to(x.device)
 class AdaIn(nn.Module):

 def __init__(self, in_channel, latent_dim):
super().__init__()
self.norm = nn.InstanceNorm2d(in_channel)
 self.style = nn.Linear(latent_dim, in_channel * 2)
 def forward(self, input, style):
style = self.style(style ).unsqueeze(2 ).unsqueeze(3)
 gamma, beta = style.chunk(2, 1)
 out = self.norm(input)
 out = gamma * out + beta
return out
class Projection(nn.Module):
def __init__(self,
nz,
in_channel,
out_channel,
shape,
bias=False,
spectral_norm=False,
normalization='selfmod',
random_noise=False,
use_pixel_norm=False):
super().__init__()
self.shape = shape
self.linear = nn.Linear(in_channel, out_channel, bias=bias)
 self.conv = nn.Conv2d(shape[0], shape[0], 3, 1, 1, bias=bias)
 if spectral_norm:
self.linear = SN(self.linear)
 self.conv = SN(self.conv)
 self.noise1 = None
if random_noise:
self.noise1 = GaussianNoise(shape[0])
 self.noise2 = GaussianNoise(shape[0])
 self.pixel_norm = None
if use_pixel_norm:
self.pixel_norm = PixelNorm()
if normalization == 'adain':
self.norm1 = AdaIn(shape[0], nz)
 self.norm2 = AdaIn(shape[0], nz)
 else:
self.norm1 = BatchNormModulate2d(shape[0], nz)
 self.norm2 = BatchNormModulate2d(shape[0], nz)
 def forward(self, x, nz):
x = self.linear(x)
 x = x.view([x.shape[0]] + self.shape)
 if self.noise1 is not None:
x = self.noise1(x)
 x = F.leaky_relu(x, 0.2)
 if self.pixel_norm is not None:
x = self.pixel_norm(x)
 x = self.norm1(x, nz)
 x = self.conv(x)
 if self.noise2 is not None:
x = self.noise2(x)
 x = F.leaky_relu(x, 0.2)
 if self.pixel_norm is not None:
x = self.pixel_norm(x)
 x = self.norm2(x, nz)
 return x
class UpConvBlock(nn.Module):

 def __init__(self,
nz,
in_channel,
out_channel,
kernel=4,
stride=2,
padding=1,
bias=False,
spectral_norm=False,
normalization='selfmod',
random_noise=False,
use_pixel_norm=False):
super().__init__()
self.conv1 = nn.ConvTranspose2d(in_channel, out_channel, kernel, stride, padding, bias=bias)
 self.conv2 = nn.Conv2d(out_channel, out_channel, 3, 1, 1, bias=bias)
 if spectral_norm:
self.conv1 = SN(self.conv1)
 self.conv2 = SN(self.conv2)
 self.noise1 = None
self.noise2 = None
if random_noise:
self.noise1 = GaussianNoise(out_channel)
 self.noise2 = GaussianNoise(out_channel)
 self.pixel_norm = None
if use_pixel_norm:
self.pixel_norm = PixelNorm()
if normalization == 'adain':
self.norm1 = AdaIn(out_channel, nz)
 self.norm2 = AdaIn(out_channel, nz)
 else:
self.norm1 = BatchNormModulate2d(out_channel, nz)
 self.norm2 = BatchNormModulate2d(out_channel, nz)
 def forward(self, x, noise):
x = self.conv1(x)
 if self.noise1 is not None:
x = self.noise1(x)
 x = F.leaky_relu(x, 0.2)
 if self.pixel_norm is not None:
x = self.pixel_norm(x)
 x = self.norm1(x, noise)
 x = self.conv2(x)
 if self.noise2 is not None:
x = self.noise2(x)
 x = F.leaky_relu(x, 0.2)
 if self.pixel_norm is not None:
x = self.pixel_norm(x)
 x = self.norm2(x, noise)
 return x
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels, spectral_norm=False, normalization='selfmod', random_noise=False, use_pixel_norm=False):
super(Generator, self ).__init__()
d = nfeats*8
self.linear = Projection(nz, nz, 8*8*d, [512, 8, 8], False, spectral_norm, normalization, random_noise, use_pixel_norm)
 self.conv2 = UpConvBlock(nz, nfeats*8, nfeats*4, 4, 2, 1, False, spectral_norm, normalization, random_noise, use_pixel_norm)
 self.conv3 = UpConvBlock(nz, nfeats*4, nfeats*2, 4, 2, 1, False, spectral_norm, normalization, random_noise, use_pixel_norm)
 self.conv4 = UpConvBlock(nz, nfeats*2, nfeats, 4, 2, 1, False, spectral_norm, normalization, random_noise, use_pixel_norm)
 self.conv5 = nn.ConvTranspose2d(nfeats, nchannels, 3, 1, 1, bias=False)
 if spectral_norm:
self.conv5 = SN(self.conv5)
 def forward(self, x):
out = self.linear(x.view(x.shape[0],-1), x)
 out = self.conv2(out, x)
 out = self.conv3(out, x)
 out = self.conv4(out, x)
 out = torch.tanh(self.conv5(out))
return out
class Discriminator(nn.Module):
def __init__(self, nchannels, nfeats):
super(Discriminator, self ).__init__()
d = nfeats
self.model = nn.Sequential(
 SN(nn.Conv2d(nchannels, d, 3, 1, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True),
SN(nn.Conv2d(d, d * 2, 4, 2, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True),
SN(nn.Conv2d(d*2, d*2, 3, 1, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True),
SN(nn.Conv2d(d*2, d*4, 4, 2, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True),
SN(nn.Conv2d(d*4, d*4, 3, 1, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True),
SN(nn.Conv2d(d*4, d*8, 4, 2, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True),
SN(nn.Conv2d(d*8, d*8, 3, 1, 1, bias=False)) ,
nn.LeakyReLU(0.2, inplace=True)
)
 self.linear = SN(nn.Linear(8*8*d*8, 1, bias=False))
def forward(self, x):
out = self.model(x)
 out = out.view(out.shape[0], -1)
 out = self.linear(out)
 return out<compute_train_metric>
def text_preprocess(raw_text,tokenizer,config):
MAX_LENGTH = config.max_length
url_pattern = re.compile(r'http://.*? |https://.*? ')
 at_pattern = re.compile(r'@.*? ')
 tag_pattern = re.compile(r'
face_pattern = re.compile(r'\x89.{2}')
 bracket_pattern = re.compile('\[.*?\]')
 repeat_pattern = re.compile(r'((.) \2+)')
 sep_pattern = re.compile(r'\[SEP\]')
 result = []
input_ids = []
attention_mask =[]
for text in tqdm(raw_text):
text += ' '
url_iter = url_pattern.finditer(text)
 for url_matcher in url_iter:
url = url_matcher.group().strip()
try:
title = None
except:
title = None
title = '[SEP] '+title+' [SEP]' if title else ''
text = text.replace(url,title)
 text = text.replace('
',' [SEP] ')
 text = at_pattern.sub('',text)
 text = face_pattern.sub('',text)
 tag_iter = tag_pattern.finditer(text)
 for tag_matcher in tag_iter:
tag = tag_matcher.group().strip()
text = text.replace(tag,'[SEP] '+tag[1:]+' [SEP]')
 text = text.replace('|',' [SEP] ')
 brackets = bracket_pattern.findall(text)
 for b in brackets:
if b != '[SEP]':
text = text.replace(b,'[SEP] '+b[1:-1]+' [SEP]')
 sep_iter = sep_pattern.finditer(text)
 new_text = ''
prev_idx = 0
for sep_matcher in sep_iter:
parts = text[prev_idx:sep_matcher.start() ]
parts = parts.strip()
new_text += parts+' [SEP] ' if parts else ''
prev_idx = sep_matcher.end()
text = new_text + text[prev_idx:]
rep_iter = repeat_pattern.finditer(text)
 new_text = ''
prev_idx = 0
for rep_matcher in rep_iter:
new_text += text[prev_idx:rep_matcher.start() +1]
prev_idx = rep_matcher.end()
text = new_text + text[prev_idx:]
text = text.replace(' ',' ')
 text = text.strip().strip('-')
 text = text.strip('-' ).strip()
while text[:5]=='[SEP]':
text = text[5:].strip()
while text[-5:]=='[SEP]':
text = text[:-5].strip()
text = text.replace('[SEP]',tokenizer.sep_token)
 text = text.lower()
result.append(text)
 iid,am = tokenizer(text ).values()
if len(iid)<=MAX_LENGTH:
pad_len = MAX_LENGTH-len(iid)
 iid += [1]*pad_len
am += [0]*pad_len
else:
iid = iid[:MAX_LENGTH]
am = am[:MAX_LENGTH]
input_ids.append(iid)
 attention_mask.append(am)
 return result,input_ids,attention_mask
def preprocess(df,tokenizer,config):
raw_text = df['text'].to_list()
sample_id = df['id'].to_list()
try:
target = df['target'].to_list()
except:
target = [0]*len(sample_id)
 result,input_ids,attention_mask = text_preprocess(raw_text,tokenizer,config)
 return {'id':sample_id,
'text':result,
'input_ids':input_ids,
'attention_mask':attention_mask,
'label':target}
def gradient_penalty(x, y, f):
shape = [x.size(0)] + [1] *(x.dim() - 1)
 alpha = torch.rand(shape ).to(x.device)
 z = x + alpha *(y - x)
 z = Variable(z, requires_grad=True ).to(x.device)
 o = f(z)
 g = grad(o, z, grad_outputs=torch.ones(o.size() ).to(z.device), create_graph=True)[0].view(z.size(0), -1)
 gp =(( g.norm(p=2, dim=1)- 1)**2 ).mean()
return gp
def R1Penalty(real_img, f):
reals = Variable(real_img, requires_grad=True ).to(real_img.device)
 real_logit = f(reals)
 apply_loss_scaling = lambda x: x * torch.exp(x * torch.Tensor([np.float32(np.log(2.0)) ] ).to(real_img.device))
undo_loss_scaling = lambda x: x * torch.exp(-x * torch.Tensor([np.float32(np.log(2.0)) ] ).to(real_img.device))
real_logit = apply_loss_scaling(torch.sum(real_logit))
real_grads = grad(real_logit, reals, grad_outputs=torch.ones(real_logit.size() ).to(reals.device), create_graph=True)[0].view(reals.size(0), -1)
 real_grads = undo_loss_scaling(real_grads)
 r1_penalty = torch.sum(torch.mul(real_grads, real_grads))
return r1_penalty
def G_wgan(G, D, nz, batch_size):
noise = torch.randn(batch_size, nz, device=device)
 fake_images = G(noise)
 fake_logit = D(fake_images)
 G_loss = -fake_logit.mean()
return G_loss
def D_wgan_gp(G, D, real_images, nz, lammy=10.0, eps=0.001):
batch_size = real_images.shape[0]
real_logit = D(real_images)
 noise = torch.randn(batch_size, nz, device=device)
 fake_images = G(noise)
 fake_logit = D(fake_images.detach())
 D_loss = fake_logit.mean() - real_logit.mean()
D_loss = D_loss + gradient_penalty(real_images.data, fake_images.data, D)* lammy
return D_loss, real_logit.mean() , fake_logit.mean()
def D_NS(G, D, real_images, nz, lammy=0.0):
batch_size = real_images.shape[0]
real_logit = D(real_images)
 D_loss_real = F.softplus(-real_logit ).mean()
noise = torch.randn(batch_size, nz, device=device)
 fake_images = G(noise)
 fake_logit = D(fake_images.detach())
 D_loss_fake = F.softplus(fake_logit ).mean()
D_loss = D_loss_real + D_loss_fake
if lammy != 0.0:
r1_penalty = R1Penalty(real_images.detach() , D)
 D_loss = D_loss + r1_penalty *(lammy * 0.5)
 return D_loss, D_loss_real, D_loss_fake
def G_NS(G, D, nz, batch_size):
noise = torch.randn(batch_size, nz, device=device)
 fake_images = G(noise)
 fake_logit = D(fake_images)
 G_loss = F.softplus(-fake_logit ).mean()
return G_loss
def D_BCE_NS(G, D, real_images, nz, real_labels, fake_labels, lammy=0.0):
batch_size = real_images.shape[0]
real_logit = D(real_images)
 D_loss_real = F.binary_cross_entropy_with_logits(real_logit, real_labels)
 noise = torch.randn(batch_size, nz, device=device)
 fake_images = G(noise)
 fake_logit = D(fake_images.detach())
 D_loss_fake = F.binary_cross_entropy_with_logits(fake_logit, fake_labels)
 D_loss = D_loss_real + D_loss_fake
if lammy != 0.0:
r1_penalty = R1Penalty(real_images.detach() , D)
 D_loss = D_loss + r1_penalty *(lammy * 0.5)
 return D_loss, D_loss_real, D_loss_fake
def G_BCE_NS(G, D, nz, batch_size, real_labels):
noise = torch.randn(batch_size, nz, device=device)
 fake_images = G(noise)
 fake_logit = D(fake_images)
 G_loss = F.binary_cross_entropy_with_logits(fake_logit, real_labels)
 return G_loss
class Trainer:
def __init__(self, nz, G, D, lammy=0.0, track_grads=False):
self.nz = nz
self.track_grads=track_grads
self.G = G
self.D = D
self.fixed_noise = torch.randn(64, self.nz)
 self.lammy = lammy
self.d_losses = []
self.d_losses_real = []
self.d_losses_fake = []
self.g_losses = []
self.img_list = []
self.g_grads = []
self.d_grads = []
def check_grads(self, model):
grads = []
for n, p in model.named_parameters() :
if not p.grad is None and p.requires_grad and "bias" not in n:
grads.append(float(p.grad.abs().mean()))
return grads
def train(self, epochs, loader, criterion, optim_G, optim_D, scheduler_D, scheduler_G, print_every, show_every, plot_every, loss='NS'):
step = 0
for epoch in tqdm(range(epochs)) :
for ii, real_images in enumerate(loader):
batch_size = real_images.size(0)
 self.D.zero_grad()
real_images = real_images.to(device)
 if loss == 'WGAN':
D_loss, Dx, D_Gz = D_wgan_gp(self.G, self.D, real_images, self.nz)
 elif loss == 'BCE':
if USE_SOFT_NOISY_LABELS:
real_labels = torch.empty(( batch_size, 1), device=device ).uniform_(0.80, 0.95)
 fake_labels = torch.empty(( batch_size, 1), device=device ).uniform_(0.05, 0.20)
 else:
real_labels = torch.full(( batch_size, 1), 0.95, device=device)
 fake_labels = torch.full(( batch_size, 1), 0.05, device=device)
 if INVERT_LABELS and random.random() < 0.01:
real_labels, fake_labels = fake_labels, real_labels
D_loss, Dx, D_Gz = D_BCE_NS(self.G, self.D, real_images, self.nz, real_labels, fake_labels, self.lammy)
 else:
D_loss, Dx, D_Gz = D_NS(self.G, self.D, real_images, self.nz, self.lammy)
 D_loss.backward()
optim_D.step()
self.G.zero_grad()
if loss == 'WGAN':
G_loss = G_wgan(self.G, self.D, self.nz, batch_size)
 elif loss == 'BCE':
G_loss = G_BCE_NS(self.G, self.D, self.nz, batch_size, real_labels)
 else:
G_loss = G_NS(self.G, self.D, self.nz, batch_size)
 G_loss.backward()
optim_G.step()
if step % print_every == 0:
print('[%d/%d][%d/%d]\tLoss_D: %.4f\tLoss_G: %.4f\tD(x): %.4f\tD(G(z)) : %.4f'
%(epoch, epochs, ii, len(loader),
D_loss.item() , G_loss.item() , Dx, D_Gz))
if step % plot_every == 0:
self.d_losses.append(D_loss.item())
 self.d_losses_real.append(Dx)
 self.d_losses_fake.append(D_Gz)
 self.g_losses.append(G_loss.item())
 if self.track_grads:
self.g_grads.append(self.check_grads(self.G))
self.d_grads.append(self.check_grads(self.D))
step += 1
scheduler_D.step()
scheduler_G.step()<choose_model_class>
raw = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 raw.head()
netG = Generator(nz, 64, 3, SPECTRAL_NORM, NORMALIZATION, RANDOM_NOISE, PIXEL_NORM ).to(device)
<choose_model_class>
tokenizer = RobertaTokenizer.from_pretrained("roberta-base" )
netD = Discriminator(3, 64 ).to(device)
<normalization>
raw = preprocess(raw,tokenizer,config )
def weights_init(m):
if type(m)== nn.Linear or type(m)== nn.Conv2d or type(m)== nn.ConvTranspose2d:
torch.nn.init.kaiming_uniform(m.weight)
 if m.bias is not None:
m.bias.data.zero_() 
<choose_model_class>
def kfold(data,k=5,shuffle=True,file_prefix='/kaggle/working/kfold',file_suffix='tuple2'):
keys = list(data.keys())
 total_idx = list(range(len(data[keys[0]])))
 total = len(total_idx)
 valid_num = int(total/k)
 if shuffle:
random.shuffle(total_idx)
 for i in trange(k):
train_data = {}
valid_data = {}
for key in data.keys() :
train_data[key] = []
valid_data[key] = []
valid_idx = total_idx[valid_num*(k-1):valid_num*k]
for idx in valid_idx:
for key in valid_data.keys() :
valid_data[key].append(data[key][idx])
 train_idx = total_idx[:valid_num*(k-1)]+total_idx[valid_num*k:]
for idx in train_idx:
for key in train_data.keys() :
train_data[key].append(data[key][idx])
 with open('{pf}_{num}.{sf}'.format(pf=file_prefix,num=i,sf=file_suffix),'wb')as f:
pickle.dump(( train_data,valid_data),f)
 def dataset_split(data,train_ratio=0.6,valid_ratio=0.3,shuffle=True):
assert train_ratio+valid_ratio<=1 , "The train_ratio + valid_ratio should not be greater than 1"
keys = list(data.keys())
 total_idx = list(range(len(data[keys[0]])))
 total = len(total_idx)
 train_num = int(total*train_ratio)
 valid_num = int(total*valid_ratio)
 if shuffle:
random.shuffle(total_idx)
 train_data = {}
valid_data = {}
test_data = {}
for k in data.keys() :
train_data[k] = []
valid_data[k] = []
test_data[k] = []
for idx in total_idx[:train_num]:
for k in train_data.keys() :
train_data[k].append(data[k][idx])
 for idx in total_idx[train_num:train_num+valid_num]:
for k in valid_data.keys() :
valid_data[k].append(data[k][idx])
 for idx in total_idx[train_num+valid_num:]:
for k in test_data.keys() :
test_data[k].append(data[k][idx])
 return train_data,valid_data,test_data
netG.apply(weights_init)
 netD.apply(weights_init)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=lrD, betas=(beta1, beta2))
optimizerG = optim.Adam(netG.parameters() , lr=lrG, betas=(beta1, beta2))
scheduler_D = optim.lr_scheduler.ExponentialLR(optimizerD, gamma=0.99)
 scheduler_G = optim.lr_scheduler.ExponentialLR(optimizerG, gamma=0.99 )<train_model>
class MyDataset(Dataset):
def __init__(self, train_data):
self.id = train_data['id']
self.text = train_data['text']
self.input_ids = train_data['input_ids']
self.attention_mask = train_data['attention_mask']
self.label = train_data['label']
assert len(self.text)==len(self.label)
 self.idx = list(range(len(self.label)))
 def __getitem__(self, index):
idx = index
return {
'id':torch.tensor(self.id[idx],dtype=torch.long),
'text':self.text[idx],
'input_ids':torch.tensor(self.input_ids[idx],dtype=torch.long),
'attention_mask':torch.tensor(self.attention_mask[idx],dtype=torch.long),
'label':torch.tensor(self.label[idx],dtype=torch.long)
 }
def __len__(self):
return len(self.label )
trainer = Trainer(nz, netG, netD, track_grads=True)
 trainer.train(EPOCHS, train_loader, criterion, optimizerG, optimizerD, scheduler_D, scheduler_G, PRINT_EVERY, SHOW_EVERY, PLOT_EVERY, loss=LOSS)
<choose_model_class>
class ClassificationHead(nn.Module):

 def __init__(self, input_size, output_size, d_model_0, d_model_1, dropout_prob):
super().__init__()
self.norm= nn.BatchNorm1d(input_size)
 self.dense = nn.Linear(input_size, d_model_0)
 self.norm_1= nn.BatchNorm1d(d_model_0)
 self.dropout = nn.Dropout(dropout_prob)
 self.dense_1 = nn.Linear(d_model_0, d_model_1)
 self.norm_2= nn.BatchNorm1d(d_model_1)
 self.out_proj = nn.Linear(d_model_1, output_size)
 def forward(self, features, **kwargs):
x = self.norm(features)
 x = self.dropout(x)
 x = self.dense(x)
 x = torch.relu(self.norm_1(x))
x = self.dropout(x)
 x = self.dense_1(x)
 x = torch.relu(self.norm_2(x))
x = self.dropout(x)
 x = self.out_proj(x)
 return x
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2<load_pretrained>
class Model0(nn.Module):
def __init__(self,encoder,config):
super().__init__()
self.encoder = encoder
self.classifier=ClassificationHead(768,2,256,64,0.2)
 self.classifier.apply(self._init_weights)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
module.weight.data.normal_(mean=0.0, std=0.02)
 def forward(self,input_ids,attention_mask,label=None):
hidden_states = self.encoder(input_ids=input_ids,attention_mask=attention_mask)[1]
logits = self.classifier(hidden_states)
 loss_fct = CrossEntropyLoss()
if label is None:
loss = 0
else:
loss = loss_fct(logits, label)
 prob=torch.softmax(logits,-1)
 return prob,loss
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<import_modules>
class Model1(nn.Module):
def __init__(self,encoder,config):
super().__init__()
self.encoder = encoder
self.conv1d = nn.Conv1d(768*2,1024,config.max_length)
 self.classifier=ClassificationHead(1024,2,512,256,0.2)
 self.classifier.apply(self._init_weights)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
module.weight.data.normal_(mean=0.0, std=0.02)
 def forward(self,input_ids,attention_mask,label=None):
hidden_states = self.encoder(input_ids=input_ids,attention_mask=attention_mask)[-1][-2:]
hidden_states = torch.cat(hidden_states,-1)
 hidden_states = hidden_states.permute(0,2,1)
 hidden_states = self.conv1d(hidden_states)
 hidden_states = torch.squeeze(hidden_states)
 logits = self.classifier(hidden_states)
 loss_fct = CrossEntropyLoss()
if label is None:
loss = 0
else:
loss = loss_fct(logits, label)
 prob=torch.softmax(logits,-1)
 return prob,loss
In this kernel I wrote pytorch Dataset loader that loads and preprocess all images once and stores them into RAM.
 def tic() :
start_time = time.time()
return start_time
def toc(start_time = 0):
elapsed_time = time.time() - start_time
return elapsed_time
class EasyDict(dict):
def __init__(self, d=None, **kwargs):
if d is None:
d = {}
if kwargs:
d.update(**kwargs)
 for k, v in d.items() :
setattr(self, k, v)
 for k in self.__class__.__dict__.keys() :
if not(k.startswith('__')and k.endswith('__')) and not k in('update', 'pop'):
setattr(self, k, getattr(self, k))
def __setattr__(self, name, value):
if isinstance(value,(list, tuple)) :
value = [self.__class__(x)
 if isinstance(x, dict)else x for x in value]
elif isinstance(value, dict)and not isinstance(value, self.__class__):
value = self.__class__(value)
 super(EasyDict, self ).__setattr__(name, value)
 super(EasyDict, self ).__setitem__(name, value)
 __setitem__ = __setattr__
def update(self, e=None, **f):
d = e or dict()
d.update(f)
 for k in d:
setattr(self, k, d[k])
 def pop(self, k, d=None):
delattr(self, k)
 return super(EasyDict, self ).pop(k, d)
 def _get_default_config() :
c = EasyDict()
c.data = EasyDict()
c.data.n_images = -1
c.data.sbox = True
c.data.real_label = [0.6, 0.8]
c.data.fake_label = [0.0, 0.2]
c.modelG = EasyDict()
c.modelG.name = 'generator_nroyUS'
c.modelG.params = EasyDict()
c.modelG.params.winit = 'default'
c.modelG.params.nz = 128
c.modelG.params.gchs = [1024, 512, 256, 128, 64]
c.modelG.params.upsample = 'nearest'
c.modelG.params.kernel = 3
c.modelD = EasyDict()
c.modelD.name = 'discriminator_PFN'
c.modelD.params = EasyDict()
c.modelD.params.winit = 'default'
c.modelD.params.dchs = [32, 64, 128, 256]
c.modelD.params.bns = [False, False, True, True, True]
c.modelD.params.acti = True
c.modelD.params.lrelu_slope = 0.2
c.modelD.params.bn = False
c.modelD.params.mbstd = True
c.modelD.params.nfilt = 32
c.modelD.params.kernel = 5
c.train = EasyDict()
c.train.batch_size = 32
c.train.epoch = 2000
c.train.log_step = 100
c.train.params = EasyDict()
c.train.params.use_averaging_model = True
c.train.params.ema_alpha = 0.9999
c.train.params.ma_start = 100
c.train.params.ema = True
c.eval = EasyDict()
c.eval.batch_size = 50
c.optimizerG = EasyDict()
c.optimizerG.name = 'adam'
c.optimizerG.params = EasyDict()
c.optimizerG.params.lr = 0.0005
c.optimizerG.params.beta1 = 0.5
c.optimizerG.params.beta2 = 0.999
c.optimizerD = EasyDict()
c.optimizerD.name = 'adam'
c.optimizerD.params = EasyDict()
c.optimizerD.params.lr = 0.0005
c.optimizerD.params.beta1 = 0.5
c.optimizerD.params.beta2 = 0.999
c.loss = EasyDict()
c.loss.name = 'sGAN'
c.scheduler = EasyDict()
c.scheduler.name = 'none'
c.scheduler.params = EasyDict()
return c
config = _get_default_config()
pprint.PrettyPrinter(indent=2 ).pprint(config)
<init_hyperparams>
class Model2(nn.Module):
def __init__(self,encoder,config):
super().__init__()
self.encoder = encoder
self.conv1d = nn.Conv1d(768,1024,config.max_length)
 self.classifier=ClassificationHead(1024,2,512,256,0.2)
 self.classifier.apply(self._init_weights)
 def _init_weights(self, module):

 if isinstance(module,(nn.Linear, nn.Embedding)) :
module.weight.data.normal_(mean=0.0, std=0.02)
 def forward(self,input_ids,attention_mask,label=None):
hidden_states = self.encoder(input_ids=input_ids,attention_mask=attention_mask)[-1][-2:]
hidden_states = hidden_states[0] + hidden_states[1]
hidden_states = hidden_states.permute(0,2,1)
 hidden_states = self.conv1d(hidden_states)
 hidden_states = torch.squeeze(hidden_states)
 logits = self.classifier(hidden_states)
 loss_fct = CrossEntropyLoss()
if label is None:
loss = 0
else:
loss = loss_fct(logits, label)
 prob=torch.softmax(logits,-1)
 return prob,loss
if True:
root_dir = '.. /input/generative-dog-images'
else:
root_dir = '.. /input'
eval_do = False
use_scheduler = False
if config.data.sbox:
exrw = 1
exrh = 1
else:
exrw = 0
exrh = 0
n_images = config.data.n_images
g_model = config.modelG.name
d_model = config.modelD.name
g_params = config.modelG.params
d_params = config.modelD.params
nz = g_params['nz']
print(g_params)
 print(d_params)
 epochs = config.train.epoch
batch_size = config.train.batch_size
lrG = config.optimizerG.params.lr
G_beta1 = config.optimizerG.params.beta1
G_beta2 = config.optimizerD.params.beta2
lrD = config.optimizerD.params.lr
D_beta1 = config.optimizerD.params.beta1
D_beta2 = config.optimizerD.params.beta2
use_averaging_model = config.train.params.use_averaging_model
real_label = config.data.real_label
fake_label = config.data.fake_label
num_classes = 120
sz = 64
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
<normalization>
def infer(model,test_data,tokenizer,config,save=False,save_path='submission.csv'):
model.eval()
test_dl = DataLoader(MyDataset(test_data),shuffle=False,batch_size=config.batch_size)
 preds = []
idx = []
dl_iter = tqdm(test_dl,desc='[Infering]',position=0)
 device = config.device
for data in dl_iter:
text = data['text']
input_ids = data['input_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 model.eval()
prob,loss = model(input_ids,attention_mask)
 pred = torch.argmax(prob,-1 ).detach().cpu().numpy()
preds.append(pred)
 idx.append(data['id'].detach().cpu().numpy())
 dl_iter.set_postfix(id=data['id'][0].item())
 preds = np.concatenate(preds)
 idx = np.concatenate(idx)
 if save:
pd.DataFrame({'id':idx,'preds':preds} ).to_csv(save_path)
 return idx,preds
transform1 = transforms.Compose([transforms.Resize(sz),
transforms.CenterCrop(sz)])
 random_transforms = [transforms.RandomRotation(degrees=5)]
transform2 = transforms.Compose([transforms.RandomHorizontalFlip(p=0.5),
transforms.RandomApply(random_transforms, p=0.3),
transforms.ToTensor() ,
transforms.Normalize(( 0.5, 0.5, 0.5),(0.5, 0.5, 0.5)) ])
 train_dataset = DogDataset(img_dir=root_dir + '/all-dogs/all-dogs/',
anno_dir=root_dir + '/annotation/Annotation',
transform1=transform1,
transform2=transform2,
exrw=exrw,
exrh=exrh,
n_images=n_images)
 train_loader = DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=4 )<categorify>
def train(model,config,save_name='model',fold_num=0):
with open('kfold_{fold_num}.tuple2'.format(fold_num=fold_num),'rb')as f:
train_data,valid_data = pickle.load(f)
 train_dl = DataLoader(MyDataset(train_data),shuffle=True,batch_size=config.batch_size)
 valid_dl = DataLoader(MyDataset(valid_data),shuffle=True,batch_size=config.batch_size)
 optimizer = AdamW(model.parameters() , lr=1e-5, eps=1e-8,weight_decay=0.08)
 scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=int(len(train_dl)*config.train_epochs*0.2),num_training_steps=len(train_dl)*config.train_epochs)
 step_count = 0
total_loss = 0
device = torch.device("cuda")if torch.cuda.is_available() else torch.device("cpu")
 model.to(device)
 model.train()
total_f1_score = 0
for epoch in range(config.train_epochs):
dl_iter = tqdm(train_dl,desc='[Training]',position=0)
 for data in dl_iter:
text = data['text']
input_ids = data['input_ids'].to(device)
 attention_mask = data['attention_mask'].to(device)
 label = data['label'].to(device)
 model.train()
prob,loss = model(input_ids,attention_mask,label)
 loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , config.max_grad_norm)
 optimizer.step()
optimizer.zero_grad()
scheduler.step()
step_count += 1
total_loss += loss.item()
avg_loss = total_loss/step_count
pred = torch.argmax(prob,-1 ).detach().cpu().numpy()
label = label.detach().cpu().numpy()
total_f1_score += sklearn.metrics.f1_score(label, pred)
 f1_score = total_f1_score/step_count
dl_iter.set_postfix(epoch=epoch,loss=loss.item() ,avg_loss=avg_loss,f1_score=f1_score)
 model.eval()
f1_score = eval(model,valid_dl,device,-1)
 tqdm.write('[epcoh_{epoch}] f1_score: {f1_score}'.format(epoch=epoch,f1_score=f1_score))
if(epoch+1)%config.save_epoch == 0:
torch.save(model,'{model_name}_fold{fold_num}_epoch{epoch}.pt'.format(model_name=save_name,fold_num=fold_num,epoch=epoch))
model.train()
def to_img(x):
x = 0.5 *(x + 1)
 x = x.clamp(0, 1)
 x = x.view(x.size(0), 3, 64, 64)
 return x
def show_generated_img(epoch=0):
if not eval_do:
return
if epoch % 10 == 0 or epoch==epochs-1:
noise = fixed_noise
labels = fixed_labels
gen_tensors = netG(noise, labels ).to("cpu" ).clone().detach().squeeze(0)
 gen_tensors = gen_tensors/2+0.5
gen_tensors = gen_tensors.clamp(0, 1)
 gen_images = gen_tensors.numpy().transpose(0, 2, 3, 1)
 plt.subplot(1,2,1)
 plt.imshow(gen_images[0])
 plt.subplot(1,2,2)
 plt.imshow(gen_images[1])
 plt.pause(0.1)
 print('gen_image:(%f, %f)' %(gen_images.min() , gen_images.max()))
def generate_samples(outimg_dir, netG, n_images=10000):
if not os.path.exists(outimg_dir):
os.mkdir(outimg_dir)
 im_batch_size = 50
for i_batch in range(0, n_images, im_batch_size):
gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)
 gen_labels = torch.randint(num_classes,(im_batch_size,), device=device)
 gen_images =(netG(gen_z, gen_labels)+ 1)/2
images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join(outimg_dir, f'image_{i_batch+i_image:05d}.png'))
def truncated_noise_sample(batch_size=1, dim_z=128, truncation=1., seed=None):

 state = None if seed is None else np.random.RandomState(seed)
 values = truncnorm.rvs(-2, 2, size=(batch_size, dim_z), random_state=state ).astype(np.float32)
 return truncation * values
def generate_samples_truncated(outimg_dir, netG, th=1, n_images=10000):
if not os.path.exists(outimg_dir):
os.mkdir(outimg_dir)
 im_batch_size = 50
for i_batch in range(0, n_images, im_batch_size):
if th>0:
tns = truncated_noise_sample(im_batch_size, nz, th)
 tns = np.expand_dims(tns, -1)
 tns = np.expand_dims(tns, -1)
 gen_z = torch.from_numpy(tns ).to(device)
 else:
gen_z = torch.randn(im_batch_size, nz, 1, 1, device=device)
 gen_labels = torch.randint(num_classes,(im_batch_size,), device=device)
 gen_images =(netG(gen_z, gen_labels)+ 1)/2
images = gen_images.to("cpu" ).clone().detach()
images = images.numpy().transpose(0, 2, 3, 1)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join(outimg_dir, f'image_{i_batch+i_image:05d}.png'))<load_pretrained>
RobertaModel.from_pretrained('roberta-base' )
def init_weights_normal(m):
w_mean = 0.0
w_std = 0.02
if type(m)== nn.ConvTranspose2d:
print('gaussian initialization')
 torch.nn.init.normal_(m.weight, w_mean, w_std)
 elif type(m)== nn.Conv2d:
print('gaussian initialization')
 torch.nn.init.normal_(m.weight, w_mean, w_std)
 def init_weights_xavier(m):
if type(m)== nn.Linear or type(m)== nn.Conv2d:
print('xavier_uniform initialization')
 torch.nn.init.xavier_uniform_(m.weight)
 if m.bias is not None:
m.bias.data.fill_(0.)
class PixelwiseNorm(nn.Module):
def __init__(self):
super(PixelwiseNorm, self ).__init__()
self.alpha = 1e-8
def forward(self, x):

 if False:
y = x.pow(2.).mean(dim=1, keepdim=True ).add(self.alpha ).sqrt()
y = x / y
else:
mean = torch.mean(x * x, 1, keepdim=True)
 dom = torch.rsqrt(mean + self.alpha)
 y = x * dom
return y
class MinibatchStdDev(nn.Module):

 def __init__(self):

 super(MinibatchStdDev, self ).__init__()
self.alpha = 1e-8
def forward(self, x):

 batch_size, _, height, width = x.shape
y = x - x.mean(dim=0, keepdim=True)
 y = th.sqrt(y.pow(2.).mean(dim=0, keepdim=False)+ self.alpha)
 y = y.mean().view(1, 1, 1, 1)
 y = y.repeat(batch_size,1, height, width)
 y = th.cat([x, y], 1)
 return y
class UpsamplingNN(nn.Module):
def __init__(self, n_input, n_output, k_size=3, stride=1, padding=1, bias=False):
super().__init__()
layers = [
nn.Upsample(scale_factor=2, mode='nearest'),
nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=bias),
]
self.convlayer = nn.Sequential(*layers)
 self.weight = self.convlayer[1].weight
def forward(self, x):
y = self.convlayer(x)
 return y
class Generator(nn.Module):
def __init__(self, nz, nfeats, nchannels, upsample='nearest', kernel=3):
super().__init__()
if upsample=='pixelshuffle_conv':
upsample = ConvPS
elif upsample=='conv_pixelshuffle':
upsample = ConvPS_convfirst
elif upsample=='nearest':
upsample = UpsamplingNN
elif upsample=='bilinear':
upsample = UpsamplingBL
else:
print('unknown upsample: ', upsample)
 sys.exit()
print(self.__class__)
 print('upsample: ', upsample)
 print('nfeats: ', nfeats)
 print('kernel: ', kernel)
 pad = kernel//2
self.conv1 = spectral_norm(nn.ConvTranspose2d(nz, nfeats * 8, 4, 1, 0, bias=False))
self.conv2 = spectral_norm(upsample(nfeats * 8, nfeats * 8, kernel, 1, pad, bias=False))
self.conv3 = spectral_norm(upsample(nfeats * 8, nfeats * 4, kernel, 1, pad, bias=False))
self.conv4 = spectral_norm(upsample(nfeats * 4, nfeats * 2, kernel, 1, pad, bias=False))
self.conv5 = spectral_norm(upsample(nfeats * 2, nfeats, kernel, 1, pad, bias=False))
self.conv6 = spectral_norm(nn.Conv2d(nfeats, nchannels, 5, 1, 2, bias=False))
self.pixnorm = PixelwiseNorm()
def forward(self, x, labels=None):
x = F.leaky_relu(self.conv1(x))
x = F.leaky_relu(self.conv2(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv3(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv4(x))
x = self.pixnorm(x)
 x = F.leaky_relu(self.conv5(x))
x = self.pixnorm(x)
 x = torch.tanh(self.conv6(x))
return x
def get_generator(nz, channels=3, nfilt=32, winit=None, upsample='nearest', kernel=3, **_):
model = Generator(nz, nfeats=nfilt, nchannels=channels, upsample=upsample, kernel=kernel)
 if winit=='normal':
model.apply(init_weights_normal)
 elif winit=='xavier':
model.apply(init_weights_xavier)
 return model<define_search_model>
try:
os.mkdir('/kaggle/working/models/')
 except:
pass
config.train_epochs = 3
config.save_epoch = 3
for fn in range(5):
tqdm.write('='*10+'FOLD_{fn}'.format(fn=fn)+'='*10)
 encoder = RobertaModel.from_pretrained('roberta-base')
 encoder.config.output_hidden_states = True
model = Model2(encoder,config)
 train(model,config,save_name='/kaggle/working/models/model2',fold_num=fn)
 del model
del encoder
<choose_model_class><EOS> 
test_raw = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test_data = preprocess(test_raw,tokenizer,config)
 vote = []
model_num = 0
for home, dirs, files in os.walk('/kaggle/working/models/'):
for filename in files:
if filename.endswith('.pt'):
model_num += 1
model = torch.load(os.path.join(home,filename))
idx,preds = infer(model,test_data,tokenizer,config,save_path='{mn}.csv'.format(mn=filename[:-3]))
vote.append(preds)
 del model
vote = np.array(vote ).T
result = np.sum(vote,axis=1)[:,None]
threshold = int(model_num/2)
 print('Model Num: {mn}
Threshold: {ts}'.format(mn=model_num,ts=threshold))
result = np.apply_along_axis(lambda x:1 if x>threshold else 0,1,result)
 pd.DataFrame({'id':idx,'target':result} ).to_csv('/kaggle/working/submission.csv',index=0 )