Kaggle competition: Natural Language Processing with Disaster Tweets
class ConditionalGANLoss:

 def __init__(self, dis):
self.dis = dis
def dis_loss(self, real_samps, fake_samps, labels, height, alpha):
raise NotImplementedError("dis_loss method has not been implemented")
 def gen_loss(self, real_samps, fake_samps, labels, height, alpha):
raise NotImplementedError("gen_loss method has not been implemented" )<compute_train_metric>
train_tweet = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test_tweet = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 df_map = {"train": train_tweet, "test": test_tweet}
class StandardLoss(ConditionalGANLoss):
def __init__(self, dis):
super().__init__(dis)
 self.criterion = BCEWithLogitsLoss(reduction='sum')
 def dis_loss(self, real_samps, fake_samps, labels, height, alpha):
assert real_samps.device == fake_samps.device, "Different devices"
preds_real = self.dis(real_samps, labels, height, alpha)
 preds_fake = self.dis(fake_samps, labels, height, alpha)
 labels_real = torch.from_numpy(np.random.uniform(0.5, 0.99, real_samps.size() [0])).float().cuda()
labels_fake = torch.from_numpy(np.random.uniform(0, 0.25, fake_samps.size() [0])).float().cuda()
real_loss = self.criterion(preds_real.view(-1), labels_real)
 fake_loss = self.criterion(preds_fake.view(-1), labels_fake)
 return real_loss + fake_loss
def gen_loss(self, _, fake_samps, labels, height, alpha):
preds_fake = self.dis(fake_samps, labels, height, alpha)
 labels_real = torch.from_numpy(np.random.uniform(0.5, 0.99, fake_samps.size() [0])).float().cuda()
return self.criterion(preds_fake.view(-1), labels_real )<compute_test_metric>
check_null_values(df_map )
class Hinge(ConditionalGANLoss):
def __init__(self, dis):
super().__init__(dis)
 def dis_loss(self, real_samps, fake_samps, labels, height, alpha):
r_preds = self.dis(real_samps, labels, height, alpha)
 f_preds = self.dis(fake_samps, labels, height, alpha)
 loss =(torch.mean(th.nn.ReLU()(1 - r_preds)) + torch.mean(th.nn.ReLU()(1 + f_preds)))
 return loss
def gen_loss(self, _, fake_samps, labels, height, alpha):
return - torch.mean(self.dis(fake_samps, labels, height, alpha))<normalization>
def word_list(sentences):
result = [word.lower() for sentence in sentences
for word in word_tokenize(sentence)]
return result
def create_corpus(train_tweet):
df = train_tweet.copy()
class_0_corpus = word_list(df.query("target == 0")["text"])
 class_1_corpus = word_list(df.query("target == 1")["text"])
 return class_0_corpus, class_1_corpus
def update_average(model_old, model_new, beta):
def toggle_grad(model, requires_grad):
for p in model.parameters() :
p.requires_grad_(requires_grad)
 toggle_grad(model_old, False)
 toggle_grad(model_new, False)
 param_dict_new = dict(model_new.named_parameters())
 for param_name, param_old in model_old.named_parameters() :
param_new = param_dict_new[param_name]
assert(param_old is not param_new)
 param_old.copy_(beta * param_old +(1.- beta)* param_new)
 toggle_grad(model_old, True)
 toggle_grad(model_new, True)
<load_pretrained>
class_0_corpus, class_1_corpus = create_corpus(train_tweet)
 punc_list = [punct for punct in string.punctuation]
stop = list(stop )
def save_model_weights(model, filename, verbose=1):
if verbose:
print(f'-> Saving weights to {filename}')
 torch.save(model.state_dict() , filename)
 def load_model_weights(model, filename, verbose=1):
if verbose:
print(f'-> Loading weights from {filename}')
 model.load_state_dict(torch.load(filename))
return model<choose_model_class>
def sum_all() :
start = time.time()
result = 0
for num in range(100):
result += num
end = time.time()
print("Elapsed time: {:4f} sec".format(end - start))
print("The sum is {}".format(result))
sum_all()
class ConditionalProGAN:
def __init__(self, num_classes=120, depth=7, latent_size=128, embed_dim=64,
lr_g=0.001, lr_d=0.001, n_critic=1, use_eql=True, use_spec_norm=False,
loss=StandardLoss, use_ema=True, ema_decay=0.999):
self.gen = Generator(depth=depth, latent_size=latent_size,
use_eql=use_eql, use_spec_norm=False ).cuda()
self.dis = ConditionalDiscriminator(num_classes, height=depth, feature_size=latent_size,
use_eql=use_eql, use_spec_norm=use_spec_norm ).cuda()
self.gen = DataParallel(self.gen)
 self.dis = DataParallel(self.dis)
 self.latent_size = latent_size
self.num_classes = num_classes
self.depth = depth
self.use_ema = use_ema
self.ema_decay = ema_decay
self.n_critic = n_critic
self.use_eql = use_eql
self.drift = 0.001
self.lr_g = lr_g
self.lr_d = lr_d
self.gen_optim = Adam(self.gen.parameters() , lr=self.lr_g, betas=(0.5, 0.99), eps=1e-8)
 self.dis_optim = Adam(self.dis.parameters() , lr=self.lr_d, betas=(0.5, 0.99), eps=1e-8)
 try:
self.loss = loss(self.dis)
 except:
self.loss = loss(self.dis, drift=self.drift, use_gp=True)
 if self.use_ema:
self.gen_shadow = copy.deepcopy(self.gen)
 self.ema_updater = update_average
self.ema_updater(self.gen_shadow, self.gen, beta=0)
 def __progressive_downsampling(self, real_batch, depth, alpha):

 down_sample_factor = int(np.power(2, self.depth - depth - 1))
prior_downsample_factor = max(int(np.power(2, self.depth - depth)) , 0)
 ds_real_samples = AvgPool2d(down_sample_factor )(real_batch)
 if depth > 0:
prior_ds_real_samples = interpolate(AvgPool2d(prior_downsample_factor )(real_batch),
scale_factor=2)
 else:
prior_ds_real_samples = ds_real_samples
real_samples =(alpha * ds_real_samples)+(( 1 - alpha)* prior_ds_real_samples)
 return real_samples
def optimize_discriminator(self, noise, real_batch, labels, depth, alpha):
real_samples = self.__progressive_downsampling(real_batch, depth, alpha)
 loss_val = 0
for _ in range(self.n_critic):
fake_samples = self.gen(noise, depth, alpha ).detach()
loss = self.loss.dis_loss(real_samples, fake_samples, labels, depth, alpha)
 self.dis_optim.zero_grad()
loss.backward()
self.dis_optim.step()
loss_val += loss.item()
return loss_val / self.n_critic
def optimize_generator(self, noise, real_batch, labels, depth, alpha):
real_samples = self.__progressive_downsampling(real_batch, depth, alpha)
 fake_samples = self.gen(noise, depth, alpha)
 loss = self.loss.gen_loss(real_samples, fake_samples, labels, depth, alpha)
 self.gen_optim.zero_grad()
loss.backward()
self.gen_optim.step()
if self.use_ema:
self.ema_updater(self.gen_shadow, self.gen, self.ema_decay)
 return loss.item()
def one_hot_encode(self, labels):
if not hasattr(self, "label_oh_encoder"):
self.label_oh_encoder = th.nn.Embedding(self.num_classes, self.num_classes)
 self.label_oh_encoder.weight.data = th.eye(self.num_classes)
 return self.label_oh_encoder(labels.view(-1))
@staticmethod
def scale(imgs):
def norm(img, inf, sup):
img.clamp_(min=inf, max=sup)
 img.add_(-inf ).div_(sup - inf + 1e-5)
 for img in imgs:
norm(img, float(img.min()), float(img.max()))
@staticmethod
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values
def generate(self, depth=None, alpha=1, noise=None, races=None, n=64, n_plot=0):
if depth is None:
depth = self.depth - 1
if noise is None:
noise = th.randn(n, self.latent_size - self.num_classes ).cuda()
if races is None:
races = torch.from_numpy(np.random.choice(range(self.num_classes), size=n)).long()
label_information = self.one_hot_encode(races ).cuda()
gan_input = th.cat(( label_information, noise), dim=-1)
 if self.use_ema:
generated_images = self.gen_shadow(gan_input, depth, alpha ).detach().cpu()
else:
generated_images = self.gen(gan_input, depth, alpha ).detach().cpu()
generated_images.add_(1 ).div_(2)
 images = generated_images.clone().numpy().transpose(0, 2, 3, 1)
 if n_plot >= 5:
plt.figure(figsize=(15, 3 * n_plot//5))
for i in range(n_plot):
plt.subplot(n_plot//5, 5, i+1)
 plt.imshow(images[i])
 plt.axis('off')
 plt.title(dataset.classes[races.cpu().numpy() [i]])
 plt.show()
return generated_images
def generate_score(self, depth=None, alpha=1, noise=None, races=None, n=64, n_plot=0):
if depth is None:
depth = self.depth - 1
if noise is None:
noise = th.randn(n, self.latent_size - self.num_classes ).cuda()
if races is None:
races = torch.from_numpy(np.random.choice(range(self.num_classes), size=n)).long()
label_information = self.one_hot_encode(races ).cuda()
gan_input = th.cat(( label_information, noise), dim=-1)
 if self.use_ema:
generated_images = self.gen_shadow(gan_input, depth, alpha ).detach().cpu()
else:
generated_images = self.gen(gan_input, depth, alpha ).detach().cpu()
generated_images.add_(1 ).div_(2)
 images = generated_images.clone().numpy().transpose(0, 2, 3, 1)
 scores = nn.Sigmoid()(self.dis(generated_images, races, depth, alpha)).cpu().detach().numpy()
if n_plot >= 5:
plt.figure(figsize=(15, 3 * n_plot//5))
for i in range(n_plot):
plt.subplot(n_plot//5, 5, i+1)
 plt.imshow(images[i])
 plt.axis('off')
 plt.title(dataset.classes[races.cpu().numpy() [i]] + f' - {scores[i]:.3f}')
 plt.show()
return images, generated_images, scores, races.cpu().numpy()
def plot_race(self, race_idx, depth=4, alpha=1, n_plot=5, n=128):
races = np.concatenate(( np.array([race_idx]* n_plot),
np.random.choice(range(self.num_classes), size=n - n_plot)))
 races = torch.from_numpy(races ).long()
self.generate(depth, alpha=alpha, races=races, n=n, n_plot=n_plot)
 def compute_mifid(self, alpha=1, folder='.. /tmp_images', n_images=10000, im_batch_size=100):
if os.path.exists(folder):
shutil.rmtree(folder, ignore_errors=True)
 os.mkdir(folder)
 for i_b in range(0, n_images, im_batch_size):
gen_images = self.generate(n=im_batch_size)
 for i_img in range(gen_images.size(0)) :
save_image(gen_images[i_img, :, :, :], os.path.join(folder, f'img_{i_b+i_img}.png'))
if len(os.listdir('.. /tmp_images')) != n_images:
print(len(os.listdir('.. /tmp_images')))
 mifid = compute_mifid(folder, DATA_PATH, WEIGHTS_PATH, model_params)
 shutil.rmtree(folder, ignore_errors=True)
 return mifid
def train(self, dataset, epochs, batch_sizes, fade_in_percentage, ema_decays, start_depth=0, verbose=1):
assert self.depth == len(batch_sizes), "batch_sizes not compatible with depth"
infos = {'resolution': [], 'discriminator_loss': [], 'generator_loss': []}
self.gen.train()
self.dis.train()
if self.use_ema:
self.gen_shadow.train()
fixed_noise = torch.randn(128, self.latent_size - self.num_classes ).cuda()
fixed_races = torch.from_numpy(np.random.choice(range(self.num_classes), size=128)).long()
for current_depth in range(start_depth, self.depth):
current_res = np.power(2, current_depth + 2)
 print("
-> Current resolution: %d x %d
" %(current_res, current_res))
data = torch.utils.data.DataLoader(dataset, batch_size=batch_sizes[current_depth], num_workers=4, shuffle=True)
 self.ema_decay = ema_decays[current_depth]
ticker = 1
for epoch in range(1, epochs[current_depth] + 1):
start_time = time.time()
d_loss = 0
g_loss = 0
fader_point = fade_in_percentage[current_depth] // 100 * epochs[current_depth] * len(iter(data))
step = 0
if current_res == 64 and(epoch % 50)== 0:
self.ema_decay = 0.9 + self.ema_decay / 10
for(i, batch)in enumerate(data, 1):
alpha = ticker / fader_point if ticker <= fader_point else 1
images, labels = batch
images = images.cuda()
labels = labels.view(-1, 1)
 label_information = self.one_hot_encode(labels ).cuda()
latent_vector = th.randn(images.shape[0], self.latent_size - self.num_classes ).cuda()
gan_input = th.cat(( label_information, latent_vector), dim=-1)
 dis_loss = self.optimize_discriminator(gan_input, images,
labels, current_depth, alpha)
 d_loss += dis_loss / len(data)
 gen_loss = self.optimize_generator(gan_input, images,
labels, current_depth, alpha)
 g_loss += gen_loss / len(data)
 ticker += 1
step += 1
infos['discriminator_loss'].append(d_loss)
 infos['generator_loss'].append(g_loss)
 infos['resolution'].append(current_res)
 if epoch % verbose == 0:
elapsed_time = time.time() - start_time
print(f'Epoch {epoch}/{epochs[current_depth]} lr_g={self.lr_g:.1e} lr_d={self.lr_d:.1e} ema_decay={self.ema_decay:.4f}', end=' ')
 print(f'disc_loss={d_loss:.3f} gen_loss={g_loss:.3f} t={elapsed_time:.0f}s')
 if epoch %(verbose * 25)== 0 and current_res == 64:
for i in range(5):
self.plot_race(i, depth=current_depth, alpha=alpha, n_plot=5, n=batch_sizes[0])
 -> MiFID at epoch {epoch} is {score:.3f}
')
 seed_everything(seed + epoch)
 elif epoch %(verbose * 10)== 0:
self.generate(current_depth, alpha=alpha, noise=fixed_noise, races=fixed_races, n=batch_sizes[0], n_plot=10)
 if time.time() - KERNEL_START_TIME > 32000:
print('Time limit reached, interrupting training.')
 break
self.gen.eval()
self.dis.eval()
if self.use_ema:
self.gen_shadow.eval()
return infos<define_variables>
def mul_all() :
start = time.time()
result = 0
for num in range(100):
result *= num
end = time.time()
print("Elapsed time: {:4f} sec".format(end - start))
print("The product is {}".format(result))
mul_all()
depth = 5
latent_size = 256<choose_model_class>
def simple_decorator(func):
def decorated() :
start = time.time()
result = func()
end = time.time()
print("{} function".format(func.__name__))
print("Elapsed time: {:4f} sec".format(end - start))
print("The result is {}".format(result))
print()
return result
return decorated
loss = Hinge
lr_d = 6e-3
lr_g = 6e-3<choose_model_class>
@simple_decorator
def sum_all() :
result = 0
for x in range(100):
result += x
return result
@simple_decorator
def mul_all() :
result = 0
for x in range(100):
result *= x
return result
pro_gan = ConditionalProGAN(num_classes=nb_classes, depth=depth, latent_size=latent_size,
loss=loss, lr_d=lr_d, lr_g=lr_g,
use_ema=True, use_eql=True, use_spec_norm=False )<define_search_space>
common_words(class_0_corpus, name="Class 0 Train")
 common_words(class_1_corpus, name="Class 1 Train" )
num_epochs = [5, 10, 20, 40, 100]
fade_ins = [50, 20, 20, 10, 5]
batch_sizes = [64] * 5
ema_decays = [0.9, 0.9, 0.99, 0.99, 0.99]<train_model>
noise_patterns = {
"url" : r"https?(://\S+|\S+)|www\.\S+",
"html_tag" : r"<.*?>",
"non_ascii" : r"[^\x00-\x7f]+",
"RT word" : r"\b[Rr][Tt]\b",
"amp word" : r"\bamp\b",
}
def noise_check(df_map, patterns):
for key, df in df_map.items() :
print("{} DataFrame noises: ".format(key))
print("-" * 30)
 for indicator, pattern in patterns.items() :
count = df["text"].str.contains(pattern ).sum()
print("There were {:5} rows with {}".format(count, indicator))
print()
noise_check(df_map, noise_patterns )
infos = pro_gan.train(
 dataset=dataset,
epochs=num_epochs,
fade_in_percentage=fade_ins,
batch_sizes=batch_sizes,
ema_decays=ema_decays,
verbose=1
)<define_variables>
def add_paren(text):
return "({})".format(text)
 noises_all = "|".join(add_paren(pattern)for pattern in noise_patterns.values())
 print(noises_all )
use_class_probs = False<categorify>
def clean_noise(text):
return re.sub(noises_all, r" ", text)
 train_tweet["text"] = train_tweet["text"].apply(clean_noise)
 test_tweet["text"] = test_tweet["text"].apply(clean_noise)
 noise_check(df_map, noise_patterns )
class_mifids = {0: 59.588, 1: 72.73, 2: 67.551, 3: 55.2313, 4: 106.411, 5: 91.73, 6: 76.835, 7: 116.532, 8: 133.186, 9: 97.613, 10: 60.579, 11: 88.973, 12: 100.450, 13: 70.434, 14: 109.0, 15: 67.517, 16: 70.190, 17: 64.704, 18: 93.84, 19: 85.469, 20: 76.62, 21: 59.845, 22: 88.898, 23: 106.39, 24: 82.955, 25: 65.451, 26: 58.066, 27: 51.016, 28: 86.535, 29: 55.7707, 30: 101.185, 31: 107.090, 32: 72.13, 33: 82.097, 34: 75.549, 35: 101.369, 36: 58.6547, 37: 58.2671, 38: 119.541, 39: 102.502, 40: 50.871, 41: 62.708, 42: 68.374, 43: 91.805, 44: 121.485, 45: 63.633, 46: 81.873, 47: 84.495, 48: 70.738, 49: 86.552, 50: 88.067, 51: 71.89, 52: 94.634, 53: 113.624, 54: 83.190, 55: 98.981, 56: 95.340, 57: 64.076, 58: 117.179, 59: 88.268, 60: 72.986, 61: 98.971, 62: 85.285, 63: 85.808, 64: 79.254, 65: 64.701, 66: 86.365, 67: 70.703, 68: 93.268, 69: 86.699, 70: 52.1906, 71: 113.921, 72: 97.362, 73: 91.260, 74: 86.280, 75: 79.446, 76: 93.833, 77: 90.210, 78: 78.724, 79: 71.74, 80: 63.501, 81: 68.04, 82: 63.95, 83: 64.699, 84: 67.830, 85: 74.525, 86: 95.0, 87: 80.052, 88: 66.01, 89: 82.810, 90: 89.005, 91: 70.746, 92: 71.503, 93: 74.91, 94: 93.378, 95: 82.145, 96: 83.216, 97: 82.569, 98: 65.660, 99: 104.155, 100: 92.105, 101: 81.490, 102: 61.454, 103: 93.271, 104: 75.405, 105: 64.791, 106: 67.661, 107: 96.165, 108: 66.574, 109: 80.197, 110: 89.485, 111: 98.295, 112: 73.864, 113: 59.862, 114: 61.0195, 115: 86.696, 116: 71.2, 117: 91.355, 118: 62.981, 119: 65.444}<define_variables>
def random_sampling(df_map, n=5):
for key, df in df_map.items() :
print("{} sample from {} dataset".format(n, key))
print("-" * 30)
 print(df["text"].sample(n))
print()
random_sampling(df_map )
threshold = 200
kept_classes = [k for k in class_mifids.keys() if class_mifids[k] < threshold]
print(f'Kept {len(kept_classes)} classes' )<define_variables>
word1 = "The"
word2 = "the"
print(word1.lower() == word2 )
probs = [class_mifids[k] for k in class_mifids.keys() ]
probs =(probs - np.min(probs)) / np.sum(probs)/ 4
probs = np.clip(( 1/120 - probs), 1/1000, 1/50)
 probs /= np.sum(probs )<define_variables>
def decontracted(phrase):
phrase = re.sub(r"won't", "will not", phrase)
 phrase = re.sub(r"can't", "can not", phrase)
 phrase = re.sub(r"n't", " not", phrase)
 phrase = re.sub(r"'re", " are", phrase)
 phrase = re.sub(r"'s", " is", phrase)
 phrase = re.sub(r"'d", " would", phrase)
 phrase = re.sub(r"'ll", " will", phrase)
 phrase = re.sub(r"'t", " not", phrase)
 phrase = re.sub(r"'ve", " have", phrase)
 phrase = re.sub(r"'m", " am", phrase)
 return phrase
print(decontracted("I'm Lord of Darkness"))
print(decontracted("Who've made this burger?"))
im_batch_size = 100
n_images = 10000<define_variables>
def punct2espace(text):
return re.sub(",", "", text)
 def punct2wspace(text):
return re.sub(r"[{}]+".format(string.punctuation), " ", text)
 def residual_punc(text):
table = str.maketrans("", "", string.punctuation)
 return text.translate(table)
 def normalize_wspace(text):
return re.sub(r"\s+", " ", text)
 def replace_punctuations(text):
text = punct2espace(text)
 text = punct2wspace(text)
 text = residual_punc(text)
 text = normalize_wspace(text)
 return text.strip()
replace_punctuations(r"@@rakka@@ alhazimi@@hai typhoon--devastation \\\\\conclusively" )
for i_batch in tqdm(range(0, n_images, im_batch_size)) :
if use_class_probs:
races = np.random.choice(kept_classes, size=im_batch_size, p=probs)
 races = torch.from_numpy(races ).long()
gen_images = pro_gan.generate(races=races, n=im_batch_size)
 else:
gen_images = pro_gan.generate(n=im_batch_size)
 for i_image in range(gen_images.size(0)) :
save_image(gen_images[i_image, :, :, :], os.path.join('.. /output_images', f'img_{i_batch+i_image}.png'))<load_pretrained>
!pip install pyspellchecker
shutil.make_archive('images', 'zip', '.. /output_images' )<train_model>
spell = SpellChecker()
def correct_spellings(text):
corrected_text = []
misspelled_words = spell.unknown(text.split())
 for word in text.split() :
if word in misspelled_words:
corrected_text.append(spell.correction(word))
else:
corrected_text.append(word)
 return " ".join(corrected_text)
 text = "speling correctin"
correct_spellings(text )
print(f"Done in {(time.time() - begin)/60:.1f} minutes" )<import_modules>
lemmatizer = WordNetLemmatizer()
wordnet_map = {"N":wordnet.NOUN, "V":wordnet.VERB, "J":wordnet.ADJ, "R":wordnet.ADV}
def lemmatize_words(text):
pos_tagged_text = nltk.pos_tag(text.split())
 return " ".join([lemmatizer.lemmatize(word, wordnet_map.get(pos[0], wordnet.NOUN)) for word, pos in pos_tagged_text])
 lemmatize_words("rakka alhazimi are cool" )
import numpy as np
import pandas as pd
import os
import cv2
import random
import matplotlib.pyplot as plt
from PIL import Image
from skimage.util import random_noise
import xml.etree.ElementTree as ET
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.autograd import Variable
from collections import OrderedDict
from torch.utils.data import TensorDataset, DataLoader,Dataset
from tqdm import tqdm_notebook as tqdm<set_options>
def normalize_text(text):
text = text.lower()
text = decontracted(text)
 text = replace_punctuations(text)
 text = lemmatize_words(text)
 return text
normalize_text("Hello andrew, I'm from kuvukiland nuce to meet you" )
def seed_everything(seed=19960720):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<categorify>
!pip install pandarallel
def summary(model, input_size, batch_size=-1, device="cuda"):
def register_hook(module):
def hook(module, input, output):
class_name = str(module.__class__ ).split(".")[-1].split("'")[0]
module_idx = len(summary)
 m_key = "%s-%i" %(class_name, module_idx + 1)
 summary[m_key] = OrderedDict()
summary[m_key]["input_shape"] = list(input[0].size())
 summary[m_key]["input_shape"][0] = batch_size
if isinstance(output,(list, tuple)) :
summary[m_key]["output_shape"] = [
[-1] + list(o.size())[1:] for o in output
]
else:
summary[m_key]["output_shape"] = list(output.size())
 summary[m_key]["output_shape"][0] = batch_size
params = 0
if hasattr(module, "weight")and hasattr(module.weight, "size"):
params += torch.prod(torch.LongTensor(list(module.weight.size())))
 summary[m_key]["trainable"] = module.weight.requires_grad
if hasattr(module, "bias")and hasattr(module.bias, "size"):
params += torch.prod(torch.LongTensor(list(module.bias.size())))
 summary[m_key]["nb_params"] = params
if(
 not isinstance(module, nn.Sequential)
 and not isinstance(module, nn.ModuleList)
 and not(module == model)
):
hooks.append(module.register_forward_hook(hook))
device = device.lower()
assert device in [
"cuda",
"cpu",
], "Input device is not valid, please specify 'cuda' or 'cpu'"
if device == "cuda" and torch.cuda.is_available() :
dtype = torch.cuda.FloatTensor
else:
dtype = torch.FloatTensor
if isinstance(input_size, tuple):
input_size = [input_size]
x = [torch.rand(2, *in_size ).type(dtype)for in_size in input_size]
summary = OrderedDict()
hooks = []
model.apply(register_hook)
 model(*x)
 for h in hooks:
h.remove()
print("----------------------------------------------------------------")
 line_new = "{:>20} {:>25} {:>15}".format("Layer(type)", "Output Shape", "Param
print(line_new)
 print("================================================================")
 total_params = 0
total_output = 0
trainable_params = 0
for layer in summary:
line_new = "{:>20} {:>25} {:>15}".format(
 layer,
str(summary[layer]["output_shape"]),
"{0:,}".format(summary[layer]["nb_params"]), 
)
 total_params += summary[layer]["nb_params"]
total_output += np.prod(summary[layer]["output_shape"])
 if "trainable" in summary[layer]:
if summary[layer]["trainable"] == True:
trainable_params += summary[layer]["nb_params"]
print(line_new)
 total_input_size = abs(np.prod(input_size)* batch_size * 4./(1024 ** 2.))
 total_output_size = abs(2.* total_output * 4./(1024 ** 2.))
 total_params_size = abs(total_params.numpy() * 4./(1024 ** 2.))
 total_size = total_params_size + total_output_size + total_input_size
print("================================================================")
 print("Total params: {0:,}".format(total_params))
print("Trainable params: {0:,}".format(trainable_params))
print("Non-trainable params: {0:,}".format(total_params - trainable_params))
print("----------------------------------------------------------------")
 print("Input size(MB): %0.2f" % total_input_size)
 print("Forward/backward pass size(MB): %0.2f" % total_output_size)
 print("Params size(MB): %0.2f" % total_params_size)
 print("Estimated Total Size(MB): %0.2f" % total_size)
 print("----------------------------------------------------------------")
<normalization>
pandarallel.initialize()
start = time.time()
train_tweet["text"] = train_tweet["text"].parallel_apply(normalize_text)
 test_tweet["text"] = test_tweet["text"].parallel_apply(normalize_text)
 elapsed = time.time() - start
print("Elapsed time: {:.4f} min".format(elapsed / 60))
class Generator(nn.Module):
def __init__(self, nz=128, channels=3):
super(Generator, self ).__init__()
self.nz = nz
self.channels = channels
def convlayer(n_input, n_output, k_size=4, stride=2, padding=0):
block = [
nn.ConvTranspose2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False),
nn.BatchNorm2d(n_output),
nn.ReLU(inplace=True),
]
return block
self.model = nn.Sequential(
 *convlayer(self.nz, 1024, 4, 1, 0),
*convlayer(1024, 512, 4, 2, 1),
*convlayer(512, 256, 4, 2, 1),
*convlayer(256, 128, 4, 2, 1),
*convlayer(128, 64, 4, 2, 1),
nn.ConvTranspose2d(64, self.channels, 3, 1, 1),
nn.Tanh() 
)
 def forward(self, z):
z = z.view(-1, self.nz, 1, 1)
 img = self.model(z)
 return img<categorify>
train_tweet.to_csv("train_tweet.csv")
 test_tweet.to_csv("test_tweet.csv" )
class Discriminator(nn.Module):
def __init__(self, channels=3):
super(Discriminator, self ).__init__()
self.channels = channels
def convlayer(n_input, n_output, k_size=4, stride=2, padding=0, bn=False):
block = [nn.Conv2d(n_input, n_output, kernel_size=k_size, stride=stride, padding=padding, bias=False)]
if bn:
block.append(nn.BatchNorm2d(n_output))
block.append(nn.LeakyReLU(0.2, inplace=True))
return block
self.model = nn.Sequential(
 *convlayer(self.channels, 64, 3, 1, 1),
*convlayer(64, 128, 4, 2, 1),
*convlayer(128, 256, 4, 2, 1, bn=True),
*convlayer(256, 512, 4, 2, 1, bn=True),
*convlayer(512, 1024, 4, 2, 1, bn=True),
nn.Conv2d(1024, 1, 4, 1, 0, bias=False), 
)
 def forward(self, imgs):
logits = self.model(imgs)
 out = torch.sigmoid(logits)
 return out.view(-1, 1 )<define_variables>
english_vocab = nltk.corpus.words.words("en")
 english_vocab = set(english_vocab)
 corpus_0, corpus_1 = create_corpus(train_tweet)
 corpus = corpus_0 + corpus_1
tweet_words = set(corpus )
image = glob.glob('.. /input/all-dogs/all-dogs/*')
 breed = glob.glob('.. /input/annotation/Annotation/*')
 annot = glob.glob('.. /input/annotation/Annotation/*/*')
 print(len(image), len(breed), len(annot))<define_variables>
unknown_words = list(tweet_words.difference(english_vocab))
print("There were {} unknown words.".format(len(unknown_words)) )
def get_image(annot):

 img_path = '.. /input/all-dogs/all-dogs/'
file = annot.split('/')
 img_class = file[-2]
img_filename = img_path+file[-1]+'.jpg'
return img_filename, img_class<define_variables>
MAX_LEN = 50
def tokenize(df_map):
sentence_list = []
for key, df in df_map.items() :
sentence = [" ".join(word for word in sen.split() if not word.isdigit())
 for sen in df["text"]]
length = df.shape[0]
sentence_list += sentence
tokenizer = preprocessing.text.Tokenizer()
tokenizer.fit_on_texts(sentence_list)
 tensor = tokenizer.texts_to_sequences(sentence_list)
 tensor = preprocessing.sequence.pad_sequences(tensor, padding="post", maxlen=MAX_LEN)
 input_tensor_train, input_tensor_test = tensor[:-length], tensor[-length:]
return input_tensor_train, input_tensor_test, tokenizer
X_train, X_test, tokenizer = tokenize(df_map)
 y_train = train_tweet["target"].values
y_train = y_train.reshape(-1, 1 )
select_dogs = []
select_bbox = []<feature_engineering>
with open(".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec", "r")as vector:
fasttext = vector.readlines()
len(fasttext )
def get_class_dict(breed):
class_dict = {}
for i in range(len(breed)) :
class_ = breed[i].split('/')
 class_dict[class_[-1]] = i
return class_dict
class_dict = get_class_dict(breed )<prepare_x_and_y>
with open(".. /input/glove-global-vectors-for-word-representation/glove.6B.200d.txt", "r")as vector:
glove = vector.readlines()
len(glove )
def get_cleaned_data(image, annot, class_dict):
select_dogs = []
select_bbox = []
select_labels = []
for i in range(len(image)) :
bbox = get_bbox(annot[i])
 bbox = bbox[0]
dog, dog_class = get_image(annot[i])
 if dog == '.. /input/all-dogs/all-dogs/n02105855_2933.jpg':
continue
im = Image.open(dog)
 xdiff = abs(bbox[2] - bbox[0])
 ydiff = abs(bbox[3] - bbox[1])
 if(( abs(ydiff - xdiff)/min(ydiff, xdiff)<=0.5)and(( xdiff>=64)and(ydiff>=64))):
select_dogs.append(dog)
 select_bbox.append(bbox)
 select_labels.append(class_dict[dog_class])
 return select_dogs, select_bbox, select_labels
select_dogs, select_bbox, select_labels = get_cleaned_data(image, annot, class_dict )<categorify>
def embed2dict(tokenizer, embedding):
word_index = tokenizer.word_index
embedding_dict = {line.split() [0]: line.split() [1:] for line in embedding[1:]
if word_index.get(line.split() [0])}
return embedding_dict
fasttext_dict = embed2dict(tokenizer, fasttext)
 glove_dict = embed2dict(tokenizer, glove )
def prepare_labels(y):
values = np.array(y)
 onehot_encoder = OneHotEncoder(sparse=False, categories='auto')
 onehot_encoded = onehot_encoder.fit_transform(values.reshape(values.shape[0], 1))
y = onehot_encoded
return y
labels_encoded = prepare_labels(select_labels )<import_modules>
fasttext_word = set(fasttext_dict.keys())
 glove_word = set(glove_dict.keys())
 intersection_word = fasttext_word.intersection(glove_word)
 print("Similar embedding words:", len(intersection_word))
from PIL import Image
from skimage import util<categorify>
def update_embed_dict(embed_dict):
result = {word: vector for word, vector in embed_dict.items()
if word in intersection_word}
return result
fasttext_dict = update_embed_dict(fasttext_dict)
 glove_dict = update_embed_dict(glove_dict )
class dogs_Dataset(Dataset):
def __init__(self, datafolder, datatype='train', index=[], \
transform = transforms.Compose([transforms.RandomResizedCrop(128),transforms.ToTensor() ])) :
self.datafolder = datafolder
self.datatype = datatype
self.image_files_list = [select_dogs[i] for i in index]
self.bbox_list = [select_bbox[i] for i in index]
self.label_list = [labels_encoded[i] for i in index]
self.transform = transform
self.imgs = []
for idx in range(len(self.image_files_list)) :
img_name = self.image_files_list[idx]
bbox = self.bbox_list[idx]
img = cv2.imread(img_name, 1)[bbox[1]:bbox[3], bbox[0]:bbox[2]]
mean, var = 0, 0.01
noise = np.random.normal(mean, var ** 0.5, img.shape)
 out = img + noise
if out.min() < 0:
low_clip = -1.
 else:
low_clip = 0.
 out = np.clip(out, low_clip, 1.0)
 out = np.uint8(out*255)
 image = Image.fromarray(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))
self.imgs.append(image)
 def __len__(self):
return len(self.imgs)
 def __getitem__(self, idx):
image = self.imgs[idx]
image = self.transform(image)
 if self.datatype == 'train':
label = self.label_list[idx]
else:
label = torch.zeros(1)
 return image, label<define_variables>
fasttext_dim = 300
glove_dim = 200
num_words = len(tokenizer.word_index)+ 1
def dict2vector(tokenizer, embed_dict, embed_dim):
vector = np.zeros(shape=(num_words, embed_dim))
word_index = tokenizer.word_index
for word, index in word_index.items() :
if index > num_words:
continue
if embed_dict.get(word):
vector[index] = embed_dict.get(word)
 return vector
fasttext_vector = dict2vector(tokenizer, fasttext_dict, fasttext_dim)
 glove_vector = dict2vector(tokenizer, glove_dict, glove_dim )
imgs, label = next(iter(train_loader))
imgs = imgs.numpy().transpose(0, 2, 3, 1 )<init_hyperparams>
def concat_vector(vectors):
result = np.hstack(vectors)
 return result
embedding_vector = concat_vector([fasttext_vector, glove_vector] )
LR_G = 0.0005
LR_D = 0.0005
beta1 = 0.5
real_label = 0.8
fake_label = 0.0
nz = 128
device = torch.device("cuda" if torch.cuda.is_available() else "cpu" )<choose_model_class>
BATCH_SIZE = 32
EPOCHS = 8
embedding_dim = embedding_vector.shape[1]
netG = Generator(nz ).to(device)
 netD = Discriminator().to(device)
 criterion = nn.BCELoss()
optimizerD = optim.Adam(netD.parameters() , lr=LR_D, betas=(beta1, 0.999))
optimizerG = optim.Adam(netG.parameters() , lr=LR_G, betas=(beta1, 0.999))
fixed_noise = torch.randn(25, nz, 1, 1, device=device )<define_variables>
def build_model(emb_init=None, emb_train=True):
embed_params = {"input_dim": num_words,
"output_dim": embedding_dim,
"embeddings_initializer": keras.initializers.Constant(
 embedding_vector if emb_init is None else emb_init),
"trainable": emb_train,
"mask_zero": True,
}
model = keras.Sequential([
keras.layers.Embedding(**embed_params),
keras.layers.Bidirectional(keras.layers.LSTM(128,
dropout=0.3,
recurrent_dropout=0.3,
return_sequences=True 
)) ,
keras.layers.Reshape(( 5, 10, 256)) ,
keras.layers.Conv2D(32, 2),
keras.layers.GlobalMaxPooling2D() ,
keras.layers.Dropout(0.3),
keras.layers.Dense(1, activation="sigmoid")])
 optimizer = keras.optimizers.Adam(1e-4, clipvalue=0.5)
 model.compile(optimizer=optimizer, loss="binary_crossentropy", metrics=["acc"])
 return model
epochs = 600<init_hyperparams>
test_model.fit(X_train, y_train,
epochs=10,
batch_size=BATCH_SIZE,
validation_split=0.2, )
for epoch in range(epochs):
for ii,(real_images, train_labels)in enumerate(train_loader):
netD.zero_grad()
real_images = real_images.to(device)
 batch_size = real_images.size(0)
 labels = torch.full(( batch_size, 1), real_label, device=device)
 output = netD(real_images)
 errD_real = criterion(output, labels)
 errD_real.backward()
D_x = output.mean().item()
noise = torch.randn(batch_size, nz, 1, 1, device=device)
 fake = netG(noise)
 labels.fill_(fake_label)
 output = netD(fake.detach())
 errD_fake = criterion(output, labels)
 errD_fake.backward()
D_G_z1 = output.mean().item()
errD = errD_real + errD_fake
optimizerD.step()
netG.zero_grad()
labels.fill_(real_label)
 output = netD(fake)
 errG = criterion(output, labels)
 errG.backward()
D_G_z2 = output.mean().item()
optimizerG.step()
if(ii+1)%(len(train_loader)//2)== 0:
print('[%d/%d][%d/%d] Loss_D: %.4f Loss_G: %.4f D(x): %.4f D(G(z)) : %.4f / %.4f'
%(epoch + 1, epochs, ii+1, len(train_loader),
errD.item() , errG.item() , D_x, D_G_z1, D_G_z2)) 
<compute_test_metric>
oov_embedding = test_model.get_weights() [0]
oov_embedding
def truncated_normal(size, threshold=1):
values = truncnorm.rvs(-threshold, threshold, size=size)
 return values<categorify>
def merge_embeddings(oov_embed, embed_vector):
for index, row in enumerate(embed_vector):
if row.sum() == 0:
embed_vector[index] = oov_embed[index]
return embed_vector
new_embedding = merge_embeddings(oov_embedding, embedding_vector )
gen_z = torch.randn(32, nz, 1, 1, device=device)
 gen_images = netG(gen_z ).to("cpu" ).clone().detach()
gen_images = gen_images.numpy().transpose(0, 2, 3, 1 )<categorify>
n_splits = 5
splits = list(KFold(n_splits=n_splits, shuffle=True, random_state=0 ).split(X_train,y_train))
predictions = np.zeros(shape=(X_test.shape[0], 1))
for n, fold in enumerate(splits):
print()
print("Fold {}".format(n + 1))
model_fold = "lstm_fold0{}.h5".format(n + 1)
 model_checkpoint = keras.callbacks.ModelCheckpoint(filepath=model_fold,
monitor="val_acc",
save_best_only=True,)
 callbacks_list = [model_checkpoint]
X_train_fold, y_train_fold = X_train[fold[0]], y_train[fold[0]]
X_val_fold, y_val_fold = X_train[fold[1]], y_train[fold[1]]
model = build_model(emb_init=new_embedding, emb_train=False)
 model.fit(X_train_fold, y_train_fold,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
validation_data=(X_val_fold, y_val_fold),
callbacks=callbacks_list,)
 model = keras.models.load_model(model_fold)
 predictions += model.predict(X_test)
 predictions /= n_splits
<define_variables><EOS> 
class_prediction = np.where(predictions > 0.5, 1, 0)
 submission = pd.DataFrame({"id": test_tweet["id"],
"target": class_prediction.flatten() })
 submission.to_csv("submission.csv", index=False )