Kaggle competition: Natural Language Processing with Disaster Tweets
print("Final bootstrapping is performed based on {} features.".format(X_train.shape[-1]))<init_hyperparams>
warnings.filterwarnings("ignore" )
n_fold = 6
random_state = 1127
params = {'bagging_fraction': 0.51,
'boosting': 'gbdt',
'feature_fraction': 0.9,
'lambda_l1': 2,
'lambda_l2': 0.03,
'learning_rate': 0.1,
'max_bin': 48,
'max_depth': 8,
'metric': 'MAE',
'min_data_in_bin': 57,
'min_data_in_leaf': 11,
'min_gain_to_split': 0.53,
'num_leaves': 83,
'objective': custom_objective,
'subsample': 0.55}<predict_on_test>
np.random.seed(42 )
taf_est_train, taf_est_test, taf_est_oof, _ = get_prediction(X_train, y_train_new,
X_test=X_test,
n_fold=n_fold,random_state=random_state )<predict_on_test>
train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv' )
X_train['taf_estimate'] = taf_est_oof
X_test['taf_estimate'] = taf_est_test
ttf_est_train, ttf_est_test, ttf_est_oof, _ = get_prediction(X_train, y_train,
X_test=X_test,
n_fold=n_fold,random_state=random_state)
 ttf_est_test_bsp1 = ttf_est_test
print("TTF OOF MAE after adding TAF to training features: {:.7f}"\ 
.format(( np.abs(y_train['time_to_failure'] - ttf_est_oof)).mean()))<prepare_output>
all_data = pd.concat([train, test], ignore_index=True )
X_train['ttf_estimate'] = ttf_est_oof
X_test['ttf_estimate'] = ttf_est_test<find_best_model_class>
def null_value_counts(df, col, null=True):
if null:
return len(df[df[col].isnull() ][col].to_list())
 else:
return len(df[~df[col].isnull() ][col].to_list())
num_bootstrap = 20
coeff_bsp = [0.4, 0.6]
for b in tqdm_notebook(range(num_bootstrap)) :
print("
Bootstrapping iteration {0}.".format(b+1))
random_state = b
taf_est_train, taf_est_test, taf_est_oof, _ = \
get_prediction(X_train.drop(columns=['taf_estimate']),
y_train_new,
X_test=X_test.drop(columns=['taf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TAF training OOF MAE: {:.7f}"\ 
.format(np.abs(y_train_new['taf'] - taf_est_oof ).mean()))
X_train['taf_estimate'] = coeff_bsp[0]*X_train['taf_estimate'] + coeff_bsp[1]*taf_est_oof
X_test['taf_estimate'] = coeff_bsp[0]*X_test['taf_estimate'] + coeff_bsp[1]*taf_est_test
ttf_est_train, ttf_est_test, ttf_est_oof, model = \
get_prediction(X_train.drop(columns=['ttf_estimate']),
y_train,
X_test=X_test.drop(columns=['ttf_estimate']),
n_fold=n_fold,random_state=random_state)
 print("TTF training OOF MAE: {:.7f}"\ 
.format(( np.abs(y_train['time_to_failure'] - ttf_est_oof)).mean()))
X_train['ttf_estimate'] = coeff_bsp[0]*X_train['ttf_estimate'] + coeff_bsp[1]*ttf_est_oof
X_test['ttf_estimate'] = coeff_bsp[0]*X_test['ttf_estimate'] + coeff_bsp[1]*ttf_est_test<save_to_csv>
def punctuations_checker(texts, get_top_15=True):
punctuations = ['\' + p for p in list(string.punctuation)if p not in list(',.?"'@
dirty_words = defaultdict()
for text in texts:
for word in text.split() :
if all([(len(re.findall(r'|'.join(punctuations), word)) > 1),(len(word)> 1)]):
try:
dirty_words[word] += 1
except:
dirty_words[word] = 1
if get_top_15:
dirty_words = {k: v for k, v in sorted(dirty_words.items() , key=lambda item: item[1], reverse=True)[:15]}
else:
dirty_words = {k: v for k, v in sorted(dirty_words.items() , key=lambda item: item[1], reverse=True)}
return dirty_words
submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv',
index_col='seg_id')
 submission_nonbsp = submission.copy()
submission_bsp1 = submission.copy()
submission['time_to_failure'] = ttf_est_test
submission_bsp1['time_to_failure'] = ttf_est_test_bsp1
submission_nonbsp['time_to_failure'] = ttf_test_orig
submission.to_csv('submission.csv')
 submission_nonbsp.to_csv('submission_nonbsp.csv')
 submission_bsp1.to_csv('submission_bsp1.csv' )<set_options>
dirty_words_real_disaster = punctuations_checker(all_data.text, get_top_15=True)
warnings.filterwarnings("ignore")
 warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
print(all_data[all_data.text.str.contains(';')].text.to_list() [:30] )
train_X_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_X_features_865.csv")
 train_X_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_X_features_865.csv")
 y_0 = pd.read_csv(".. /input/lanl-master-s-features-creating-0/train_y.csv", index_col=False, header=None)
 y_1 = pd.read_csv(".. /input/lanl-master-s-features-creating-1/train_y.csv", index_col=False, header=None )<concatenate>
def preprocess(texts):
clean_text = []
for text in texts:
text = html.unescape(text)
 text = ftfy.fix_text(text)
 text = re.sub(r'(?:http(?:s)?://)?\bt\b\.\bco\b(?:/[\w/-]+)?', ' ', text)
 text = re.sub(r'&gt;', '>', text)
 text = re.sub(r'&lt;', '<', text)
 text = re.sub(r'&amp;', '&', text)
 text = re.sub('\\
', '
', text)
 text = re.sub(r'\x89Û', '', text)
 text = re.sub(r'‰|_|Ï|Ò|ª|÷|å|©|£|À|Ì|Û|Ê', ' ', text)
 text = text.replace('...', '')
 text = text.replace('%20', ' ')
 for p in list(string.punctuation):
text = text.replace(p, f' {p} ')
 text = re.sub('\s+', ' ', text ).strip().lower()
clean_text.append(text)
 return clean_text
train_X = pd.concat([train_X_0, train_X_1], axis=0)
 train_X = train_X.reset_index(drop=True)
 print(train_X.shape)
 train_X.head()<concatenate>
all_data['text'] = preprocess(all_data.text)
train_y = pd.concat([y_0, y_1], axis=0)
 train_y = train_y.reset_index(drop=True)
 train_y.columns = ['time_to_failure']
train_y['time_to_failure'].shape<load_from_csv>
def words_generator(texts, n_grams=2, get_top_50=True):
gram2idx = defaultdict()
for line in texts:
word_list = line.split(' ')
 filtered_words = [word.strip() for word in word_list if word not in stopwords.words('english')]
grams = list(ngrams(filtered_words, n_grams))
grams_merge = []
for g in grams:
g = ' '.join(map(str,g))
grams_merge.append(g)
 grams = grams_merge
for g in grams:
if len(g.strip())> 3 and any([char_g.isalpha() for char_g in set(g.strip())]):
try:
gram2idx[g] += 1
except:
gram2idx[g] = 1
if get_top_50:
gram2idx = {k: v for k, v in sorted(gram2idx.items() , key=lambda item: item[1], reverse=True)[:50]}
else:
gram2idx = {k: v for k, v in sorted(gram2idx.items() , key=lambda item: item[1], reverse=True)}
return gram2idx
test_X = pd.read_csv(".. /input/lanl-master-s-features-creating-0/test_X_features_10.csv")
 test_X.drop('seg_id',inplace=True,axis=1 )<normalization>
realdis_trigram_count = words_generator(all_data[all_data.target == 1].text[::], n_grams=3, get_top_50=True)
 notdis_trigram_count = words_generator(all_data[all_data.target == 0].text[::], n_grams=3, get_top_50=True)
 realdis_bigram_count = words_generator(all_data[all_data.target == 1].text[::], n_grams=2, get_top_50=True)
 notdis_bigram_count = words_generator(all_data[all_data.target == 0].text[::], n_grams=2, get_top_50=True)
 realdis_words_count = words_generator(all_data[all_data.target == 1].text[::], n_grams=1, get_top_50=True)
 notdis_words_count = words_generator(all_data[all_data.target == 0].text[::], n_grams=1, get_top_50=True )
scaler = StandardScaler()
train_columns = train_X.columns
train_X[train_columns] = scaler.fit_transform(train_X[train_columns])
 test_X[train_columns] = scaler.transform(test_X[train_columns] )<train_model>
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
tpot = TPOTRegressor(generations=5, population_size=15, verbosity=2,scoring='neg_mean_absolute_error',cv=5)
 tpot.fit(train_X, train_y)
 print(tpot.score(train_X, train_y)) 
<save_to_csv>
words_real_disaster = ' '.join(all_data[all_data.target == 1].text.to_list())
 words_not_disaster = ' '.join(all_data[all_data.target == 0].text.to_list() )
preds = tpot.predict(test_X)
 sample_submission = pd.read_csv('.. /input/LANL-Earthquake-Prediction/sample_submission.csv')
 sample_submission['time_to_failure'] = preds.flatten()
sample_submission.to_csv('submission.csv', index=False )<set_options>
from sklearn.feature_extraction.text import CountVectorizer,TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
seed(639)
 set_random_seed(5944)
<load_from_csv>
train['clean_text'] = preprocess(train.text)
 train = train.drop_duplicates(subset='clean_text', keep='first')
 print('length train data', len(train))
print('split 80:20')
 X = train.clean_text
y = train.target
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)
 train_0, train_1 = len(y_train[y_train==0]), len(y_train[y_train==1])
 test_0, test_1 = len(y_val[y_val==0]), len(y_val[y_val==1])
 print('Train: 0=%d, 1=%d, Test: 0=%d, 1=%d' %(train_0, train_1, test_0, test_1))
cat_0 = train_0 + test_0
cat_1 = train_1 + test_1
print('Train: 0=%f, 1=%f '%(train_0/cat_0, train_1/cat_1))
print('Test: 0=%f, 1=%f'%(test_0/cat_0, test_1/cat_1))
float_data = pd.read_csv(".. /input/train.csv", dtype={"acoustic_data": np.float32, "time_to_failure": np.float32} ).values
def extract_features(z):
return np.c_[z.mean(axis=1),
z.min(axis=1),
z.max(axis=1),
z.std(axis=1)]<prepare_x_and_y>
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.linear_model import SGDClassifier, LogisticRegression, RidgeClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.preprocessing import FunctionTransformer
def create_X(x, last_index=None, n_steps=150, step_length=1000):
if last_index == None:
last_index=len(x)
 assert last_index - n_steps * step_length >= 0
temp =(x[(last_index - n_steps * step_length):last_index].reshape(n_steps, -1)- 5)/ 3
return np.c_[extract_features(temp),
extract_features(temp[:, -step_length // 10:]),
extract_features(temp[:, -step_length // 100:])]
n_features = create_X(float_data[0:150000] ).shape[1]
print("Our RNN is based on %i features"% n_features )<define_variables>
np.random.seed(42 )
def generator(data, min_index=0, max_index=None, batch_size=16, n_steps=150, step_length=1000):
if max_index is None:
max_index = len(data)- 1
while True:
rows = np.random.randint(min_index + n_steps * step_length, max_index, size=batch_size)
 samples = np.zeros(( batch_size, n_steps, n_features))
targets = np.zeros(batch_size,)
 for j, row in enumerate(rows):
samples[j] = create_X(data[:, 0], last_index=row, n_steps=n_steps, step_length=step_length)
 targets[j] = data[row - 1, 1]
yield samples, targets<choose_model_class>
model_sgd = SGDClassifier()
model_lsvm = LinearSVC()
model_rf = RandomForestClassifier()
model_lr = LogisticRegression()
model_r = RidgeClassifier()
eclf = VotingClassifier(estimators=[('lr', model_lr),('sgd', model_sgd),('lsvm', model_lsvm)], voting='hard')
 for model, label in zip([model_sgd, model_lsvm, model_rf, model_r, model_lr, eclf],
['SGD', 'LinearSVC', 'RandomForest', 'RidgeClassifier', 'LogisticRegression', 'VotingEnsemble']):
clf = Pipeline([ 
('vect', CountVectorizer(ngram_range=(1,1))), 
('tfidf', TfidfTransformer()), 
('to_dense', FunctionTransformer(lambda x: x.toarray() , accept_sparse=True)) , 
('model', model)
 ])
 clf.fit(X_train,y_train)
 print("Acc: %f using" % clf.score(X_val, y_val), f'[{label}]' )
batch_size = 32
second_earthquake = 50085877
float_data[second_earthquake, 1]
train_gen = generator(float_data, batch_size=batch_size)
 valid_gen = generator(float_data, batch_size=batch_size, max_index=second_earthquake)
 cb = [ModelCheckpoint("model.hdf5", save_best_only=True, period=3)]
model = Sequential()
model.add(CuDNNGRU(48, input_shape=(None, n_features)))
 model.add(Dense(10, activation='relu'))
model.add(Dense(1))
model.compile(optimizer=adam(lr=0.0005), loss="mae")
 model.summary()
plot_model(model )<train_model>
train_acc = clf.score(X_train, y_train)
 val_acc = clf.score(X_val, y_val)
 print('train_acc:', train_acc)
 print('val_acc:', val_acc )
history = model.fit_generator(train_gen,
steps_per_epoch=1000,
epochs=30,
verbose=0,
callbacks=cb,
validation_data=valid_gen,
validation_steps=200 )<import_modules>
!wget -O "/kaggle/working/glove.twitter.27B.zip" "http://nlp.stanford.edu/data/glove.twitter.27B.zip"
import gzip, pickle
import os
import numpy as np
import pandas as pd
import random
import shutil
import numpy as np
import os
import torch
from torch import nn, optim
import torch.nn.functional as F
from torchvision import datasets, transforms
from torchvision.utils import save_image
from torch.utils.data import Dataset, DataLoader
from scipy import linalg
import pathlib
import urllib
import warnings
from PIL import Image
from tqdm import tqdm_notebook as tqdm<define_variables>
!unzip -j "/kaggle/working/glove.twitter.27B.zip" "glove.twitter.27B.200d.txt"
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<categorify>
import torch
import torch.nn as nn
import torch.optim as optim
from torch.utils.data import DataLoader
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import random
class dog_dataset(Dataset):
def __init__(self, train_y, train_X, zeros, device):
self.train_y = torch.Tensor(train_y ).to(device)
 self.train_X = torch.Tensor(train_X ).to(device)
 self.zeros = torch.Tensor(zeros ).to(device)
 def __len__(self):
return len(train_y)
 def __getitem__(self, idx):
return self.train_y[idx], self.train_X[idx], self.zeros[idx]<prepare_x_and_y>
random.seed(360)
 np.random.seed(360)
 torch.manual_seed(360)
 torch.backends.cudnn.deterministic = True
torch.cuda.manual_seed_all(360 )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 bs = 256
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
data_set = dog_dataset(train_y, train_X, zeros, device)
 data_loader = DataLoader(data_set, bs )<normalization>
word2idx = {}
idx2word = {}
all_text = preprocess(train.text)+ preprocess(test.text)
 idx = 1
for text in all_text:
words = nltk.word_tokenize(text)
 for word in words:
if word not in word2idx.keys() :
word2idx[word] = idx
if idx not in idx2word.keys() :
idx2word[idx] = word
idx += 1
class Discriminator(nn.Module):
def __init__(self):
super(Discriminator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 self.conv1 = nn.Conv2d(1, 1,(2, 1), bias=False)
 def forward(self, imgs, imgnames):
x = self.fc1(imgnames)
 x = torch.sigmoid(x)
 x = torch.cat(( imgs, x), dim=1 ).view(-1, 1, 2, 12288)
 x = self.conv1(x)
 return x.view(-1, 12288 )<choose_model_class>
GLOVE_DIR = '/kaggle/working/'
embeddings_index = {}
f = open(os.path.join(GLOVE_DIR, 'glove.twitter.27B.200d.txt'))
for line in tqdm(f, desc='load glove embedding'):
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
lr = 0.005
epochs = 500
netD = Discriminator().to(device)
 optimizerD = optim.Adam(netD.parameters() , lr=lr)
 criteria = nn.BCELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.0],
[1.0]]]] ).to(device))
for param in netD.conv1.parameters() :
param.requires_grad = False<compute_test_metric>
embedding_matrix = np.zeros(( len(word2idx)+ 2, 200))
for word, idx in tqdm(word2idx.items() , desc='build embedding matrix'):
try:
embedding_matrix[idx] = embeddings_index[word]
except:
embedding_matrix[-1] = np.random.rand(1, 200 )
for k in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netD.zero_grad()
y_pred = netD(Zeros, X)
 loss = criteria(y_pred, y)
 loss.backward()
optimizerD.step()
if(k + 1)% 2 == 0:
print(f"Epoch: {k+1}/{epochs} | Loss: {loss}" )<normalization>
def generate_data_loader(x, y, maxlen=128, with_target=True, bs=16):
all_data = []
for text, label in zip(x, y):
tokens = word_tokenize(text)
 token_ids = [word2idx[t] for t in tokens]
if len(token_ids)> maxlen:
token_ids = token_ids[:maxlen]
else:
token_ids = token_ids +([0] *(maxlen - len(token_ids)))
 token_ids_tensor = torch.LongTensor(token_ids)
 target = torch.tensor(label)
 data = {
'token_ids': token_ids_tensor,
'target' : target
}
all_data.append(data)
 data_loader = DataLoader(all_data, batch_size=bs, num_workers=5)
 return data_loader
class Generator(nn.Module):
def __init__(self):
super(Generator, self ).__init__()
self.fc1 = nn.Linear(10000, 12288)
 def forward(self, imgnames):
x = self.fc1(imgnames)
 return x, imgnames.view(-1, 10000 )<data_type_conversions>
train_dataloader = generate_data_loader(X_train, y_train)
 val_dataloader = generate_data_loader(X_val, y_val)
def show_image() :
plt.figure(figsize=(15,3))
for j in range(5):
with torch.no_grad() :
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 inp = torch.Tensor(xx.reshape(( -1,10000)) ).to(device)
 img = netG(inp)[0].reshape(( -1,64,64,3)).to("cpu" ).clone().detach().numpy()
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()
def show_d_images(imgs, title):
with torch.no_grad() :
imgs = imgs.reshape(( -1,64,64,3))
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = imgs[j].detach().cpu().numpy()
img = Image.fromarray(( 255*img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.title(title)
 plt.imshow(img)
 plt.show()<choose_model_class>
def evaluate(data_loader, criterion, model):
n_val_total, n_val_correct, val_loss_total = 0, 0, 0
model.eval()
with torch.no_grad() :
for sample_batched in data_loader:
inputs = sample_batched['token_ids'].to(device)
 outputs = model(inputs)
 targets = sample_batched['target'].to(device ).float().unsqueeze(1)
 loss = criterion(outputs, targets)
 n_val_correct += get_n_correct(targets, outputs)
 n_val_total += len(outputs)
 val_loss_total += loss.item() * len(outputs)
 val_acc = n_val_correct / n_val_total
val_loss = val_loss_total / n_val_total
return val_acc, val_loss
lr = 0.01
beta1 = 0.5
netG = Generator().to(device)
 optimizerG = optim.Adam(netG.parameters() , lr=lr)
 criterion = nn.MSELoss()
netD.conv1.weight = nn.Parameter(torch.Tensor([[[[ -1.],
[1.]]]] ).to(device))
for param in netD.parameters() :
param.requires_grad = False<compute_train_metric>
class LSTMModel(nn.Module):
def __init__(self, embedding_matrix, output_size=1, embedding_dim=200, hidden_dim=300, hidden_dim_2=100,
n_layers=1, drop_prob=0.5, bidirectional=False, pooling=False):
super().__init__()
self.output_size = output_size
self.n_layers = n_layers
self.hidden_dim = hidden_dim
self.embedding = nn.Embedding.from_pretrained(torch.tensor(embedding_matrix, dtype=torch.float))
self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, batch_first=True, bidirectional=bidirectional)
 self.dropout = nn.Dropout(drop_prob)
 self.pooling = pooling
self.bidirectional = bidirectional
self.fc_b_pool = nn.Linear(hidden_dim*4, hidden_dim*2)
 if self.bidirectional or self.pooling:
self.fc = nn.Linear(hidden_dim*2, hidden_dim)
 self.fc_out = nn.Linear(hidden_dim, output_size)
 else:
self.fc = nn.Linear(hidden_dim, hidden_dim_2)
 self.fc_out = nn.Linear(hidden_dim_2, output_size)
 def forward(self, x):
embeds = self.embedding(x)
 len_seq = torch.as_tensor(( x != 0 ).sum(dim=1), dtype=torch.int64)
 packed_seq = pack_padded_sequence(embeds, len_seq, batch_first=True, enforce_sorted=False)
 out_packed_lstm,(h_lstm, c_lstm)= self.lstm(packed_seq)
 if self.pooling:
out_lstm, _ = pad_packed_sequence(out_packed_lstm, batch_first=True)
 mean_out = torch.mean(out_lstm, dim=1)
 max_out, _ = torch.max(out_lstm, dim=1)
 h_lstm = torch.cat([mean_out, max_out], dim=1)
 h_lstm = h_lstm.view(h_lstm.shape[0], -1)
 if self.bidirectional:
h_lstm = self.fc_b_pool(h_lstm)
 else:
h_lstm = h_lstm.transpose(0,1)
 h_lstm = h_lstm.contiguous().view(h_lstm.shape[0], -1)
 out = self.dropout(self.fc(h_lstm))
out = self.fc_out(out)
 return out
epochs = 50
for epoch in tqdm(range(epochs)) :
for i,(y, X, Zeros)in enumerate(data_loader):
netG.zero_grad()
fake, seed = netG(X)
 y_pred = netD(Zeros, seed)
 errG = criterion(fake, y_pred)
 errG.backward()
optimizerG.step()
if(epoch+1)% 5 == 0:
print(f"Epoch: {epoch+1}: G_Loss: {errG}")
 show_d_images(fake, "Generator output" )<set_options>
def train_model(model, train_dataloader, val_dataloader, learning_rate=1e-3, epochs=50, early_stopping=10, model_name='model_disaster_lstm.pt'):
PATH_OUTPUT_MODEL = '/kaggle/working'
optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay=0.01)
 scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.5)
 criterion = nn.BCEWithLogitsLoss()
min_val_loss = None
patience = 0
list_lr, list_acc, list_loss, list_val_acc, list_val_loss = [], [], [], [], []
model = model.to(device)
 for i in tqdm(range(epochs), desc='epochs'):
n_total, n_correct, loss_total = 0, 0, 0
model.train()
for sample_batched in tqdm(train_dataloader, desc='batch loader'):
optimizer.zero_grad()
inputs = sample_batched['token_ids'].to(device)
 outputs = model(inputs)
 targets = sample_batched['target'].to(device ).float().unsqueeze(1)
 loss = criterion(outputs, targets)
 loss.backward()
optimizer.step()
with torch.no_grad() :
n_correct += get_n_correct(targets, outputs)
 n_total += len(outputs)
 loss_total += loss.item() * len(outputs)
 train_acc = n_correct / n_total
train_loss = loss_total / n_total
learning_rate = optimizer.param_groups[0]['lr']
print(f'Epoch {i} :')
 print('LR:', learning_rate)
 print('Acc:', train_acc)
 print('Loss:', train_loss)
 val_acc, val_loss = evaluate(val_dataloader, criterion, model)
 if min_val_loss == None:
min_val_loss = val_loss
if min_val_loss > val_loss:
min_val_loss = val_loss
print('Found Best Val Loss...')
 torch.save(model.state_dict() , os.path.join(PATH_OUTPUT_MODEL, model_name))
patience = 0
else:
patience += 1
scheduler.step()
print('Val Acc:', val_acc)
 print('Val Loss:', val_loss)
 list_lr.append(learning_rate)
 list_loss.append(train_loss)
 list_val_loss.append(val_loss)
 list_acc.append(train_acc)
 list_val_acc.append(val_acc)
 if patience == early_stopping:
print(f'Validation Loss not decreasing {early_stopping} times in a row.. ')
 print('Early Stopping.. ')
 print('Best Val Loss:', min_val_loss)
 break
return min_val_loss,(list_lr, list_acc, list_loss, list_val_acc, list_val_loss )
class DogGenerator() :
index = 0
t = [
transforms.RandomCrop(( 48, 48), padding=None, pad_if_needed=True, fill=0, padding_mode='symmetric'),
transforms.Scale(( 64,64))
]
tfms = transforms.Compose([
transforms.RandomHorizontalFlip(p=0.5),
transforms.ColorJitter(brightness=(1,1.3), contrast=(1,1.3), saturation=0, hue=0)
 ])
 def getDog(self,seed):
xx = torch.Tensor(np.zeros(( 10000)) ).to(device)
 xx[self.index] = 0.999999
xx[np.random.randint(10000)] = 0.000001
img = netG(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3)).detach().cpu().numpy() * 255
self.index =(self.index+1)%10000
return self.tfms(Image.fromarray(img.astype('uint8')) )<define_variables>
class MetricsBuilder(object):
def __init__(self):
self.df_metrics = pd.DataFrame({})
 def add_metrics(self, tuple_metrics, name='LSTM'):
list_lr, list_acc, list_loss, list_val_acc, list_val_loss = tuple_metrics
metrics_data = pd.DataFrame({'acc':list_acc, 'val_acc':list_val_acc, 'loss':list_loss, 'val_loss':list_val_loss, 'lr':list_lr,
'metrics': [name for i in range(len(list_lr)) ], 'epochs':[i for i in range(len(list_lr)) ]})
 self.df_metrics = pd.concat([self.df_metrics, metrics_data])
 def __get_df_metrics__(self):
return self.df_metrics
metrics_builder = MetricsBuilder()
ComputeLB = False
DogsOnly = False
ROOT = '.. /input/generative-dog-images/'
if not ComputeLB: ROOT = '.. /input/'
IMAGES = os.listdir(ROOT + 'all-dogs/all-dogs/')
 breeds = os.listdir(ROOT + 'annotation/Annotation/')
 idxIn = 0; namesIn = []
imagesIn = np.zeros(( 25000,64,64,3))
if DogsOnly:
for breed in breeds:
for dog in os.listdir(ROOT+'annotation/Annotation/'+breed):
try: img = Image.open(ROOT+'all-dogs/all-dogs/'+dog+'.jpg')
 except: continue
tree = ET.parse(ROOT+'annotation/Annotation/'+breed+'/'+dog)
 root = tree.getroot()
objects = root.findall('object')
 for o in objects:
bndbox = o.find('bndbox')
 xmin = int(bndbox.find('xmin' ).text)
 ymin = int(bndbox.find('ymin' ).text)
 xmax = int(bndbox.find('xmax' ).text)
 ymax = int(bndbox.find('ymax' ).text)
 w = np.min(( xmax - xmin, ymax - ymin))
img2 = img.crop(( xmin, ymin, xmin+w, ymin+w))
img2 = img2.resize(( 64,64), Image.ANTIALIAS)
 imagesIn[idxIn,:,:,:] = np.asarray(img2)
 namesIn.append(breed)
 idxIn += 1
idx = np.arange(idxIn)
 np.random.shuffle(idx)
 imagesIn = imagesIn[idx,:,:,:]
namesIn = np.array(namesIn)[idx]
else:
IMAGES = np.sort(IMAGES)
 np.random.seed(810)
 x = np.random.choice(np.arange(20579),10000)
 np.random.seed(None)
 for k in range(len(x)) :
img = Image.open(ROOT + 'all-dogs/all-dogs/' + IMAGES[x[k]])
 w = img.size[0]; h = img.size[1];
if(k%2==0)|(k%3==0):
w2 = 100; h2 = int(h/(w/100))
a = 18; b = 0
else:
a=0; b=0
if w<h:
w2 = 64; h2 = int(( 64/w)*h)
 b =(h2-64)//2
else:
h2 = 64; w2 = int(( 64/h)*w)
 a =(w2-64)//2
img = img.resize(( w2,h2), Image.ANTIALIAS)
 img = img.crop(( 0+a, 0+b, 64+a, 64+b))
imagesIn[idxIn,:,:,:] = np.asarray(img)
 namesIn.append(IMAGES[x[k]])
 idxIn += 1
x = np.random.randint(0,idxIn,25)
 for k in range(5):
plt.figure(figsize=(15,3))
for j in range(5):
plt.subplot(1,5,j+1)
 img = Image.fromarray(imagesIn[x[k*5+j],:,:,:].astype('uint8'))
plt.axis('off')
 if not DogsOnly: plt.title(namesIn[x[k*5+j]],fontsize=11)
 else: plt.title(namesIn[x[k*5+j]].split('-')[1],fontsize=11)
 plt.imshow(img)
 plt.show()<import_modules>
lstm_model = LSTMModel(embedding_matrix, bidirectional=False)
 min_val_loss_lstm, metrics_lstm = train_model(lstm_model, train_dataloader, val_dataloader, model_name='lstm.pt' )
from keras.models import Model
from keras.layers import Input, Dense, Conv2D, Reshape, Flatten, concatenate
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
from keras.optimizers import SGD, Adam<train_model>
lstm_model_pool = LSTMModel(embedding_matrix, bidirectional=False, pooling=True)
 min_val_loss_lstm_pool, metrics_lstm_pool = train_model(lstm_model_pool, train_dataloader, val_dataloader, model_name='lstm_pool.pt' )
train_y =(imagesIn[:10000,:,:,:]/255.).reshape(( -1,12288))
train_X = np.zeros(( 10000,10000))
for i in range(10000): train_X[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 0.5
for k in range(5):
annealer = LearningRateScheduler(lambda x: lr)
 h = discriminator.fit([zeros,train_X], train_y, epochs = 10, batch_size=256, callbacks=[annealer], verbose=0)
 print('Epoch',(k+1)*10,'/30 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1]<0.533: lr = 0.1<drop_column>
bilstm_model = LSTMModel(embedding_matrix, bidirectional=True)
 min_val_loss_bilstm, metrics_bilstm = train_model(bilstm_model, train_dataloader, val_dataloader, model_name='bilstm.pt' )
del train_X, train_y, imagesIn<choose_model_class>
bilstm_model_pool = LSTMModel(embedding_matrix, bidirectional=True, pooling=True)
 min_val_loss_bilstm_pool, metrics_bilstm_pool = train_model(bilstm_model_pool, train_dataloader, val_dataloader, model_name='bilstm_pool.pt' )
discriminator.trainable=False
gan_input = Input(shape=(10000,))
x = generator(gan_input)
 gan_output = discriminator(x)
 gan = Model(gan_input, gan_output)
 gan.get_layer('model_1' ).get_layer('conv' ).set_weights([np.array([[[[-1 ]]],[[[255.]]]])])
 gan.compile(optimizer=Adam(5), loss='mean_squared_error')
 gan.summary()<train_model>
print('min_val_loss_bilstm',min_val_loss_bilstm)
 print('min_val_loss_bilstm_pool',min_val_loss_bilstm_pool)
 print('min_val_loss_lstm',min_val_loss_lstm)
 print('min_val_loss_lstm_pool',min_val_loss_lstm_pool )
train = np.zeros(( 10000,10000))
for i in range(10000): train[i,i] = 1
zeros = np.zeros(( 10000,12288))
lr = 5.
 for k in range(50):
annealer = LearningRateScheduler(lambda x: lr)
 h = gan.fit(train, zeros, epochs = 1, batch_size=256, callbacks=[annealer], verbose=0)
 if(k<10)|(k%5==4):
print('Epoch',(k+1)*10,'/500 - loss =',h.history['loss'][-1])
 if h.history['loss'][-1] < 25: lr = 1.
 if h.history['loss'][-1] < 1.5: lr = 0.5
if k<10:
plt.figure(figsize=(15,3))
for j in range(5):
xx = np.zeros(( 10000))
xx[np.random.randint(10000)] = 1
plt.subplot(1,5,j+1)
 img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( -1,64,64,3))
img = Image.fromarray(( img ).astype('uint8' ).reshape(( 64,64,3)))
 plt.axis('off')
 plt.imshow(img)
 plt.show()<categorify>
metrics_builder.add_metrics(metrics_lstm, name='LSTM')
 metrics_builder.add_metrics(metrics_lstm_pool, name='LSTM_Pool')
 metrics_builder.add_metrics(metrics_bilstm, name='BiLSTM')
 metrics_builder.add_metrics(metrics_bilstm_pool, name='BiSTM_Pool' )
class DogGenerator:
index = 0
def getDog(self,seed):
xx = np.zeros(( 10000))
xx[self.index] = 0.999
xx[np.random.randint(10000)] = 0.001
img = generator.predict(xx.reshape(( -1,10000)))[0].reshape(( 64,64,3))
self.index =(self.index+1)%10000
return Image.fromarray(img.astype('uint8'))<choose_model_class>
df_metrics = metrics_builder.__get_df_metrics__()
class KernelEvalException(Exception):
pass
model_params = {
'Inception': {
'name': 'Inception',
'imsize': 64,
'output_layer': 'Pretrained_Net/pool_3:0',
'input_layer': 'Pretrained_Net/ExpandDims:0',
'output_shape': 2048,
'cosine_distance_eps': 0.1
}
}
def create_model_graph(pth):

 with tf.gfile.FastGFile(pth, 'rb')as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
 _ = tf.import_graph_def(graph_def, name='Pretrained_Net')
 def _get_model_layer(sess, model_name):
layername = model_params[model_name]['output_layer']
layer = sess.graph.get_tensor_by_name(layername)
 ops = layer.graph.get_operations()
for op_idx, op in enumerate(ops):
for o in op.outputs:
shape = o.get_shape()
if shape._dims != []:
shape = [s.value for s in shape]
new_shape = []
for j, s in enumerate(shape):
if s == 1 and j == 0:
new_shape.append(None)
 else:
new_shape.append(s)
 o.__dict__['_shape_val'] = tf.TensorShape(new_shape)
 return layer
def get_activations(images, sess, model_name, batch_size=50, verbose=False):

 inception_layer = _get_model_layer(sess, model_name)
 n_images = images.shape[0]
if batch_size > n_images:
print("warning: batch size is bigger than the data size.setting batch size to data size")
 batch_size = n_images
n_batches = n_images//batch_size + 1
pred_arr = np.empty(( n_images,model_params[model_name]['output_shape']))
for i in tqdm(range(n_batches)) :
if verbose:
print("\rPropagating batch %d/%d" %(i+1, n_batches), end="", flush=True)
 start = i*batch_size
if start+batch_size < n_images:
end = start+batch_size
else:
end = n_images
batch = images[start:end]
pred = sess.run(inception_layer, {model_params[model_name]['input_layer']: batch})
 pred_arr[start:end] = pred.reshape(-1,model_params[model_name]['output_shape'])
 if verbose:
print(" done")
 return pred_arr
def normalize_rows(x: np.ndarray):

 return np.nan_to_num(x/np.linalg.norm(x, ord=2, axis=1, keepdims=True))
def cosine_distance(features1, features2):
features1_nozero = features1[np.sum(features1, axis=1)!= 0]
features2_nozero = features2[np.sum(features2, axis=1)!= 0]
norm_f1 = normalize_rows(features1_nozero)
 norm_f2 = normalize_rows(features2_nozero)
 d = 1.0-np.abs(np.matmul(norm_f1, norm_f2.T))
print('d.shape=',d.shape)
 print('np.min(d, axis=1 ).shape=',np.min(d, axis=1 ).shape)
 mean_min_d = np.mean(np.min(d, axis=1))
print('distance=',mean_min_d)
 return mean_min_d
def distance_thresholding(d, eps):
if d < eps:
return d
else:
return 1
def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):

 mu1 = np.atleast_1d(mu1)
 mu2 = np.atleast_1d(mu2)
 sigma1 = np.atleast_2d(sigma1)
 sigma2 = np.atleast_2d(sigma2)
 assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"
diff = mu1 - mu2
covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
 if not np.isfinite(covmean ).all() :
msg = "fid calculation produces singular product; adding %s to diagonal of cov estimates" % eps
warnings.warn(msg)
 offset = np.eye(sigma1.shape[0])* eps
covmean = linalg.sqrtm(( sigma1 + offset ).dot(sigma2 + offset))
if np.iscomplexobj(covmean):
if not np.allclose(np.diagonal(covmean ).imag, 0, atol=1e-3):
m = np.max(np.abs(covmean.imag))
raise ValueError("Imaginary component {}".format(m))
covmean = covmean.real
print('covmean.shape=',covmean.shape)
 tr_covmean = np.trace(covmean)
 return diff.dot(diff)+ np.trace(sigma1)+ np.trace(sigma2)- 2 * tr_covmean
def calculate_activation_statistics(images, sess, model_name, batch_size=50, verbose=False):

 act = get_activations(images, sess, model_name, batch_size, verbose)
 mu = np.mean(act, axis=0)
 sigma = np.cov(act, rowvar=False)
 return mu, sigma, act
def _handle_path_memorization(path, sess, model_name, is_checksize, is_check_png):
path = pathlib.Path(path)
 files = list(path.glob('*.jpg')) + list(path.glob('*.png'))
imsize = model_params[model_name]['imsize']
x = np.array([np.array(img_read_checks(fn, imsize, is_checksize, imsize, is_check_png)) for fn in files])
 m, s, features = calculate_activation_statistics(x, sess, model_name)
 del x
return m, s, features
def img_read_checks(filename, resize_to, is_checksize=False, check_imsize = 64, is_check_png = False):
im = Image.open(str(filename))
if is_checksize and im.size !=(check_imsize,check_imsize):
raise KernelEvalException('The images are not of size '+str(check_imsize))
if is_check_png and im.format != 'PNG':
raise KernelEvalException('Only PNG images should be submitted.')
 if resize_to is None:
return im
else:
return im.resize(( resize_to,resize_to),Image.ANTIALIAS)
 def calculate_kid_given_paths(paths, model_name, model_path, feature_path=None, mm=[], ss=[], ff=[]):

 tf.reset_default_graph()
create_model_graph(str(model_path))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
 m1, s1, features1 = _handle_path_memorization(paths[0], sess, model_name, is_checksize = True, is_check_png = True)
 if len(mm)!= 0:
m2 = mm
s2 = ss
features2 = ff
elif feature_path is None:
m2, s2, features2 = _handle_path_memorization(paths[1], sess, model_name, is_checksize = False, is_check_png = False)
 else:
with np.load(feature_path)as f:
m2, s2, features2 = f['m'], f['s'], f['features']
print('m1,m2 shape=',(m1.shape,m2.shape),'s1,s2=',(s1.shape,s2.shape))
print('starting calculating FID')
 fid_value = calculate_frechet_distance(m1, s1, m2, s2)
 print('done with FID, starting distance calculation')
 distance = cosine_distance(features1, features2)
 return fid_value, distance, m2, s2, features2<load_pretrained>
best_bilstm_pool = LSTMModel(embedding_matrix, bidirectional=True, pooling=True ).to(device)
 best_bilstm_pool.load_state_dict(torch.load('/kaggle/working/bilstm_pool.pt', map_location=device))
if ComputeLB:
with zipfile.ZipFile(".. /working/images.zip","r")as z:
z.extractall(".. /tmp/images2/")
 m2 = []; s2 =[]; f2 = []
user_images_unzipped_path = '.. /tmp/images2/'
images_path = [user_images_unzipped_path,'.. /input/generative-dog-images/all-dogs/all-dogs/']
public_path = '.. /input/dog-face-generation-competition-kid-metric-input/classify_image_graph_def.pb'
fid_epsilon = 10e-15
fid_value_public, distance_public, m2, s2, f2 = calculate_kid_given_paths(images_path, 'Inception', public_path, mm=m2, ss=s2, ff=f2)
 distance_public = distance_thresholding(distance_public, model_params['Inception']['cosine_distance_eps'])
 print("FID_public: ", fid_value_public, "distance_public: ", distance_public, "multiplied_public: ",
fid_value_public /(distance_public + fid_epsilon))
! rm -r.. /tmp<import_modules>
def predict_test_set(data_loader, model):
model.eval()
all_outputs = []
with torch.no_grad() :
for sample_batched in data_loader:
inputs = sample_batched.to(device)
 outputs = model(inputs)
 outputs =(torch.sigmoid(outputs ).squeeze(1)> 0.5 ).long().tolist()
all_outputs.extend(outputs)
 return all_outputs
import os,sys
import numpy as np
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.utils.class_weight import compute_sample_weight
from sklearn import metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, GridSearchCV
import xgboost as xgb
from xgboost.sklearn import XGBClassifier<load_from_csv>
def generate_test_data_loader(x, maxlen=128, bs=16):
all_data = []
for text in x:
tokens = word_tokenize(text)
 token_ids = [word2idx[t] for t in tokens]
if len(token_ids)> maxlen:
token_ids = token_ids[:maxlen]
else:
token_ids = token_ids +([0] *(maxlen - len(token_ids)))
 token_ids_tensor = torch.LongTensor(token_ids)
 all_data.append(token_ids_tensor)
 data_loader = DataLoader(all_data, batch_size=bs, num_workers=5)
 return data_loader
train_data = pd.read_csv('/kaggle/input/costa-rican-household-poverty-prediction/train.csv')
 train_target = train_data['Target']
train_data.drop(['Target'],axis=1,inplace=True)
<count_values>
test_dataloader = generate_test_data_loader(preprocess(test['text']))
predicted_target = predict_test_set(test_dataloader, best_bilstm_pool)
<categorify><EOS> 
my_submission = pd.DataFrame({'Id': test.id, 'target': predicted_target})
 my_submission.to_csv('submission.csv', index=False)