Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<prepare_output>
tqdm.pandas()
for i in range(10):
print(np.argmax(lbl[i]))<normalization>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print('Train data dimension: ', train_df.shape)
 display(train_df.head())
 print('Test data dimension: ', test_df.shape)
 display(test_df.head() )
aug = rot_gen.flow(x_train, Y_train, batch_size = 40)
 x_train_aug = np.ndarray(shape =(x_train.shape[0]*41, 28, 28, 1))
Y_train_aug = np.ndarray(shape =(Y_train.shape[0]*41, 10))
x_train_aug[: x_train.shape[0], :, :, :] = x_train
Y_train_aug[: x_train.shape[0], :] = Y_train
for i in range(x_train.shape[0] - 1):
nn = aug.next()
x_train_aug[x_train.shape[0] + i*40: x_train.shape[0] +(i + 1)*40, :, :, :] = nn[0]/255
Y_train_aug[x_train.shape[0] + i*40: x_train.shape[0] +(i + 1)*40, :] = nn[1]<predict_on_test>
enable_local_test = False
if enable_local_test:
n_test = len(test_df)*4
train_df,local_test_df =(train_df.iloc[:-n_test].reset_index(drop=True),
train_df.iloc[-n_test:].reset_index(drop=True))
else:
local_test_df = pd.DataFrame([[None,None,0],[None,None,0]],columns=['qid','question_text','target'])
 n_test = 2
class makestats(Callback):
def on_epoch_end(self, batch, logs=None):
if batch % 5 == 0 and batch != 0:
YY_test = np.argmax(Y_test, axis=1)
 y_pred = model.predict(x_test)
 print(classification_report(YY_test, np.argmax(y_pred, axis = 1)))
 print(confusion_matrix(( YY_test), y_pred.argmax(axis=1)) )<define_search_model>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(4396 )
inputA = Input(shape =(28, 28, 1))
x = Conv2D(32, kernel_size=(5, 5), strides=(1),
activation='relu',
padding = 'same' )(inputA)
 x = MaxPooling2D(pool_size=(2, 2), strides=(2))(x)
 x = Dropout(0.3 )(x)
 x = Conv2D(64,(5, 5), activation='relu' )(x)
 x = MaxPooling2D(pool_size=(2, 2))(x)
 x = Dropout(0.3 )(x)
 x = Flatten()(x)
 x = Dense(1000, activation='relu' )(x)
 x = Dropout(0.3 )(x)
 z = Dense(500, activation="relu" )(x)
 x = Dropout(0.3 )(x)
 z = Dense(10, activation="softmax" )(z)
 model = Model(inputs=inputA, outputs=z)
 epochs = 40
lrate = 0.0003
decay = lrate/epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)
 model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'] )<train_model>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in tqdm([i * 0.01 for i in range(100)]):
score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
model.fit(x_train_aug, Y_train_aug, validation_data=(x_test, Y_test), verbose = 1,
epochs=epochs, batch_size=32, callbacks = [history, makestats() ] )<predict_on_test>
def sigmoid(x):
return 1 /(1 + np.exp(-x))
TEST = np.ndarray(shape =(test.values.shape[0], 28, 28, 1))
i = 0
for img in test.values:
TEST[i, :, :, 0] = np.reshape(img,(28, 28)) /255
i += 1<predict_on_test>
embed_size = 300
max_features = 120000
maxlen = 70
y_pred = model.predict(TEST )<define_variables>
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def clean_contractions(text, mapping):
specials = ["’", "‘", "´", "`"]
for s in specials:
text = text.replace(s, "'")
 text = ' '.join([mapping[t] if t in mapping else t for t in text.split(" ")])
 return text
preds = []
for yy in y_pred:
preds.append(np.argmax(yy))<save_to_csv>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
out_file = open("predictions.csv", "w")
 out_file.write("ImageId,Label
")
 for i in range(len(preds)) :
out_file.write(str(i+1)+ "," + str(int(preds[i])) + "
")
 out_file.close()<set_options>
def reverse_text(text):
t = text.split()
list.reverse(t)
 return " ".join(t )
%matplotlib inline 
<load_from_csv>
with timer("processing"):
train_df["clean_question"] = train_df["question_text"].str.lower()
test_df["clean_question"] = test_df["question_text"].str.lower()
train_df["clean_question"] = train_df["clean_question"].apply(lambda x: clean_text(x))
test_df["clean_question"] = test_df["clean_question"].apply(lambda x:clean_text(x))
train_df['reverse_question'] = train_df['clean_question'].apply(lambda x:reverse_text(x))
test_df['reverse_question'] = test_df['clean_question'].apply(lambda x:reverse_text(x))
x_train = train_df["reverse_question"].fillna("_
x_test = test_df["reverse_question"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(x_train)+list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = pad_sequences(x_train, maxlen=maxlen)
 x_test = pad_sequences(x_test, maxlen=maxlen)
 y_train = train_df['target'].values
mnist_train = pd.read_csv(".. /input/digit-recognizer/train.csv")
 mnist_test = pd.read_csv(".. /input/digit-recognizer/test.csv" )<feature_engineering>
def fast_load_all(embedding_path,emb_mean,emb_std,ready_pos = None,\
encoding='latin1',unicode_errors='strict',datatype=np.float32,word_index=None):
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, 300))
in_words = 0
stop = 0
new_pos = []
count = 0
with utils.smart_open(embedding_path)as fin:
end = False
while end==False:
count+=1
word = []
while True:
ch = fin.read(1)
 if ch == b' ':
break
elif ch == b'':
end=True
break
else:
word.append(ch)
 word_length = len(word)
 try:
word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)
 except:
word = utils.to_unicode(b''.join(word), encoding='latin1', errors=unicode_errors)
 if end==True:
print(count)
 break
if word not in word_index:
fin.readline()
stop += 1
else:
in_words+=1
i = word_index[word]
if i>=max_features:
continue
if ready_pos is not None:
if ready_pos[i]==1:
continue
else:
ready_pos[i]=1
new_pos.append(i)
 embedding_matrix[i] = np.asarray(fin.readline().split() [:300],dtype='float32')
 stop = 0
print("total ooi words now:{0},total finding words:{2},stopping at:{1}".format(in_words,stop,np.sum(ready_pos)))
 return in_words,embedding_matrix,new_pos
mnist_train_data = mnist_train.loc[:, "pixel0":]
mnist_train_label = mnist_train.loc[:, "label"]
mnist_train_data = mnist_train_data/255.0
mnist_test = mnist_test/255.0<categorify>
def load_word2vec(fname,emb_mean,emb_std,ready_pos = None,\
encoding='utf-8',unicode_errors='strict',datatype=np.float32,word_index=None):
embedding_matrix = np.random.normal(emb_mean, emb_std,(max_features, 300))
in_words = 0
stop = 0
new_pos = []
count = 0
with utils.smart_open(fname)as fin:
header = utils.to_unicode(fin.readline() , encoding=encoding)
 vocab_size,vector_size =(int(x)for x in header.split())
 binary_len = np.dtype(datatype ).itemsize*vector_size
for _ in range(vocab_size-180000):
word = []
while True:
ch = fin.read(1)
 if ch==b' ':
break
if ch==b'':
raise EOFError("unexpected end of input")
 if ch!=b'
':
word.append(ch)
 try:
word = utils.to_unicode(b''.join(word), encoding=encoding, errors=unicode_errors)
 except:
word = utils.to_unicode(b''.join(word), encoding='latin1', errors=unicode_errors)
 weights = np.fromstring(fin.read(binary_len),dtype=datatype ).astype(datatype)
 if word not in word_index:
stop += 1
continue
else:
in_words+=1
i = word_index[word]
if i>=max_features:
continue
if ready_pos is not None:
if ready_pos[i]==1:
continue
else:
ready_pos[i]=1
new_pos.append(i)
 embedding_matrix[i] = weights
stop = 0
print("total ooi words now:{0},total finding words:{2},stopping at:{1}".format(in_words,stop,np.sum(ready_pos)))
 return in_words,embedding_matrix,new_pos
nclasses = mnist_train_label.max() - mnist_train_label.min() + 1
mnist_train_label = np_utils.to_categorical(mnist_train_label, num_classes = nclasses)
 print("Shape of y_train after encoding: ", mnist_train_label.shape )<prepare_x_and_y>
seed_everything(4396)
 glove_pos = np.zeros(max_features)
 para_pos = np.zeros(max_features)
 fast_pos = np.zeros(max_features)
 with timer("build embeddings"):
in_glove,embedding_glove,glove_pos = fast_load_all('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt',-0.005838499,0.48782197,encoding='utf-8',word_index=tokenizer.word_index,ready_pos=glove_pos)
 in_para,embedding_para,para_pos = fast_load_all('.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',-0.0053247833,0.49346462,encoding='utf-8',word_index=tokenizer.word_index,ready_pos=para_pos)
 in_fast,embedding_fasttext,fast_pos = fast_load_all('.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',-0.0051106834, 0.18445626,encoding='utf-8',word_index=tokenizer.word_index,ready_pos=fast_pos)
 print(len(glove_pos),len(para_pos),len(fast_pos))
x_train=mnist_train_data
x_test=mnist_test_arr
y_train=mnist_train_label
x_train.shape<prepare_x_and_y>
diff_glove_para = list(set(para_pos)-set(glove_pos))
diff_glove_fast = list(set(fast_pos)-set(glove_pos))
diff_glove_p_f = list(set(diff_glove_fast)-set(diff_glove_para))
x_val=x_train[33600:]
x=x_train[:33600]
y_val= y_train[33600:]
y=y_train[:33600]
y.shape<import_modules>
splits = list(StratifiedKFold(n_splits=9, shuffle=True, random_state=10 ).split(x_train, y_train))
from tensorflow.keras.preprocessing.image import ImageDataGenerator<init_hyperparams>
class CyclicLR(object):
def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,
step_size=2000, mode='triangular', gamma=1.,
scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):
if not isinstance(optimizer, Optimizer):
raise TypeError('{} is not an Optimizer'.format(
 type(optimizer ).__name__))
self.optimizer = optimizer
self.lr_history = []
if isinstance(base_lr, list)or isinstance(base_lr, tuple):
if len(base_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} base_lr, got {}".format(
 len(optimizer.param_groups), len(base_lr)))
 self.base_lrs = list(base_lr)
 else:
self.base_lrs = [base_lr] * len(optimizer.param_groups)
 if isinstance(max_lr, list)or isinstance(max_lr, tuple):
if len(max_lr)!= len(optimizer.param_groups):
raise ValueError("expected {} max_lr, got {}".format(
 len(optimizer.param_groups), len(max_lr)))
 self.max_lrs = list(max_lr)
 else:
self.max_lrs = [max_lr] * len(optimizer.param_groups)
 self.step_size = step_size
if mode not in ['triangular', 'triangular2', 'exp_range'] \
and scale_fn is None:
raise ValueError('mode is invalid and scale_fn is None')
 self.mode = mode
self.gamma = gamma
if scale_fn is None:
if self.mode == 'triangular':
self.scale_fn = self._triangular_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = self._triangular2_scale_fn
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = self._exp_range_scale_fn
self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.batch_step(last_batch_iteration + 1)
 self.last_batch_iteration = last_batch_iteration
def batch_step(self, batch_iteration=None):
if batch_iteration is None:
batch_iteration = self.last_batch_iteration + 1
self.last_batch_iteration = batch_iteration
for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
param_group['lr'] = lr
def _triangular_scale_fn(self, x):
return 1.
 def _triangular2_scale_fn(self, x):
return 1 /(2.**(x - 1))
def _exp_range_scale_fn(self, x):
return self.gamma**(x)
 def get_lr(self):
step_size = float(self.step_size)
 cycle = np.floor(1 + self.last_batch_iteration /(2 * step_size))
x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)
 lrs = []
param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)
 for param_group, base_lr, max_lr in param_lrs:
base_height =(max_lr - base_lr)* np.maximum(0,(1 - x))
if self.scale_mode == 'cycle':
lr = base_lr + base_height * self.scale_fn(cycle)
 else:
lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)
 lrs.append(lr)
 self.lr_history.append(lrs)
 return lrs
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(x )<import_modules>
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )
from keras.optimizers import SGD<choose_model_class>
use_pretrained_embedding = True
hidden_size = 60
gru_len = hidden_size
Routings = 4
Num_capsule = 5
Dim_capsule = 5
T_epsilon = 1e-7
class Caps_layer(nn.Module):
def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \
routings=Routings, kernel_size=(9, 1), share_weights=True,
activation='default', **kwargs):
super(Caps_layer, self ).__init__(**kwargs)
 self.num_capsule = num_capsule
self.dim_capsule = dim_capsule
self.routings = routings
self.kernel_size = kernel_size
self.share_weights = share_weights
if activation == 'default':
self.activation = self.squash
else:
self.activation = nn.ReLU(inplace=True)
 if self.share_weights:
self.W = nn.Parameter(
 nn.init.xavier_normal_(torch.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))
 else:
self.W = nn.Parameter(
 torch.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))
def forward(self, x):
if self.share_weights:
u_hat_vecs = torch.matmul(x, self.W)
 else:
print('add later')
 batch_size = x.size(0)
 input_num_capsule = x.size(1)
 u_hat_vecs = u_hat_vecs.view(( batch_size, input_num_capsule,
self.num_capsule, self.dim_capsule))
u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3)
 b = torch.zeros_like(u_hat_vecs[:, :, :, 0])
 for i in range(self.routings):
b = b.permute(0, 2, 1)
 c = F.softmax(b, dim=2)
 c = c.permute(0, 2, 1)
 b = b.permute(0, 2, 1)
 outputs = self.activation(torch.einsum('bij,bijk->bik',(c, u_hat_vecs)))
 if i < self.routings - 1:
b = torch.einsum('bik,bijk->bij',(outputs, u_hat_vecs))
return outputs
def squash(self, x, axis=-1):
s_squared_norm =(x ** 2 ).sum(axis, keepdim=True)
 scale = torch.sqrt(s_squared_norm + T_epsilon)
 return x / scale
model = tf.keras.models.Sequential([
tf.keras.layers.Conv2D(32,(5,5), activation='relu', input_shape=(x_train.shape[1:])) ,
tf.keras.layers.Conv2D(32,(5,5), activation='relu'),
tf.keras.layers.MaxPooling2D(pool_size=(2,2), strides=(2,2)) ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Conv2D(64,(5,5), activation='relu'),
tf.keras.layers.Conv2D(64,(5,5), activation='relu'),
tf.keras.layers.MaxPooling2D(pool_size=(2,2), strides=(2,2)) ,
tf.keras.layers.Dropout(0.2),
tf.keras.layers.Flatten() ,
tf.keras.layers.Dense(2048, activation='relu'),
tf.keras.layers.Dropout(0.5),
tf.keras.layers.Dense(10, activation='softmax')
 ])
 learning_r= 0.01
weight_decay=1e-6
mometum=0.5
sgd= SGD(lr=learning_r,decay=weight_decay,momentum=mometum, nesterov=True)
 model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])
 model.summary() 
<train_model>
class NeuralNet_2(nn.Module):
def __init__(self,embeddings=None):
super(NeuralNet_2,self ).__init__()
hidden_size = 60
fc_layer = 16
fc_layer1 = 16
self.embedding = nn.Embedding(max_features,embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embeddings,dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(embed_size,hidden_size,bidirectional=True,batch_first = True)
 self.gru = nn.GRU(hidden_size*2,hidden_size,bidirectional=True,batch_first=True)
 self.lstm_attention = Attention(hidden_size*2,maxlen)
 self.gru_attention = Attention(hidden_size*2,maxlen)
 self.bn = nn.BatchNorm1d(16,momentum=0.5)
 self.linear = nn.Linear(hidden_size*8+1,fc_layer1)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.fc = nn.Linear(fc_layer**2,fc_layer)
 self.out = nn.Linear(fc_layer,1)
 self.lincaps = nn.Linear(Num_capsule*Dim_capsule,1)
 self.caps_layer = Caps_layer()
def forward(self,x):
h_embedding = self.embedding(x)
 h_lstm,_ = self.lstm(h_embedding)
 h_gru,_ = self.gru(h_lstm)
 contents3 = self.caps_layer(h_gru)
 contents3 = self.dropout(contents3)
 batch_size =contents3.size(0)
 contents3 = contents3.view(batch_size,-1)
 contents3 = self.relu(self.lincaps(contents3))
h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru,1)
 max_pool,_ = torch.max(h_gru,1)
 conc = torch.cat(( h_lstm_atten,h_gru_atten,contents3,avg_pool,max_pool),1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
hist= model.fit(datagen.flow(x,y),epochs=50, validation_data=(x_val,y_val), verbose=1 )<predict_on_test>
class NeuralNet(nn.Module):
def __init__(self,embeddings=None):
super(NeuralNet, self ).__init__()
hidden_size = 60
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embeddings, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)
 self.lstm_attention = Attention(hidden_size * 2, maxlen)
 self.gru_attention = Attention(hidden_size * 2, maxlen)
 self.linear = nn.Linear(hidden_size*8, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, _ = self.gru(h_lstm)
 h_lstm_atten = self.lstm_attention(h_lstm)
 h_gru_atten = self.gru_attention(h_gru)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( h_lstm_atten, h_gru_atten, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out
p = model.predict(x_test )<define_variables>
def create_pseudo_data(model,unlabeled_data_loader,batch_size,sample_rate = 0.2):
num_of_samples = int(len(unlabeled_data_loader.dataset)*sample_rate)
 pseudo_preds = np.zeros(len(unlabeled_data_loader.dataset))
print("creating pseudo labels!!")
 model.eval()
for i,(x_batch,)in enumerate(unlabeled_data_loader):
y_pred = model(x_batch ).detach()
pseudo_preds[i*batch_size:(i+1)*batch_size] = sigmoid(y_pred.cpu().numpy())[:,0]
model.train()
pseudo_index = np.random.choice(range(len(pseudo_preds)) ,int(len(pseudo_preds)*sample_rate),replace=False)
 return pseudo_index,pseudo_preds[pseudo_index]
def create_augmented_train(model,train_data,train_y,unlabeled_data,batch_size,sample_rate=0.2):
unlabeled_data_cuda = torch.tensor(unlabeled_data,dtype=torch.long ).cuda()
unlabeled_data_ = torch.utils.data.TensorDataset(unlabeled_data_cuda)
 unlabeled_data_loader = torch.utils.data.DataLoader(unlabeled_data_,batch_size=batch_size,shuffle=False)
 pseudo_index,pseudo_labels = create_pseudo_data(model,unlabeled_data_loader,batch_size,sample_rate=sample_rate)
 pseudo_data = unlabeled_data[pseudo_index].copy()
new_train_data = train_data.copy()
new_train_y = train_y.copy()
new_train_data = np.concatenate([new_train_data,pseudo_data],axis=0)
 new_train_y = np.concatenate([new_train_y,pseudo_labels[:,np.newaxis]],axis=0)
 new_x_train_fold = torch.tensor(new_train_data,dtype=torch.long ).cuda()
new_y_train_fold = torch.tensor(new_train_y,dtype=torch.float32 ).cuda()
new_train = torch.utils.data.TensorDataset(new_x_train_fold,new_y_train_fold)
 new_train_loader = torch.utils.data.DataLoader(new_train,batch_size=batch_size,shuffle=True)
 return new_train_loader
p_t = []
for i in p:
p_t.append(np.argmax(i)) 
<save_to_csv>
batch_size = 512
n_epochs =4
submission = pd.DataFrame({
"ImageId": mnist_test.index+1,
"Label": p_t
})
 submission.to_csv('my_submission.csv', index=False )<set_options>
def train_model(model,x_train,y_train,x_val=None,y_val=None,n_epochs=4,validate=True,using_pseudo=False):
x_train_fold = torch.tensor(x_train, dtype=torch.long ).cuda()
y_train_fold = torch.tensor(y_train, dtype=torch.float32 ).cuda()
train = torch.utils.data.TensorDataset(x_train_fold,y_train_fold)
 train_loader = torch.utils.data.DataLoader(train,batch_size=batch_size,shuffle=True)
 if validate:
x_val_fold = torch.tensor(x_val, dtype=torch.long ).cuda()
y_val_fold = torch.tensor(y_val, dtype=torch.float32 ).cuda()
valid = torch.utils.data.TensorDataset(x_val_fold,y_val_fold)
 val_loader = torch.utils.data.DataLoader(valid,batch_size=batch_size,shuffle=False)
 base_lr, max_lr = 0.001, 0.003
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=max_lr)
 scheduler = CyclicLR(optimizer,base_lr =base_lr ,max_lr=max_lr,
step_size =len(train_loader),mode='triangular',gamma=0.994)
 loss_fn = torch.nn.BCEWithLogitsLoss(reduction='mean')
 best_score = -np.inf
test_preds_local = np.zeros(len(test_local_loader.dataset))
for epoch in range(n_epochs):
start_time = time.time()
model.train()
avg_loss = 0.
 aug_num = 0
if epoch==n_epochs-1:
if using_pseudo==True:
train_loader = create_augmented_train(model,x_train,y_train,x_test_local,batch_size=batch_size,sample_rate=0.2)
 for x_batch, y_batch in tqdm(train_loader, disable=True):
y_pred = model(x_batch)
 if scheduler:
scheduler.batch_step()
loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
if validate:
valid_preds = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for i,(x_batch,y_batch)in enumerate(val_loader):
y_pred = model(x_batch ).detach()
avg_val_loss+= loss_fn(y_pred,y_batch ).item() /len(val_loader)
 valid_preds[i*batch_size:(i+1)*batch_size] = sigmoid(y_pred.cpu().numpy())[:,0]
search_result = threshold_search(y_val, valid_preds)
 val_f1, val_threshold = search_result['f1'], search_result['threshold']
elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t val_loss={:.4f} \t val_f1={:.4f} best_t={:.2f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, avg_val_loss, val_f1, val_threshold, elapsed_time))
else:
elapsed_time = time.time() -start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if validate:
valid_preds = np.zeros(( x_val_fold.size(0)))
 avg_val_loss = 0.
 for i,(x_batch, y_batch)in enumerate(val_loader):
y_pred = model(x_batch ).detach()
avg_val_loss += loss_fn(y_pred, y_batch ).item() / len(val_loader)
 valid_preds[i * batch_size:(i+1)* batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]
print('Validation loss: ', avg_val_loss)
 test_preds = np.zeros(len(test_df))
test_preds_local = np.zeros(len(test_local_loader.dataset))
for i,(x_batch,)in enumerate(test_local_loader):
y_pred = model(x_batch ).detach()
test_preds_local[i*batch_size:(i+1)*batch_size]=sigmoid(y_pred.cpu().numpy())[:,0]
if validate:
return valid_preds,test_preds,test_preds_local
else:
return test_preds,test_preds_local
deviceCount = torch.cuda.device_count()
print(deviceCount)
 cuda0 = None
if deviceCount > 0:
print(torch.cuda.get_device_name(0))
cuda0 = torch.device('cuda:0' )<load_from_csv>
x_test_local = x_test
x_train = x_train
y_train = y_train
single_or_multi = 2
df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 print(df.shape)
 dataset_size = df.shape[0]
df.head()<prepare_x_and_y>
if single_or_multi==1:
seed=6017
train_preds = np.zeros(len(train_df))
test_preds = np.zeros(( len(test_df), len(splits)))
 test_preds_local = np.zeros(( n_test, len(splits)))
 x_test_local_cuda = torch.tensor(x_test_local,dtype=torch.long ).cuda()
test_local = torch.utils.data.TensorDataset(x_test_local_cuda)
 test_local_loader = torch.utils.data.DataLoader(test_local, batch_size=batch_size, shuffle=False)
 for i,(train_idx,valid_idx)in enumerate(splits):
x_train_fold = x_train[train_idx]
y_train_fold = y_train[train_idx, np.newaxis]
x_val_fold = x_train[valid_idx]
y_val_fold = y_train[valid_idx, np.newaxis]
print(f'Fold {i+1}')
 seed_everything(seed+i)
 model = NeuralNet()
model.cuda()
valid_preds_fold, test_preds_fold, test_preds_local_fold = train_model(model,
x_train_fold,
y_train_fold,
x_val_fold,
y_val_fold,
n_epochs=n_epochs,
validate=True,
using_pseudo=False)
 train_preds[valid_idx] = valid_preds_fold
test_preds[:,i] = test_preds_fold
test_preds_local[:,i] = test_preds_local_fold
x = df.drop('label', axis=1)
 x = x.values.reshape(dataset_size, 1, 28, 28)
 y = df['label'].values.reshape(dataset_size)
 df = None<split>
embedding_matrix_1 = embedding_glove.copy()
embedding_matrix_1[diff_glove_para] = embedding_para[diff_glove_para]
embedding_matrix_1[diff_glove_p_f] = embedding_fasttext[diff_glove_p_f]
embedding_matrix_2 = embedding_para.copy()
embedding_matrix_2[diff_glove_p_f] = embedding_fasttext[diff_glove_p_f]
embedding_matrix_3 =(embedding_glove+embedding_para)/2.0
embedding_matrix_4 =(0.45*embedding_glove+0.35*embedding_para+0.20*embedding_fasttext)
cross_validation_ratio = 0.05
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=cross_validation_ratio, random_state=93)
 x = None
y = None<categorify>
del embedding_glove,embedding_para,embedding_fasttext
gc.collect()
start_time = time.time()
def shift_image(image, dx, dy):
shifted_image = shift(image, [dy, dx], cval=0, mode="constant")
 return shifted_image
augmented_x = []
augmented_y = []
random.seed(103)
 percentage_processed = 0
for i in range(len(x_train)) :
augmented_x.append(x_train[i].astype(int))
augmented_y.append(y_train[i].astype(int))
for j in range(5):
image = x_train[i].reshape(( 28, 28))
x_shift = random.randint(-1, 1)
 y_shift = random.randint(-1, 1)
 rotation_deg = float(random.randint(-15, 15))
image = rotate(image, angle=rotation_deg, cval=0, mode="constant", preserve_range=True)
 image = np.rint(image)
 image = image.astype(int)
 image = shift_image(image, x_shift, y_shift)
 image = x_train[i].reshape(( 1, 28, 28))
augmented_x.append(image)
 augmented_y.append(y_train[i].astype(int))
if(i+1)/len(x_train)*100 >= percentage_processed+10:
print(f'{int(( i+1)/len(x_train)*100):3}% images processed')
 percentage_processed += 10
train_dataset_size = len(augmented_x)
 test_dataset_size = len(x_test)
 x_train = None
y_train = None
print(f'
Duration: {time.time() - start_time:.0f} seconds' )<categorify>
models={
"glove_main_model_1":NeuralNet(embeddings=embedding_matrix_1),
"avg_2_model_1":NeuralNet(embeddings=embedding_matrix_3),
"avg_3_model_1":NeuralNet(embeddings=embedding_matrix_4),
"glove_main_model_2":NeuralNet_2(embeddings=embedding_matrix_1),
"para_main_model_2":NeuralNet_2(embeddings=embedding_matrix_2),
"avg_2_model_2":NeuralNet_2(embeddings=embedding_matrix_3),
"avg_3_model_2":NeuralNet_2(embeddings=embedding_matrix_4)
 }
random.seed(93)
 for i in range(len(augmented_x)) :
index = random.randint(0, len(augmented_x)-1)
 tempx = augmented_x[i]
tempy = augmented_y[i]
augmented_x[i] = augmented_x[index]
augmented_y[i] = augmented_y[index]
augmented_x[index] = tempx
augmented_y[index] = tempy<create_dataframe>
if single_or_multi==2:
seed = 6017
epochs = [4]*len(models)
 y_train = y_train[:, np.newaxis]
test_preds = np.zeros(( len(test_df),len(models)))
 test_preds_local = np.zeros(( n_test,len(models)))
 x_test_local_cuda = torch.tensor(x_test_local,dtype=torch.long ).cuda()
test_local = torch.utils.data.TensorDataset(x_test_local_cuda)
 test_local_loader = torch.utils.data.DataLoader(test_local, batch_size=batch_size, shuffle=False)
 i=0
for key,model in models.items() :
print('model {0}'.format(key))
model.cuda()
test_preds_fold,test_preds_local_fold = train_model(
 model,x_train,y_train,n_epochs=epochs[i],validate=False)
 test_preds[:,i] = test_preds_local_fold
i+=1
trainingDataset = TensorDataset(x_train, y_train)
 testDataset = TensorDataset(x_test, y_test)
 trainloader = DataLoader(trainingDataset, batch_size=512, shuffle=True)
 testloader = DataLoader(testDataset, batch_size=512, shuffle=False )<choose_model_class>
magic_numbers = [0.22161071,0.13261581,0.13514855,0.18424001,0.09617107,0.13212665,0.0977806]
<choose_model_class><EOS> 
submission = test_df[['qid']].copy()
submission['prediction'] = np.sum(test_preds*np.array(magic_numbers),axis=1)>0.37
submission.to_csv('submission.csv',index=False )