Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<feature_engineering>
import pandas as pd
import numpy as np
import torch
from torch import nn
from sklearn.metrics import f1_score
import torchtext
from tqdm import tqdm, tqdm_notebook
from nltk import word_tokenize
import random
from torch import optim
df['Embarked']=np.where(( df['PassengerId']==62)|(df['PassengerId']==830)&(df['Embarked'].isnull()),'S',df['Embarked'])
 df[(df['PassengerId']==830)|(df['PassengerId']==62)].head()<filter>
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=70)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 test = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)} )
nonull_df=df[(df['Fare'].notnull())&(df['Age'].notnull())&(df['Embarked'].notnull())&(df['block'].notnull())]
nonull_df.count() ['PassengerId']<categorify>
text.build_vocab(train, test, min_freq=3)
 qid.build_vocab(test )
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train.drop(columns=['Cabin','Name','PassengerId','Survived','Ticket'])
 nonull_df_train.head() 
<prepare_x_and_y>
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 tqdm_notebook().pandas()
X_nonull_df_train = nonull_df_train.drop(['Age'],axis=1)
 y_nonull_df_train = nonull_df_train[['Age']]<categorify>
text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300 )
nullage=df[df['Age'].isnull() & df['block'].notnull() ]
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nullage_test = pd.get_dummies(nullage,columns=cat_feats,drop_first=False)
 nullage_test=nullage_test.drop(columns=['Age','Name','Ticket','PassengerId','Cabin','Survived'])
 nullage_test.head() 
<prepare_x_and_y>
class TextCNN(nn.Module):
def __init__(self, lm, padding_idx, static=True, kernel_num=128, fixed_length=50, kernel_size=[2, 5, 10], dropout=0.2):
super(TextCNN, self ).__init__()
self.dropout = nn.Dropout(p=dropout)
 self.embedding = nn.Embedding.from_pretrained(lm)
 if static:
self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.conv = nn.ModuleList([nn.Conv2d(1, kernel_num,(i, self.embedding.embedding_dim)) for i in kernel_size])
 self.maxpools = [nn.MaxPool2d(( fixed_length+1-i,1)) for i in kernel_size]
self.fc = nn.Linear(len(kernel_size)*kernel_num, 1)
 def forward(self, input):
x = self.embedding(input ).unsqueeze(1)
 x = [self.maxpools[i](torch.tanh(cov(x)) ).squeeze(3 ).squeeze(2)for i, cov in enumerate(self.conv)]
x = torch.cat(x, dim=1)
 y = self.fc(self.dropout(x))
return y
X_nullage_test = nullage_test<drop_column>
def search_best_f1(true, pred):
tmp = [0,0,0]
delta = 0
for tmp[0] in np.arange(0.1, 0.501, 0.01):
tmp[1] = f1_score(true, np.array(pred)>tmp[0])
 if tmp[1] > tmp[2]:
delta = tmp[0]
tmp[2] = tmp[1]
return tmp[2], delta
def training(epoch, model, loss_func, optimizer, train_iter):
e = 0
while e < epoch:
train_iter.init_epoch()
losses, preds, true = [], [], []
for train_batch in tqdm(list(iter(train_iter)) , 'epcoh {} training'.format(e)) :
model.train()
x = train_batch.text.cuda()
y = train_batch.target.type(torch.Tensor ).cuda()
true.append(train_batch.target.numpy())
 model.zero_grad()
pred = model.forward(x ).view(-1)
 loss = loss_function(pred, y)
 preds.append(torch.sigmoid(pred ).cpu().data.numpy())
 losses.append(loss.cpu().data.numpy())
 loss.backward()
optimizer.step()
train_f1, alpha_train = search_best_f1([j for i in true for j in i], [j for i in preds for j in i])
 print('epcoh {:02} - train_loss {:.4f} - train f1 {:.4f} - delta {:.4f}'.format(
 e, np.mean(losses), train_f1, alpha_train))
e += 1
return alpha_train
X_nonull_df_train=X_nonull_df_train.drop(columns=['Titles_ Capt','Titles_ Col','Titles_ Dona','Titles_ Dr','Titles_ Lady','Titles_ Major','Titles_ Master',
'Titles_ Mlle','Titles_ Mme','Titles_ Sir','Titles_ the Countess','block_G'])
<train_model>
random.seed(1234)
 batch_size = 512
train_iter = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
shuffle=True,
sort=False )
dtreeReg.fit(X_nonull_df_train,y_nonull_df_train )<predict_on_test>
def init_network(model, method='xavier', exclude='embedding', seed=123):
torch.manual_seed(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed_all(seed)
 for name, w in model.named_parameters() :
if not exclude in name:
if 'weight' in name:
if method is 'xavier':
nn.init.xavier_normal_(w)
 elif method is 'kaiming':
nn.init.kaiming_normal_(w)
 else:
nn.init.normal_(w)
 elif 'bias' in name:
nn.init.constant_(w, 0.0)
 else:
pass
def print_model(model, ignore='embedding'):
total = 0
for name, w in model.named_parameters() :
if not ignore or ignore not in name:
total += w.nelement()
print('{} : {} {} parameters'.format(name, w.shape, w.nelement()))
print('-------'*4)
 print('Total {} parameters'.format(total))
predictions_nullage = dtreeReg.predict(X_nullage_test)
 predictions_nullage 
<drop_column>
text.fix_length = 70
model = TextCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], kernel_size=[1, 2, 3, 5], kernel_num=128, static=False, fixed_length=text.fix_length, dropout=0.1 ).cuda()
init_network(model)
 optimizer = optim.Adam(params=model.parameters() , lr=1e-3)
 loss_function = nn.BCEWithLogitsLoss()
print_model(model, ignore=None )
nullage.drop(columns=['Age'])
 nullage['Age']=predictions_nullage
nullage.head()<feature_engineering>
alpha = training(3, model, loss_function, optimizer, train_iter )
df['Age']=np.where(((df['Age'].isnull())&(df['block'].notnull())) ,(df['Age'].fillna(nullage['Age'])) ,(df['Age']))<filter>
def predict(model, test_list):
pred = []
with torch.no_grad() :
for test_batch in test_list:
model.eval()
x = test_batch.text.cuda()
pred += torch.sigmoid(model.forward(x ).view(-1)).cpu().data.numpy().tolist()
return pred
nonull_df=df[(df['Fare'].notnull())&(df['Age'].notnull())&(df['Embarked'].notnull())&(df['block'].notnull())]
nonull_df.count() ['PassengerId']<categorify>
test_list = list(torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
sort=False,
train=False))
cat_feats=['Embarked','Sex','Titles','block','Pclass']
nonull_df_train = pd.get_dummies(nonull_df,columns=cat_feats,drop_first=False)
 nonull_df_train=nonull_df_train.drop(columns=['Cabin','Name','PassengerId','Survived','Ticket'])
 nonull_df_train.head() 
<prepare_x_and_y>
preds = predict(model, test_list)
 sub = pd.DataFrame()
sub['qid'] = [qid.vocab.itos[j] for i in test_list for j in i.qid.view(-1 ).numpy() ]
sub['prediction'] =(preds > alpha ).astype(int)
 sub.head()
<categorify><EOS> 
sub.to_csv("submission.csv", index=False )