Kaggle competition: Quora Insincere Questions Classification
n_comp =20
svd_tf = TruncatedSVD(n_components=20,algorithm='arpack')
 svd_tf.fit(full_tfidf)
 train_svd = pd.DataFrame(svd_tf.transform(train_tfidf))
test_svd = pd.DataFrame(svd_tf.transform(test_tfidf))
train_svd_cols = ['svd_word_'+str(i)for i in range(n_comp)]
test_svd_cols = ['svd_word_'+str(i)for i in range(n_comp)]
train_df = pd.concat([train_df,train_svd],axis=1)
 test_df = pd.concat([test_df,test_svd],axis=1)
 train_df.head()<compute_train_metric>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(6017)
 print('Seeding done...' )
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_df):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runLR(dev_X, dev_y, val_X, val_y, test_X)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores )<feature_engineering>
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train_dataset = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 train, test = train_dataset.split(split_ratio=[0.8,0.2],stratified=True,strata_field='target',random_state=random.getstate())
 submission_x = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)})
 text.build_vocab(train_dataset, submission_x, min_freq=3)
 qid.build_vocab(submission_x)
 print('train dataset len:',len(train_dataset))
print('train len:',len(train))
print('test len:',len(test))
tfidf_vec = CountVectorizer(stop_words='english', ngram_range=(1,2))
tfidf_vec.fit(train_df['text'].values.tolist() + test_df['text'].values.tolist())
 train_tfidf = tfidf_vec.transform(train_df['text'].values.tolist())
 test_tfidf = tfidf_vec.transform(test_df['text'].values.tolist() )<compute_train_metric>
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300 )
cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 for dev_index, val_index in kf.split(train_X):
dev_X, val_X = train_tfidf[dev_index], train_tfidf[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runMNB(dev_X, dev_y, val_X, val_y, test_tfidf)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("Mean cv score : ", np.mean(cv_scores))
pred_full_test = pred_full_test / 5.
 print("cv scores : ", cv_scores)
 train_df["nb_cvec_eap"] = pred_train[:,0]
train_df["nb_cvec_hpl"] = pred_train[:,1]
train_df["nb_cvec_mws"] = pred_train[:,2]
test_df["nb_cvec_eap"] = pred_full_test[:,0]
test_df["nb_cvec_hpl"] = pred_full_test[:,1]
test_df["nb_cvec_mws"] = pred_full_test[:,2] 
<compute_train_metric>
batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
shuffle=True,
sort=False)
 test_loader = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
shuffle=False,
sort=False)
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_df):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runLR(dev_X, dev_y, val_X, val_y, test_X)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores)
<feature_engineering>
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 class SentimentLSTM(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(SentimentLSTM,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.n_layers = 2
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.lstm = nn.LSTM(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 def forward(self,x):
hidden =(torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 e = self.embedding(x)
 _, hidden = self.lstm(e, hidden)
 out = torch.cat(( hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1 ).cuda()
out = self.linear1(out)
 return self.linear2(self.dropout(F.relu(out)))
 class SentimentBase(nn.Module):
def __init__(self):
super(SentimentBase,self ).__init__()
self.embedding = nn.Embedding(75966,300)
 self.linear1 = nn.Linear(300*100,128)
 self.linear2 = nn.Linear(128,1)
 def forward(self,x):
emb = self.embedding(x)
 pooled = emb.reshape(( emb.shape[0],emb.shape[1]*emb.shape[2]))
out = self.linear1(pooled)
 out = self.linear2(F.relu(out))
return out
class SentimentCNN(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(SentimentCNN,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.cnns = nn.ModuleList([nn.Conv1d(in_channels=vocab_vectors.shape[1], out_channels=self.hidden_dim, kernel_size=k)for k in [3,4,5]])
 self.linear1 = nn.Linear(3*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 @staticmethod
def conv_and_max_pool(x, conv):

 return F.relu(conv(x ).permute(0, 2, 1 ).max(1)[0])
 def forward(self,x):
e = self.embedding(x)
 e = e.permute(0,2,1)
 cnn_outs = []
for conv in self.cnns:
f =self.conv_and_max_pool(e,conv)
 cnn_outs.append(f)
 out = torch.cat(cnn_outs, dim=1 ).cuda()
out = self.linear1(out)
 return self.linear2(self.dropout(F.relu(out)))
 class SentimentGRU(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(SentimentGRU,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.n_layers = 2
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.gru = nn.GRU(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 def forward(self,x):
hidden = torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda()
e = self.embedding(x)
 _, hidden = self.gru(e, hidden)
 out = torch.cat(( hidden[-2,:,:], hidden[-1,:,:]), dim=1 ).cuda()
out = self.linear1(out)
 return self.linear2(self.dropout(F.relu(out)))
 def train(model,filename,epochs=3):
loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters())
 for epoch in range(epochs):
model.train()
avg_loss = 0
for batch,train_batch in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(train_batch.text.cuda() ).squeeze(1)
 y_true = train_batch.target.float().cuda()
loss = loss_function(y_pred,y_true)
 avg_loss += loss.item()
loss.backward()
optimizer.step()
print('EPOCH: ',epoch,': ',avg_loss/batch)
 print('-'*80)
 torch.save(model.state_dict() , filename)
 print('Training finished.... ')
wv = ".. /input/glove-global-vectors-for-word-representation/glove.6B.100d.txt"
def loadWordVecs() :
embeddings_index = {}
f = open(wv)
 for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f.close()
print('Found %s word vectors.' % len(embeddings_index))
return embeddings_index
def sent2vec(embeddings_index,s):
words = str(s ).lower()
words = word_tokenize(words)
 words = [w for w in words if not w in stopwords.words('english')]
words = [w for w in words if w.isalpha() ]
M = []
for w in words:
try:
M.append(embeddings_index[w])
 except:
continue
M = np.array(M)
 if M.shape[0] != 0:
v = M.min(axis=0)
 w = M.max(axis=0)
 return np.concatenate(( v,w),axis=0)
 else:
return np.zeros(200)
 def doGlove(x_train,x_test):
embeddings_index = loadWordVecs()
xtrain_glove = [sent2vec(embeddings_index,x)for x in tqdm(x_train)]
xtest_glove = [sent2vec(embeddings_index,x)for x in tqdm(x_test)]
xtrain_glove = np.array(xtrain_glove)
 xtest_glove = np.array(xtest_glove)
 return xtrain_glove,xtest_glove,embeddings_index
glove_vecs_train,glove_vecs_test,embeddings_index = doGlove(train_df['text'],test_df['text'] )<prepare_output>
model = SentimentLSTM(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 train(model,'lstm.pt',3)
 print('-'*80 )
train_df[['sent_vec_'+str(i)for i in range(200)]] = pd.DataFrame(glove_vecs_train.tolist())
 test_df[['sent_vec_'+str(i)for i in range(200)]] = pd.DataFrame(glove_vecs_test.tolist())
 train_df.head()<train_model>
model = SentimentBase().cuda()
print(model)
 print('-'*80)
 train(model,'base.pt',5)
 print('-'*80 )
def runXGB(train_X, train_y, test_X, test_y=None, test_X2=None, seed_val=0, child=1, colsample=0.3):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.1
param['max_depth'] = 3
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = child
param['subsample'] = 0.8
param['colsample_bytree'] = colsample
param['seed'] = seed_val
num_rounds = 2000
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=50, verbose_eval=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest, ntree_limit = model.best_ntree_limit)
 if test_X2 is not None:
xgtest2 = xgb.DMatrix(test_X2)
 pred_test_y2 = model.predict(xgtest2, ntree_limit = model.best_ntree_limit)
 return pred_test_y, pred_test_y2, model 
<split>
model = SentimentCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 train(model,'cnn.pt',3)
 print('-'*80 )
cols_to_drop = ['id','text']
train_X = train_df.drop(cols_to_drop+['author'],axis=1)
 test_X = test_df.drop(cols_to_drop,axis=1)
 kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2017)
 cv_scores = []
pred_full_test = 0
pred_train = np.zeros([train_df.shape[0], 3])
 for dev_index, val_index in kf.split(train_df):
dev_X, val_X = train_X.loc[dev_index], train_X.loc[val_index]
dev_y, val_y = train_y[dev_index], train_y[val_index]
pred_val_y, pred_test_y, model = runXGB(dev_X, dev_y, val_X, val_y, test_X, seed_val=0, colsample=0.7)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index,:] = pred_val_y
cv_scores.append(metrics.log_loss(val_y, pred_val_y))
print("cv scores : ", cv_scores)
<create_dataframe>
model = SentimentGRU(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 train(model,'gru.pt',3)
 print('-'*80 )
p = pred_full_test/5
result = pd.DataFrame()
result['id'] = test_id
result['EAP'] = p[:,0]
result['HPL'] = p[:,1]
result['MWS'] = p[:,2]<save_to_csv>
def disable_grad(layer):
for p in layer.parameters() :
p.requires_grad=False
class Ensemble(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(Ensemble,self ).__init__()
self.lstm = SentimentLSTM(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
self.lstm.load_state_dict(torch.load('lstm.pt'))
disable_grad(self.lstm)
 self.gru = SentimentGRU(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
self.gru.load_state_dict(torch.load('gru.pt'))
disable_grad(self.gru)
 self.cnn = SentimentCNN(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
self.cnn.load_state_dict(torch.load('cnn.pt'))
disable_grad(self.cnn)
 self.base = SentimentBase().cuda()
self.base.load_state_dict(torch.load('base.pt'))
disable_grad(self.base)
 self.l_in = nn.Linear(4,1024,bias=False)
 self.l_out = nn.Linear(1024,1,bias=False)
 def forward(self,x):
o1 = self.lstm(x)
 o2 = self.gru(x)
 o3 = self.cnn(x)
 o4 = self.base(x)
 out = torch.cat([o1,o2,o3,o4],1)
 return self.l_out(F.relu(self.l_in(out)))
 model = Ensemble(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 train(model,'ensemble.pt', 3)
 print('-'*80 )
result.head()
result.to_csv("result.csv", index=False )<set_options>
print(os.listdir())
 model = Ensemble(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('ensemble.pt'))
<load_from_csv><EOS> 
print('Threshold:',search_result['threshold'])
 submission_list = list(torchtext.data.BucketIterator(dataset=submission_x,
batch_size=batch_size,
sort=False,
train=False))
pred = []
with torch.no_grad() :
for submission_batch in submission_list:
model.eval()
x = submission_batch.text.cuda()
pred += torch.sigmoid(model(x ).squeeze(1)).cpu().data.numpy().tolist()
pred = np.array(pred)
 df_subm = pd.DataFrame()
df_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1 ).numpy() ]
df_subm['prediction'] =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )