Kaggle competition: Quora Insincere Questions Classification
train_X, train_y = train.drop('revenue', axis=1), train['revenue']
test_X = test.drop('revenue', axis=1 )<define_search_space>
df = pd.read_csv('.. /input/train.csv')
 df["question_text"].fillna("_
parameters = [{'alpha': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000],
'normalize': [True, False]}]<save_to_csv>
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )
lr = LinearRegression(normalize=True ).fit(train_X, train_y)
 lr_res = np.e ** lr.predict(test_X)
 pd.DataFrame(lr_res, columns=['Prediction'] ).to_csv('linear_regression.csv', index=True, index_label='Id' )<train_on_grid>
maxlen = 100
num_words = 75966
dim = 300
def load_and_prec() :
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape)
 print('lower...')
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 print('clean misspellings...')
 train_df["question_text"] = train_df["question_text"].apply(lambda x: replace_typical_misspell(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: replace_typical_misspell(x))
print('lower...')
 train_df["question_text"] = train_df["question_text"].apply(lambda x: x.lower())
 test_df["question_text"] = test_df["question_text"].apply(lambda x: x.lower())
 print('clean text...')
 train_df["question_text"] = train_df["question_text"].apply(lambda x: clean_text(x))
test_df["question_text"] = test_df["question_text"].apply(lambda x: clean_text(x))
print('fill na...')
 train_X = train_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
print('tokenize...')
 tokenizer = Tokenizer(num_words=num_words)
 tokenizer.fit_on_texts(list(train_X)+list(test_X))
train_X = tokenizer.texts_to_sequences(train_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
return train_X, test_X, train_y, tokenizer.word_index
grid1 = GridSearchCV(Ridge() , parameters)
 grid1.fit(train_X, train_y)
 grid1.best_estimator_<save_to_csv>
x_train,x_val,y_train,y_val = train_test_split(train_X,train_y,random_state=SEED,stratify=train_y)
 x_train,x_test,y_train,y_test = train_test_split(x_train,y_train,test_size=0.1,random_state=SEED,stratify=y_train)
 print(x_train.shape)
 print(x_test.shape)
 print(x_val.shape)
 print(y_train.shape)
 print(y_test.shape)
 print(y_val.shape )
ridge_grid = grid1.predict(test_X)
 ridge_grid = np.e ** ridge_grid
pd.DataFrame(ridge_grid, columns=['Prediction'] ).to_csv('ridge_grid.csv', index=True, index_label='Id' )<train_on_grid>
print('Glove...')
 def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
embedding_matrix_glov = np.zeros(( num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_glov[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
grid2 = GridSearchCV(Lasso() , parameters)
 grid2.fit(train_X, train_y)
 grid2.best_estimator_<save_to_csv>
print('Para...')
 EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
print(len(all_embs))
embedding_matrix_para = np.zeros(( num_words, dim))
count=0
for word, i in word_index.items() :
if i >= num_words:
break
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix_para[i] = embedding_vector
else:
count += 1
print('embedding matrix size:',embedding_matrix_glov.shape)
 print('Number of words not in vocab:',count)
 del embeddings_index,all_embs
gc.collect()
lasso_grid = grid2.predict(test_X)
 lasso_grid = np.e ** lasso_grid
pd.DataFrame(lasso_grid, columns=['Prediction'] ).to_csv('lasso_grid.csv', index=True, index_label='Id' )<train_on_grid>
print('Concatting matrixes...')
 matrixes = [embedding_matrix_glov,embedding_matrix_para]
matrix = np.mean(matrixes,axis=0)
 print(matrix.shape)
 del embedding_matrix_glov,embedding_matrix_para
gc.collect()
params = [{'alpha': [0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30, 100, 300, 1000, 3000, 10000],
'normalize': [True, False]}]
grid3 = GridSearchCV(ElasticNet(max_iter=10**4), parameters)
 grid3.fit(train_X, train_y)
 grid3.best_estimator_<save_to_csv>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss
elastic_grid = grid3.predict(test_X)
 elastic_grid = np.e ** elastic_grid
pd.DataFrame(elastic_grid, columns=['Prediction'] ).to_csv('elastic_grid.csv', index=True, index_label='Id' )<train_model>
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_train, dtype=torch.long ).cuda() ,
torch.tensor(y_train, dtype=torch.float32 ).cuda()),
batch_size=batch_size,
shuffle=True)
 val_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_val, dtype=torch.long ).cuda() ,
torch.tensor(y_val, dtype=torch.float32 ).cuda()),
batch_size=batch_size,
shuffle=False)
 test_loader = torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(x_test, dtype=torch.long ).cuda() ,
torch.tensor(y_test, dtype=torch.float32 ).cuda()),
batch_size=batch_size,
shuffle=False)
 class Sentiment(nn.Module):
def __init__(self,matrix,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',matrix.shape)
 self.batch_size = batch_size
self.hidden_dim = 128
self.n_layers = 2
self.embedding = nn.Embedding(matrix.shape[0],matrix.shape[1])
 self.embedding.weight = nn.Parameter(torch.tensor(matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.lstm = nn.LSTM(input_size=matrix.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear1 = nn.Linear(self.n_layers*self.hidden_dim,self.hidden_dim)
 self.linear2 = nn.Linear(self.hidden_dim,1)
 self.dropout = nn.Dropout(0.2)
 def forward(self,x):
hidden =(torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 e = self.embedding(x)
 _, hidden = self.lstm(e, hidden)
 out = torch.cat(( hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1 ).cuda()
out = self.linear1(F.relu(out))
return self.linear2(self.dropout(out))
model = Sentiment(matrix, batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
lrs = []
for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
model.train()
for batch,(x_batch,y_true)in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(x_batch ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
lrs.append(get_lr(optimizer))
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,(x_val_batch,y_true)in enumerate(list(val_loader),1):
y_pred = model(x_val_batch ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_numpy_true = y_true.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )
ridge_bayesian = BayesianRidge(normalize=True)
 ridge_bayesian.fit(train_X, train_y)
 ridge_bayesian.alpha_<save_to_csv>
print(os.listdir())
 model = Sentiment(matrix, batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))
ridge_bayesian_res = ridge_bayesian.predict(test_X)
 ridge_bayesian_res = np.e ** ridge_bayesian_res
pd.DataFrame(ridge_bayesian_res, columns=['Prediction'] ).to_csv('ridge_bayesian.csv', index=True, index_label='Id' )<compute_test_metric>
print('Threshold:',search_result['threshold'])
 print(test_X.shape)
 submission_list = list(torch.utils.data.DataLoader(dataset=torch.utils.data.TensorDataset(torch.tensor(test_X, dtype=torch.long ).cuda()),
batch_size=batch_size,
shuffle=False))
pred = []
with torch.no_grad() :
model.eval()
for(x,)in submission_list:
y_pred = torch.sigmoid(model(x ).squeeze(1)).detach()
pred += y_pred.cpu().numpy().tolist()
pred = np.array(pred)
 df_subm = pd.read_csv('.. /input/sample_submission.csv')
 df_subm.prediction =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv("submission.csv", index=False )