Kaggle competition: Quora Insincere Questions Classification
filterwarnings('ignore' )<data_type_conversions>
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train_dataset = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 train, val,test = train_dataset.split(split_ratio=[0.8,0.1,0.1],stratified=True,strata_field='target',random_state=random.getstate())
 submission_x = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)})
 text.build_vocab(train_dataset, submission_x, min_freq=3)
 qid.build_vocab(submission_x)
 print('train dataset len:',len(train_dataset))
print('train len:',len(train))
print('val len:',len(val))
print('test len:',len(test))
def preparing_data(df):
df.drop('Id', axis=1, inplace=True)
 df = df[~df['revenue'].ge(1.25 * 10**7)]
df['Open Date'] = pd.to_datetime(df['Open Date'], format='%m/%d/%Y')
 df['OpenDays'] =(df['Open Date'].max() - df['Open Date'] ).astype('timedelta64[D]' ).astype(int)+ 1
df['OpenDays'] = np.log(df['OpenDays'])
 df.drop('Open Date', axis=1, inplace=True)
 df.drop('City', axis=1, inplace=True)
 df = pd.get_dummies(df, columns=['City Group'], drop_first=True)
 df['Type'].replace('MB', 'DT', inplace=True)
 df = pd.get_dummies(df, columns=['Type'], drop_first=True)
 df['revenue'] = np.log(df['revenue'])
 return df<load_from_csv>
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300 )
train = pd.read_csv('./train.csv')
 test = pd.read_csv('./test.csv')
 df = pd.concat([train, test], axis=0 )<categorify>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss
df = preparing_data(df )<drop_column>
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
shuffle=True,
sort=False)
 val_loader = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
shuffle=False,
sort=False)
 test_loader = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
shuffle=False,
sort=False)
 class Sentiment(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.hidden_dim = int(vocab_vectors.shape[1]/2)
 self.n_layers = 2
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.lstm = nn.LSTM(input_size=vocab_vectors.shape[1], hidden_size=self.hidden_dim, bidirectional=True,batch_first=True)
 self.linear2 = nn.Linear(self.hidden_dim*vocab_vectors.shape[1],1)
 self.dropout = nn.Dropout(0.2)
 self.attn = nn.Linear(self.hidden_dim * 2, 100)
 self.attn_combine = nn.Linear(self.hidden_dim * 2, self.hidden_dim)
 def forward(self,x):
hidden =(torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda() ,
torch.zeros(self.n_layers, x.shape[0], self.hidden_dim ).cuda())
 e = self.embedding(x)
 lstm_out, hidden = self.lstm(e, hidden)
 out = torch.cat(( hidden[0][-2,:,:], hidden[0][-1,:,:]), dim=1 ).cuda()
z = torch.cat(( e, lstm_out), 1)
 attn_weights = F.softmax(self.attn(z), dim=1)
 attn_applied = torch.bmm(attn_weights,
lstm_out)
 output = torch.cat(( e, attn_applied), 1)
 attn_combine_output = self.attn_combine(output)
 out = self.dropout(F.relu(attn_combine_output))
out = out.reshape(out.shape[0],out.shape[1]*out.shape[2])
 out = self.linear2(out)
 return out
model = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
lrs = []
for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
model.train()
for batch,train_batch in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(train_batch.text.cuda() ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_true = train_batch.target.float().cuda()
y_numpy_true = train_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
lrs.append(get_lr(optimizer))
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',loss.item() ,' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,val_batch in enumerate(list(val_loader),1):
y_pred = model(val_batch.text.cuda() ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_true = val_batch.target.float().cuda()
y_numpy_true = val_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )
df = df[['P2', 'P6', 'P23', 'P28', 'OpenDays', 'revenue']]<split>
print(os.listdir())
 model = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))
train = df[~df['revenue'].isnull() ]
test = df[df['revenue'].isnull() ]<split>
print('Threshold:',search_result['threshold'])
 submission_list = list(torchtext.data.BucketIterator(dataset=submission_x,
batch_size=batch_size,
sort=False,
train=False))
pred = []
with torch.no_grad() :
for submission_batch in submission_list:
model.eval()
x = submission_batch.text.cuda()
pred += torch.sigmoid(model(x ).squeeze(1)).cpu().data.numpy().tolist()
pred = np.array(pred)
 df_subm = pd.DataFrame()
df_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1 ).numpy() ]
df_subm['prediction'] =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )