Kaggle competition: Quora Insincere Questions Classification
train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )<data_type_conversions>
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(6017)
 print('Seeding done...' )
def create_image_from_row(digit, dest_path):
mat = digit.reshape(28,28)
 mat = mat.astype(np.uint8)
 img = Image.fromarray(mat)
 img.save(dest_path )<define_variables>
text = torchtext.data.Field(lower=True, batch_first=True, tokenize=word_tokenize, fix_length=100)
 qid = torchtext.data.Field()
target = torchtext.data.Field(sequential=False, use_vocab=False, is_target=True)
 train_dataset = torchtext.data.TabularDataset(path='.. /input/train.csv', format='csv',
fields={'question_text':('text',text),
'target':('target',target)})
 train, val,test = train_dataset.split(split_ratio=[0.8,0.1,0.1],stratified=True,strata_field='target',random_state=random.getstate())
 submission_x = torchtext.data.TabularDataset(path='.. /input/test.csv', format='csv',
fields={'qid':('qid', qid),
'question_text':('text', text)})
 text.build_vocab(train_dataset, submission_x, min_freq=3)
 qid.build_vocab(submission_x)
 print('train dataset len:',len(train_dataset))
print('train len:',len(train))
print('val len:',len(val))
print('test len:',len(test))
for index, row in train.iterrows() :
label,digit = row[0], row[1:]
directory = images_train/str(label)
 filename = f"{index}.jpg"
dest_path = directory/filename
digit = digit.values
create_image_from_row(digit, dest_path )<train_model>
glove = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 text.vocab.set_vectors(glove.stoi, glove.vectors, dim=300)
 print(glove.vectors.shape )
for index, row in test.iterrows() :
filename = f"{index}.jpg"
dest_path = images_test/filename
digit = row.values
create_image_from_row(digit, dest_path )<train_model>
class EarlyStopping:

 def __init__(self, patience=7, verbose=False):

 self.patience = patience
self.verbose = verbose
self.counter = 0
self.best_score = None
self.early_stop = False
self.val_loss_min = np.Inf
def __call__(self, val_loss, model):
score = -val_loss
if self.best_score is None:
self.best_score = score
self.save_checkpoint(val_loss, model)
 elif score < self.best_score:
self.counter += 1
if self.verbose:
print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
 if self.counter >= self.patience:
self.early_stop = True
else:
self.best_score = score
self.save_checkpoint(val_loss, model)
 self.counter = 0
def save_checkpoint(self, val_loss, model):

 if self.verbose:
print(f'Validation loss decreased({self.val_loss_min:.6f} --> {val_loss:.6f} ).Saving model...')
 torch.save(model.state_dict() , 'checkpoint.pt')
 self.val_loss_min = val_loss
print('A few samples of the training images.')
 displayTrainingData()<import_modules>
def threshold_search(y_true, y_proba):
best_threshold = 0
best_score = 0
for threshold in [i * 0.01 for i in range(100)]:
with warnings.catch_warnings() :
warnings.simplefilter("ignore")
 score = f1_score(y_true=y_true, y_pred=y_proba > threshold)
 if score > best_score:
best_threshold = threshold
best_score = score
best threshold is % f with score %f'%(best_threshold,best_score))
search_result = {'threshold': best_threshold, 'f1': best_score}
return search_result
torch.cuda.init()
torch.cuda.empty_cache()
print('CUDA MEM:',torch.cuda.memory_allocated())
 print('cuda:', torch.cuda.is_available())
 print('cude index:',torch.cuda.current_device())
 batch_size = 512
print('batch_size:',batch_size)
 print('---')
 train_loader = torchtext.data.BucketIterator(dataset=train,
batch_size=batch_size,
shuffle=True,
sort=False)
 val_loader = torchtext.data.BucketIterator(dataset=val,
batch_size=batch_size,
shuffle=False,
sort=False)
 test_loader = torchtext.data.BucketIterator(dataset=test,
batch_size=batch_size,
shuffle=False,
sort=False)
 class Sentiment(nn.Module):
def __init__(self,vocab_vectors,padding_idx,batch_size):
super(Sentiment,self ).__init__()
print('Vocab vectors size:',vocab_vectors.shape)
 self.batch_size = batch_size
self.lstm_dim1 = 128
self.lstm_dim2 = 64
self.lstm_dim3 = 32
self.lstm_dim4 = 16
self.lstm_dim5 = 8
self.embedding = nn.Embedding.from_pretrained(vocab_vectors)
 self.embedding.weight.requires_grad = False
self.embedding.padding_idx = padding_idx
self.filter = nn.Conv1d(300,300,1)
 self.lstm1 = nn.LSTM(input_size=vocab_vectors.shape[1],
hidden_size=self.lstm_dim1,
bidirectional=True,
batch_first=True)
 self.lstm2 = nn.LSTM(input_size=self.lstm_dim1*2,
hidden_size=self.lstm_dim2,
bidirectional=True,
batch_first=True)
 self.lstm3 = nn.LSTM(input_size=self.lstm_dim2*2,
hidden_size=self.lstm_dim3,
bidirectional=True,
batch_first=True)
 self.lstm4 = nn.LSTM(input_size=self.lstm_dim3*2,
hidden_size=self.lstm_dim4,
bidirectional=True,
batch_first=True)
 self.lstm5 = nn.LSTM(input_size=self.lstm_dim4*2,
hidden_size=self.lstm_dim5,
bidirectional=True,
batch_first=True)
 self.classifier = nn.Sequential(
 nn.Linear(self.lstm_dim1*2+self.lstm_dim5*2, 1024),
nn.ReLU(inplace=True),
nn.Dropout(0.1),
nn.Linear(1024, 1)
)
 def forward(self,x):
hidden =(torch.zeros(2, x.shape[0], self.lstm_dim1 ).cuda() ,
torch.zeros(2, x.shape[0], self.lstm_dim1 ).cuda())
 hidden2 =(torch.zeros(2,x.shape[0], self.lstm_dim2 ).cuda() ,
torch.zeros(2,x.shape[0], self.lstm_dim2 ).cuda())
 hidden3 =(torch.zeros(2, x.shape[0], self.lstm_dim3 ).cuda() ,
torch.zeros(2, x.shape[0], self.lstm_dim3 ).cuda())
 hidden4 =(torch.zeros(2, x.shape[0], self.lstm_dim4 ).cuda() ,
torch.zeros(2, x.shape[0], self.lstm_dim4 ).cuda())
 hidden5 =(torch.zeros(2, x.shape[0], self.lstm_dim5 ).cuda() ,
torch.zeros(2, x.shape[0], self.lstm_dim5 ).cuda())
 e = self.embedding(x)
 e = e.permute(0,2,1)
 f = F.relu(self.filter(e))
f = f.permute(0,2,1)
 lstm_out, hidden = self.lstm1(f, hidden)
 lstm2_out,hidden2 = self.lstm2(lstm_out,hidden2)
 lstm3_out, hidden3= self.lstm3(lstm2_out,hidden3)
 lstm4_out, hidden4= self.lstm4(lstm3_out,hidden4)
 lstm5_out, hidden5= self.lstm5(lstm4_out,hidden5)
 out = torch.cat([
hidden[0][-2,:,:],
hidden[0][-1,:,:],
hidden5[0][-2,:,:],
hidden5[0][-1,:,:]], dim=1 ).cuda()
return self.classifier(out)
 model = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
print(model)
 print('-'*80)
 early_stopping = EarlyStopping(patience=2,verbose=True)
 loss_function = nn.BCEWithLogitsLoss().cuda()
optimizer = optim.Adam(model.parameters() ,lr=1e-3)
 def get_lr(optimizer):
for param_group in optimizer.param_groups:
return param_group['lr']
losses = []
val_losses=[]
epoch_acc=[]
epoch_val_acc=[]
lrs = []
for epoch in range(100):
epoch_losses=[]
epoch_val_losses = []
preds = []
val_preds=[]
targets = []
acc = []
model.train()
for batch,train_batch in enumerate(list(iter(train_loader)) ,1):
optimizer.zero_grad()
y_pred = model(train_batch.text.cuda() ).squeeze(1)
 y_numpy_pred =torch.sigmoid(y_pred ).cpu().detach().numpy()
preds += y_numpy_pred.tolist()
y_true = train_batch.target.float().cuda()
y_numpy_true = train_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
loss = loss_function(y_pred,y_true)
 epoch_losses.append(loss.item())
 loss.backward()
optimizer.step()
lrs.append(get_lr(optimizer))
acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rtraining(batch,loss,acc)| ',batch,' ===>',np.mean(epoch_losses),' acc ',np.mean(acc),end='')
 losses.append(np.mean(epoch_losses))
targets = np.array(targets)
 preds = np.array(preds)
 search_result = threshold_search(targets, preds)
 train_f1 = search_result['f1']
epoch_acc.append(np.mean(acc))
targets = []
val_acc=[]
model.eval()
with torch.no_grad() :
for batch,val_batch in enumerate(list(val_loader),1):
y_pred = model(val_batch.text.cuda() ).squeeze(1)
 y_numpy_pred = torch.sigmoid(y_pred ).cpu().detach().numpy()
val_preds += y_numpy_pred.tolist()
y_true = val_batch.target.float().cuda()
y_numpy_true = val_batch.target.cpu().detach().numpy()
targets += y_numpy_true.tolist()
val_loss = loss_function(y_pred,y_true)
 epoch_val_losses.append(val_loss.item())
 val_acc.append(accuracy_score(y_numpy_true,np.round(y_numpy_pred)))
 if batch % 100 == 0:
print('\rvalidation(batch,acc)| ',batch,' ===>', np.mean(val_acc),end='')
 val_losses.append(np.mean(epoch_val_losses))
epoch_val_acc.append(np.mean(val_acc))
targets = np.array(targets)
 val_preds = np.array(val_preds)
 search_result = threshold_search(targets, val_preds)
 val_f1 = search_result['f1']
print('
EPOCH: ',epoch,'
has acc of ',epoch_acc[-1],' ,has loss of ',losses[-1], ' ,f1 of ',train_f1,'
val acc of ',epoch_val_acc[-1],' ,val loss of ',val_losses[-1],' ,val f1 of ',val_f1)
 print('-'*80)
 if early_stopping.early_stop:
print("Early stopping at ",epoch," epoch")
 break
else:
early_stopping(1.-val_f1, model)
 print('Training finished.... ' )
from fastai.vision import *<train_model>
print(os.listdir())
 model = Sentiment(text.vocab.vectors, padding_idx=text.vocab.stoi[text.pad_token], batch_size=batch_size ).cuda()
model.load_state_dict(torch.load('checkpoint.pt'))
<define_variables><EOS> 
print('Threshold:',search_result['threshold'])
 submission_list = list(torchtext.data.BucketIterator(dataset=submission_x,
batch_size=batch_size,
sort=False,
train=False))
pred = []
with torch.no_grad() :
for submission_batch in submission_list:
model.eval()
x = submission_batch.text.cuda()
pred += torch.sigmoid(model(x ).squeeze(1)).cpu().data.numpy().tolist()
pred = np.array(pred)
 df_subm = pd.DataFrame()
df_subm['qid'] = [qid.vocab.itos[j] for i in submission_list for j in i.qid.view(-1 ).numpy() ]
df_subm['prediction'] =(pred > search_result['threshold'] ).astype(int)
 print(df_subm.head())
 df_subm.to_csv('submission.csv', index=False )