Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<data_type_conversions>
FOLD = 0
SEED = 1337
NOTIFY_EACH_EPOCH = False
WORKERS = 0
BATCH_SIZE = 512
N_SPLITS = 10
np.random.seed(SEED)
 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def save_checkpoint(state):

 print(" Saving checkpoint")
 filename = f'./checkpoint-{state["epoch"]}.pt.tar'
torch.save(state, filename)
 def initialize(model, path=None, optimizer=None):
if path == None:
checkpoints = glob.glob('./*.pt.tar')
 path = checkpoints[np.argmax([int(checkpoint.split('checkpoint-')[1].split('.')[0])for checkpoint in checkpoints])]
checkpoint = torch.load(path)
 model.load_state_dict(checkpoint['model'])
 print(f' Loaded checkpoint {path} | Trained for {checkpoint["epoch"] + 1} epochs')
 if optimizer:
optimizer.load_state_dict(checkpoint['optimizer'])
 epoch = checkpoint['epoch'] + 1
train_iteration = checkpoint['train_iteration']
val_iteration = checkpoint['val_iteration']
return model, optimizer, epoch, train_iteration, val_iteration
else:
return model
X_train =(train.iloc[:,1:].values ).astype('float32')
 y_train = train.iloc[:,0].values.astype('int32')
 X_test = test.values.astype('float32')
<train_model>
sample_submission = pd.read_csv('.. /input/sample_submission.csv')
 train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
img_rows, img_cols = 28, 28
X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols,1)
 X_test = X_test.reshape(X_test.shape[0], img_rows, img_cols,1 )<data_type_conversions>
kfold = KFold(n_splits=N_SPLITS, shuffle=True, random_state=SEED)
 train_idx, val_idx = list(kfold.split(train)) [FOLD]
x_train, x_val = train.iloc[train_idx], train.iloc[val_idx]
x_train.to_csv('train.csv')
 x_val.to_csv('val.csv' )
mean_px = X_train.mean().astype(np.float32)
 std_px = X_train.std().astype(np.float32)
 def standardize(x):
return(x-mean_px)/std_px
X_train = standardize(X_train)
 X_test = standardize(X_test )<categorify>
nlp = English()
def tokenize(sentence):
x = nlp(sentence)
 return [token.text for token in x]
y_train = to_categorical(y_train)
 num_classes = y_train.shape[1]<choose_model_class>
question_field = data.Field(tokenize=tokenize, lower=True, batch_first=True, include_lengths=True)
 target_field = data.Field(sequential=False, use_vocab=False, batch_first=True)
 train_fields = [ 
('id', None), 
('qid', None), 
('question_text', question_field), 
('target', target_field)
 ]
test_fields = [ 
('qid', None), 
('question_text', question_field)
 ]
train_dataset, val_dataset = data.TabularDataset.splits('./', train='train.csv', validation='val.csv', format='CSV', skip_header=True, fields=train_fields)
 test_dataset = data.TabularDataset('.. /input/test.csv', format='CSV', skip_header=True, fields=test_fields)
 vectors = torchtext.vocab.Vectors('.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt')
 question_field.build_vocab(train_dataset, max_size=95000)
 question_field.vocab.set_vectors(vectors.stoi, vectors.vectors, vectors.dim)
 pretrained_embedding = question_field.vocab.vectors
train_dataloader, val_dataloader = data.BucketIterator.splits(( train_dataset, val_dataset),(BATCH_SIZE, BATCH_SIZE), sort_key=lambda x: len(x.question_text), sort_within_batch=True)
 test_dataloader = data.BucketIterator(test_dataset, 1, sort=False, shuffle=False)
 print(f'Train Dataset: {len(train_dataset)}')
 print(f'Val Dataset: {len(val_dataset)}')
 print(f'Test Dataset: {len(test_dataset)}' )
gen = ImageDataGenerator(rotation_range = 10,
zoom_range = 0.1,
width_shift_range = 0.1,
height_shift_range = 0.1,
shear_range = 0.1 )<split>
class SelfAttention(nn.Module):
def __init__(self, hidden_size, batch_first=False):
super(SelfAttention, self ).__init__()
self.hidden_size = hidden_size
self.batch_first = batch_first
self.att_weights = nn.Parameter(torch.Tensor(1, hidden_size), requires_grad=True)
 stdv = 1.0 / math.sqrt(self.hidden_size)
 for weight in self.att_weights:
nn.init.uniform_(weight, -stdv, stdv)
 def get_mask(self):
pass
def forward(self, inputs, lengths):
if self.batch_first:
batch_size, max_len = inputs.size() [:2]
else:
max_len, batch_size = inputs.size() [:2]
weights = torch.bmm(inputs,
self.att_weights 
.permute(1, 0)
.unsqueeze(0)
.repeat(batch_size, 1, 1)
)
 attentions = torch.softmax(F.relu(weights.squeeze()), dim=-1)
 mask = torch.ones(attentions.size() , requires_grad=True ).cuda()
for i, l in enumerate(lengths):
if l < max_len:
mask[i, l:] = 0
masked = attentions * mask
_sums = masked.sum(-1 ).unsqueeze(-1)
 attentions = masked.div(_sums)
 weighted = torch.mul(inputs, attentions.unsqueeze(-1 ).expand_as(inputs))
representations = weighted.sum(1 ).squeeze()
return representations, attentions
class BaselineLSTM(nn.Module):
def __init__(self, embedding):
super(BaselineLSTM, self ).__init__()
self.embedding = nn.Embedding.from_pretrained(embedding)
 self.lstm = nn.LSTM(input_size=300, hidden_size=128, num_layers=2, batch_first=True, bidirectional=True)
 self.attention = SelfAttention(128*2, batch_first=True)
 self.fc = nn.Linear(128*2, 1)
 self.logit = nn.Linear(1, 1)
 def forward(self,x, x_len):
x = self.embedding(x)
 x = nn.utils.rnn.pack_padded_sequence(x, x_len, batch_first=True)
 out,(hidden, _)= self.lstm(x)
 x, lengths = nn.utils.rnn.pad_packed_sequence(out, batch_first=True)
 x, _ = self.attention(x, lengths)
 x = self.fc(x)
 x = self.logit(x ).view(-1)
 return x
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.10, random_state=4)
 batches = gen.flow(X_train, y_train, batch_size=64)
 val_batches=gen.flow(X_val, y_val, batch_size=64 )<choose_model_class>
start_epoch = 0
epochs = 7
early_stopping = 10
train_iteration = 0
val_iteration = 0
threshold = 0.35
model = BaselineLSTM(pretrained_embedding ).to(device)
 optimizer = optim.Adam(model.parameters())
 scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10, verbose=True)
 criterion = nn.BCEWithLogitsLoss()
get_n_params(model )
model = Sequential()
model.add(Conv2D(64, kernel_size=(3, 3),
activation='relu',
input_shape=(img_rows, img_cols, 1)))
 model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3, 3), strides=1, activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Conv2D(64, kernel_size=(3, 3), strides=1, activation='relu'))
model.add(BatchNormalization())
 model.add(Conv2D(64, kernel_size=(3, 3), strides=1, activation='relu'))
model.add(BatchNormalization())
 model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.2))
model.add(Flatten())
 model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax')) 
<choose_model_class>
print(f' Training Finished | Best Epoch {best_epoch} | LR: {optimizer.param_groups[0]["lr"]}
Best Train Loss: {best_train_loss} | Best Val Loss: {best_val_loss} | Best Train F1: {round(best_train_f1, 4)} | Best Val F1: {round(best_val_f1, 4)}' )
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer='adam',
metrics=['accuracy'] )<train_model>
model = BaselineLSTM(pretrained_embedding ).to(device)
 model = initialize(model)
 preds = []
model.eval()
with torch.no_grad() :
for i, batch in tqdm(enumerate(test_dataloader), total=int(len(test_dataset)/ BATCH_SIZE)) : 
(question, length)= batch.question_text
question = question.to(device)
 out = model(question, length)
 out = torch.sigmoid(out)
 pred = out.detach().cpu().numpy()
preds.append(pred)
 preds = np.concatenate(preds, axis=0 ).reshape(-1, 1)
 preds =(preds > best_threshold ).astype(int )
<save_to_csv><EOS> 
sample_submission['prediction'] = preds
mlc.kaggle.save_sub(sample_submission, 'submission.csv' )