Kaggle competition: Quora Insincere Questions Classification
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
tqdm.pandas()
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220<feature_engineering>
emb_size = 300
max_features = 200000
maxlen = 100
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<train_model>
def clean_memory(*args):
for arg in args:
del arg
gc.collect()
time.sleep(10 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
path = '/kaggle/input/quora-insincere-questions-classification'
train_df = pd.read_csv(path + "/train.csv")
 test_df = pd.read_csv(path + "/test.csv")
 train_df["question_text"].fillna("_na_", inplace=True)
 test_df["question_text"].fillna("_na_", inplace=True )
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x<normalization>
train_df["num_words"] = train_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
train_df["num_unique_words"] = train_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 train_df["num_chars"] = train_df["question_text"].progress_apply(lambda x: len(str(x)))
class Attention(nn.Module):
def __init__(self, feature_dim, step_dim, bias=True, **kwargs):
super(Attention, self ).__init__(**kwargs)
 self.supports_masking = True
self.bias = bias
self.feature_dim = feature_dim
self.step_dim = step_dim
self.features_dim = 0
weight = torch.zeros(feature_dim, 1)
 nn.init.xavier_uniform_(weight)
 self.weight = nn.Parameter(weight)
 if bias:
self.b = nn.Parameter(torch.zeros(step_dim))
def forward(self, x, mask=None):
feature_dim = self.feature_dim
step_dim = self.step_dim
eij = torch.mm(
 x.contiguous().view(-1, feature_dim),
self.weight
).view(-1, step_dim)
 if self.bias:
eij = eij + self.b
eij = torch.tanh(eij)
 a = torch.exp(eij)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 1, keepdim=True)+ 1e-10
weighted_input = x * torch.unsqueeze(a, -1)
 return torch.sum(weighted_input, 1 )<choose_model_class>
test_df["num_words"] = test_df["question_text"].progress_apply(lambda x: len(str(x ).split()))
test_df["num_unique_words"] = test_df["question_text"].progress_apply(lambda x: len(set(str(x ).split())))
 test_df["num_chars"] = test_df["question_text"].progress_apply(lambda x: len(str(x)) )
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm1_attention = Attention(LSTM_UNITS * 2, MAX_LEN)
 self.lstm2_attention = Attention(LSTM_UNITS * 2, MAX_LEN)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS * 2, DENSE_HIDDEN_UNITS * 2)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS * 2, DENSE_HIDDEN_UNITS * 2)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS * 2, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS * 2, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 h_lstm1_atten = self.lstm1_attention(h_lstm1)
 h_lstm2_atten = self.lstm2_attention(h_lstm2)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( h_lstm1_atten,h_lstm2_atten, max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
def build_vocab(sentences, verbose = True):
vocab = {}
for sentence in tqdm(sentences, disable =(not verbose)) :
for word in sentence:
try:
vocab[word] += 1
except KeyError:
vocab[word] = 1
return vocab
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<load_from_csv>
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
def load_all() :
word2vec_format = {}
glove = [o.split(" ")[0] for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) ]
for word in tqdm(glove):
word2vec_format[word] = 1
clean_memory(glove)
 return word2vec_format
emb_all = load_all()
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<prepare_x_and_y>
def check_coverage(vocab, embeddings_index):
a = {}
oov = {}
k = 0
i = 0
for word in tqdm(vocab):
try:
a[word] = embeddings_index[word]
k += vocab[word]
except:
oov[word] = vocab[word]
i += vocab[word]
pass
print('Found embeddings for {:.2%} of vocab'.format(len(a)/ len(vocab)))
 print('Found embeddings for {:.2%} of all text'.format(k /(k + i)))
 sorted_x = sorted(oov.items() , key=operator.itemgetter(1)) [::-1]
return sorted_x
x_train = preprocess(train['comment_text'])
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'])
 identity_columns = [
'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',
'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
weights = np.ones(( len(x_train),)) / 4
weights +=(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values>=0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values<0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
weights +=(((train['target'].values<0.5 ).astype(bool ).astype(np.int)+ 
(train[identity_columns].fillna(0 ).values>=0.5 ).sum(axis=1 ).astype(bool ).astype(np.int)) > 1 ).astype(bool ).astype(np.int)/ 4
loss_weight = 1.0 / weights.mean()<data_type_conversions>
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]
y_train = np.vstack([(train['target'].values>=0.5 ).astype(np.int),weights] ).T 
<define_variables>
punct = "/-'?!.,
punct_mapping = {"‘": "'", "₹": "e", "´": "'", "°": "", "€": "e", "™": "tm", "√": " sqrt ", "×": "x", "²": "2", "—": "-", "–": "-", "’": "'", "_": "-", "`": "'", '“': '"', '”': '"', '“': '"', "£": "e", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }
def clean_special_chars(text, punct=punct, mapping=punct_mapping):
for p in mapping:
if p in text:
text = text.replace(p, mapping[p])
 for p in punct:
if p in text:
text = text.replace(p, f' {p} ')
 specials = {'\u200b': ' ', '…': '...', '\ufeff': '', 'करना': '', 'है': ''}
for s in specials:
if s in text:
text = text.replace(s, specials[s])
 return text
max_features = None<string_transform>
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: clean_special_chars(x))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: clean_special_chars(x))
sentences = train_df["question_text"].progress_apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab,emb_all)
 oov[:10]
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', "mastrubating": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', "whst": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}
def correct_spelling(x, dic):
for word in dic.keys() :
if word in x:
x = x.replace(word, dic[word])
 return x
max_features = max_features or len(tokenizer.word_index)+ 1
max_features<choose_model_class>
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: correct_spelling(x, mispell_dict))
sentences = train_df["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))<choose_model_class>
go_to_more_common_words = {
'Redmi': 'Mobile',
'OnePlus': 'Mobile',
'Quorans': 'Quoran',
'cryptocurrencies': 'technology',
'Cryptocurrency': 'Technology',
'Blockchain': 'Technology',
'Upwork': 'Technology',
'HackerRank': 'Programming',
}
train_df["question_text"] = train_df["question_text"].progress_apply(lambda x: correct_spelling(x, go_to_more_common_words))
test_df["question_text"] = test_df["question_text"].progress_apply(lambda x: correct_spelling(x, go_to_more_common_words))
sentences = train_df["question_text"].apply(lambda x: x.split())
 vocab = build_vocab(sentences)
 oov = check_coverage(vocab, emb_all)
 oov[:10]
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))<concatenate>
clean_memory(oov, vocab, sentences, mispell_dict, go_to_more_common_words, punct, punct_mapping, emb_all )
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()<prepare_x_and_y>
def load_all_emb(word_index):
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 glove = dict(get_coefs(*o.split(" ")) for o in tqdm(open(emb_path + '/glove.840B.300d/glove.840B.300d.txt')) if o.split(" ")[0] in word_index)
 global max_features
all_embs = np.stack(glove.values())
 emb_mean, emb_std = all_embs.mean() , all_embs.std()
clean_memory(all_embs)
 nb_words = min(max_features, len(word_index))
print(nb_words)
 max_features = nb_words
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, emb_size))
for word, i in tqdm(word_index.items()):
if i >= nb_words: continue
glove_vector = glove.get(word)
 paragram_vector = None
if glove_vector is not None and paragram_vector is not None:
embedding_matrix[i] = 0.7 * glove_vector + 0.3 * paragram_vector
elif glove_vector is not None:
embedding_matrix[i] = glove_vector
elif paragram_vector is not None:
embedding_matrix[i] = paragram_vector
clean_memory(glove_vector, paragram_vector, glove)
 return embedding_matrix
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32 )<data_type_conversions>
emb_path = '/kaggle/input/quora-insincere-questions-classification/embeddings'
tokenizer = Tokenizer(num_words=max_features, filters="")
 tokenizer.fit_on_texts(np.concatenate(( train_df["question_text"].values, test_df["question_text"].values)))
 embeddings = load_all_emb(tokenizer.word_index)
 print(max_features )
x_test_torch = torch.tensor(x_test, dtype=torch.long )<create_dataframe>
train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=47)
 train_X = train_df["question_text"].values
val_X = val_df["question_text"].values
test_X = test_df["question_text"].values
train_X2 = train_df[['num_words', 'num_unique_words', 'num_chars']]
val_X2 = val_df[['num_words', 'num_unique_words', 'num_chars']]
test_X2 = test_df[['num_words', 'num_unique_words', 'num_chars']]
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
print('Train shape: {}'.format(train_X.shape))
print('Validation shape: {}'.format(val_X.shape))
print('Test shape: {}'.format(test_X.shape))
clean_memory(train_df, val_df, test_df )
batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )<compute_test_metric>
early_stopping = EarlyStopping(monitor="val_f1", patience=2, restore_best_weights=True, mode="max")
 reduce_lr = ReduceLROnPlateau(monitor="val_f1", mode="max")
 callbacks = [early_stopping, reduce_lr]
def custom_loss(data, targets):

 bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2] )(data[:,:1],targets[:,:1])
 bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])
 return(bce_loss_1 * loss_weight)+ bce_loss_2<train_model>
def f1(y_true, y_pred):

 def recall(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
 recall = true_positives /(possible_positives + K.epsilon())
 return recall
def precision(y_true, y_pred):
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
 predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
 precision = true_positives /(predicted_positives + K.epsilon())
 return precision
precision = precision(y_true, y_pred)
 recall = recall(y_true, y_pred)
 return 2*(( precision*recall)/(precision+recall+K.epsilon()))
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=custom_loss)
 test_preds = train_model(learn,test_dataset,output_dim=7)
 all_test_preds.append(test_preds )<save_to_csv>
inp1 = Input(shape=(maxlen,))
x = Embedding(max_features, emb_size, weights=[embeddings], trainable=False )(inp1)
 x = SpatialDropout1D(0.4 )(x)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Conv1D(64, 1 )(x)
 max_pool = GlobalMaxPooling1D()(x)
 inp2 = Input(shape=(3,))
y = Dense(64 )(inp2)
 xy = concatenate([max_pool, y], axis=1)
 xy = Dense(128, activation="relu" )(xy)
 xy = Dropout(0.1 )(xy)
 xy = BatchNormalization()(xy)
 outp = Dense(1, activation="sigmoid" )(xy)
 model = Model(inputs=[inp1, inp2], outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1] )
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<define_variables>
model.fit([train_X, train_X2], train_y, batch_size=512, epochs=20, validation_data=([val_X, val_X2], val_y), callbacks=callbacks, verbose=True )
EMBEDDING_FILES = [
'.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec',
'.. /input/glove840b300dtxt/glove.840B.300d.txt'
]<define_variables>
clean_memory(train_X, train_y )
NUM_MODELS = 2
BATCH_SIZE = 512
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4*LSTM_UNITS
EPOCHS = 4
MAX_LEN = 220<define_variables>
pred_val_y = model.predict([val_X, val_X2], batch_size=512, verbose=1)
 def scoring(y_true, y_proba, verbose=True):
def threshold_search(y_true, y_proba):
precision , recall, thresholds = precision_recall_curve(y_true, y_proba)
 thresholds = np.append(thresholds, 1.001)
 F = 2 /(1/precision + 1/recall)
 best_score = np.max(F)
 best_th = thresholds[np.argmax(F)]
return best_th
rkf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10)
 scores = []
ths = []
for train_index, test_index in rkf.split(y_true, y_true):
y_prob_train, y_prob_test = y_proba[train_index], y_proba[test_index]
y_true_train, y_true_test = y_true[train_index], y_true[test_index]
best_threshold = threshold_search(y_true_train, y_prob_train)
 sc = f1_score(y_true_test,(y_prob_test >= best_threshold ).astype(int))
scores.append(sc)
 ths.append(best_threshold)
 best_th = np.mean(ths)
 score = np.mean(scores)
 if verbose: print(f'Best threshold: {np.round(best_th, 4)}, Score: {np.round(score,5)}')
 return best_th, score
optimal_point1, optimal1 = scoring(val_y, pred_val_y)
 print("Optimal F1 {} on threshhold {}".format(optimal1, optimal_point1))
IDENTITY_COLUMNS = [
'transgender', 'female', 'homosexual_gay_or_lesbian', 'muslim', 'hindu',
'white', 'black', 'psychiatric_or_mental_illness', 'jewish'
]<define_variables>
clean_memory(val_X, val_y, pred_val_y )
AUX_COLUMNS = ['target', 'severe_toxicity','obscene','identity_attack','insult','threat']
TEXT_COLUMN = 'comment_text'
TARGET_COLUMN = 'target'
CHARS_TO_REMOVE = '!"
“”’'∞θ÷α•à−β∅³π‘₹´°£€\×™√²—'
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" } 
<define_variables>
all_preds = model.predict([test_X, test_X2], batch_size=512, verbose=1)
 pred_test_y =(np.array(all_preds)> optimal_point1 ).astype(np.int )
<string_transform><EOS> 
submit_df = pd.DataFrame({"qid": test_df["qid"], "prediction": pred_test_y1})
 submit_df.to_csv("submission.csv", index=False )