Kaggle competition: Quora Insincere Questions Classification
<SOS> metric: FScore_1 Kaggle data source: quora-insincere-questions-classification<create_dataframe>
import os
import time
import numpy as np
import pandas as pd
from tqdm import tqdm
import math
from sklearn.model_selection import train_test_split
from sklearn import metrics
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D
from keras.layers import Bidirectional, GlobalMaxPool1D
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K
from keras.engine.topology import Layer
from keras import initializers, regularizers, constraints, optimizers, layers
ds = get_dataset(files_test, CFG, augment=False, repeat=False,
labeled=False, return_image_names=True)
 image_names = np.array([img_name.numpy().decode("utf-8")
 for img, img_name in iter(ds.unbatch())] )<save_to_csv>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 print("Train shape : ",train_df.shape)
 print("Test shape : ",test_df.shape )
for i in range(CFG["net_count"]):
submission = pd.DataFrame(dict(
 image_name = image_names,
target = probs[i,:,0]))
submission = submission.sort_values('image_name')
 submission.to_csv(f'submission_model_{i}.csv', index=False )<save_to_csv>
train_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)
 embed_size = 300
max_features = 95000
maxlen = 70
train_X = train_df["question_text"].fillna("_
val_X = val_df["question_text"].fillna("_
test_X = test_df["question_text"].fillna("_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_X))
train_X = tokenizer.texts_to_sequences(train_X)
 val_X = tokenizer.texts_to_sequences(val_X)
 test_X = tokenizer.texts_to_sequences(test_X)
 train_X = pad_sequences(train_X, maxlen=maxlen)
 val_X = pad_sequences(val_X, maxlen=maxlen)
 test_X = pad_sequences(test_X, maxlen=maxlen)
 train_y = train_df['target'].values
val_y = val_df['target'].values
submission = pd.DataFrame(dict(
 image_name = image_names,
target = np.mean(probs[:,:,0], axis=0)))
 submission = submission.sort_values('image_name')
 submission.to_csv('submission_models_blended.csv', index=False )<set_options>
np.random.seed(2018)
 trn_idx = np.random.permutation(len(train_X))
val_idx = np.random.permutation(len(val_X))
train_X = train_X[trn_idx]
val_X = val_X[val_idx]
train_y = train_y[trn_idx]
val_y = val_y[val_idx]
%matplotlib inline
InteractiveShell.ast_node_interactivity = "all"<load_from_csv>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
data_dir = '.. /input/stage2wdatafiles/'
df_seed = pd.read_csv(data_dir + 'WNCAATourneySeeds.csv')
 df_result = pd.read_csv(data_dir + 'WNCAATourneyCompactResults.csv')
 df_seed.tail(3)
 df_result.tail(3 )<data_type_conversions>
filter_sizes = [1,2,3,5]
num_filters = 36
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix] )(inp)
 x = Reshape(( maxlen, embed_size, 1))(x)
 maxpool_pool = []
for i in range(len(filter_sizes)) :
conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embed_size),
kernel_initializer='he_normal', activation='elu' )(x)
 maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))
z = Concatenate(axis=1 )(maxpool_pool)
 z = Flatten()(z)
 z = Dropout(0.1 )(z)
 outp = Dense(1, activation="sigmoid" )(z)
 model = Model(inputs=inp, outputs=outp)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] )
def seed_to_int(seed):
s_int = int(seed[1:3])
 return s_int
def clean_df(df_seed, df_result):
df_seed['seed_int'] = df_seed['Seed'].apply(seed_to_int)
 df_seed.drop(['Seed'], axis=1, inplace=True)
 df_result.drop(['DayNum', 'WLoc', 'NumOT'], axis=1, inplace=True)
 return df_seed, df_result
df_seed, df_result = clean_df(df_seed, df_result)
 df_seed.head(3)
 df_result.head(3 )<merge>
model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))
def merge_seed_result(df_seed, df_result):
df_win_seed = df_seed.rename(columns={'TeamID':'WTeamID', 'seed_int':'WSeed'})
 df_loss_seed = df_seed.rename(columns={'TeamID':'LTeamID', 'seed_int':'LSeed'})
 df_result = df_result.merge(df_win_seed, how='left', on=['Season', 'WTeamID'])
 df_result = df_result.merge(df_loss_seed, how='left', on=['Season', 'LTeamID'])
 df_result['SeedDiff'] = np.abs(df_result['WSeed'] - df_result['LSeed'])
 df_result['ScoreDiff'] = np.abs(df_result['WScore'] - df_result['LScore'])
 return df_result
df_result = merge_seed_result(df_seed, df_result)
 df_result.head(3 )<filter>
pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_cnn_val_y>thresh ).astype(int))))
df_result = df_result[df_result['ScoreDiff']>3]<count_values>
pred_cnn_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
df_result['upset'] = [1 if ws > ls else 0 for ws, ls, in zip(df_result["WSeed"], df_result["LSeed"])]
print("upset probability")
 df_result['upset'].value_counts() / len(df_result)* 100<filter>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
this_season=2019
total_season=10
train = df_result[(df_result["Season"]>=(this_season - total_season)) &(df_result["Season"]<(this_season-1)) ]
print(train.shape )<feature_engineering>
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
df_result["Seed_combi"]=[str(ws)+'_'+str(ls)if ws<ls else str(ls)+'_'+str(ws)for ws, ls in zip(df_result["WSeed"], df_result["LSeed"])]
df_result.head(3 )<create_dataframe>
EMBEDDING_FILE = '.. /input/embeddings/glove.840B.300d/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
df_result_aggs = pd.DataFrame()
df_result_filter_aggs = pd.DataFrame()
df_result_season = df_result[(df_result["Season"]>=(this_season - total_season)) &(df_result["Season"]<(this_season-1)) ]
for value in range(16):
df_result_agg = df_result_season[df_result_season["SeedDiff"]==value].groupby("SeedDiff" ).agg({"upset": ["mean", "count"]})
 df_result_agg.columns = [col[0]+"_"+col[1]+"_"+"all" for col in df_result_agg.columns]
df_result_filter_agg = df_result_season[df_result_season["SeedDiff"]==value].groupby("Seed_combi" ).agg({"upset": ["mean", "count"]})
 df_result_filter_agg.columns = [col[0]+"_"+col[1] for col in df_result_filter_agg.columns]
if value==0:
df_result_agg["upset_mean_all"] = 0.5
df_result_filter_agg["upset_mean"] = 0.5
df_result_aggs = pd.concat([df_result_aggs, df_result_agg])
 df_result_filter_aggs = pd.concat([df_result_filter_aggs, df_result_filter_agg])
 df_result_aggs
df_result_filter_aggs.tail(10 )<merge>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
df_result = df_result.join(df_result_aggs, how='left', on="SeedDiff" ).join(df_result_filter_aggs, how='left', on='Seed_combi')
 df_result["upset_prob"] = [m if c > 20 else a for a, m, c in zip(df_result["upset_mean_all"], df_result["upset_mean"], df_result["upset_count"])]
df_result.tail()<compute_test_metric>
pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_glove_val_y>thresh ).astype(int))))
valid = df_result[(df_result["Season"]==(this_season-1)) ]
log_loss(valid['upset'], valid['upset_prob'] )<merge>
pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
valid = df_result[(df_result["Season"]==(this_season-1)) ]
valid = valid.join(df_result_aggs.drop("upset_count_all", axis=1), how='left', on='SeedDiff')
 valid.fillna(0, inplace=True)
 log_loss(valid['upset'], valid['upset_prob_manually'] )<compute_test_metric>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
log_loss(valid['upset'], np.clip(valid['upset_prob_manually'], 0.05, 0.95))<filter>
EMBEDDING_FILE = '.. /input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
df_seed_2019 = df_seed[df_seed["Season"]==2019]<feature_engineering>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
this_season=2019
total_season=10
train = df_result[(df_result["Season"]>=(this_season - total_season)) ]
print(train.shape)
 df_result["Seed_combi"]=[str(ws)+'_'+str(ls)if ws<ls else str(ls)+'_'+str(ws)for ws, ls in zip(df_result["WSeed"], df_result["LSeed"])]
df_result.head()<create_dataframe>
pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_fasttext_val_y>thresh ).astype(int))))
df_result_aggs = pd.DataFrame()
df_result_filter_aggs = pd.DataFrame()
for value in range(16):
df_result_agg = df_result[df_result["SeedDiff"]==value].groupby("SeedDiff" ).agg({"upset": ["mean", "count"]})
 df_result_agg.columns = [col[0]+"_"+col[1]+"_"+"all" for col in df_result_agg.columns]
df_result_filter_agg = df_result[df_result["SeedDiff"]==value].groupby("Seed_combi" ).agg({"upset": ["mean", "count"]})
 df_result_filter_agg.columns = [col[0]+"_"+col[1] for col in df_result_filter_agg.columns]
if value==0:
df_result_agg["upset_mean_all"] = 0.5
df_result_filter_agg["upset_mean"] = 0.5
df_result_aggs = pd.concat([df_result_aggs, df_result_agg])
 df_result_filter_aggs = pd.concat([df_result_filter_aggs, df_result_filter_agg])
 df_result_aggs
df_result_filter_aggs.tail(10 )<feature_engineering>
pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
df_result_aggs.loc[0, 'upset_mean_all'] = 0.5
df_result_aggs.loc[10, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[11, 'upset_mean_all'])/ 2
df_result_aggs.loc[11, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs.loc[12, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs.loc[13, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs.loc[14, 'upset_mean_all'] =(0.0 + df_result_aggs.loc[15, 'upset_mean_all'])/ 2
df_result_aggs = df_result_aggs.fillna(-1)
 sns.barplot(df_result_aggs.index, df_result_aggs.upset_mean_all)
 plt.title('probability of upset based on past result aggretation')
 plt.show()<merge>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
test = pd.read_csv(".. /input/WSampleSubmissionStage2.csv")
 test = pd.DataFrame(np.array([ID.split("_")for ID in test["ID"]]), columns=["Season", "TeamA", "TeamB"], dtype=int)
 test.head(3)
 test = test.merge(df_seed_2019, how='left', left_on=["Season", "TeamA"], right_on=["Season", "TeamID"])
 test = test.rename(columns={"seed_int": "TeamA_seed"} ).drop("TeamID", axis=1)
 test = test.merge(df_seed_2019, how='left', left_on=["Season", "TeamB"], right_on=["Season", "TeamID"])
 test = test.rename(columns={"seed_int": "TeamB_seed"} ).drop("TeamID", axis=1)
 test['SeedDiff'] = np.abs(test.TeamA_seed - test.TeamB_seed)
 test.head(3 )<merge>
EMBEDDING_FILE = '.. /input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE, encoding="utf8", errors='ignore')if len(o)>100)
 all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
word_index = tokenizer.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector
inp = Input(shape=(maxlen,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)
 x = Attention(maxlen )(x)
 x = Dense(64, activation="relu" )(x)
 x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'] )
test["Seed_combi"]=[str(a)+'_'+str(b)if a<b else str(b)+'_'+str(a)for a, b in zip(test["TeamA_seed"], test["TeamB_seed"])]
test.head()
test = test.join(df_result_aggs, how='left', on="SeedDiff" ).join(df_result_filter_aggs, how='left', on='Seed_combi' ).fillna(-1)
 test["upset_prob"] = [m if c > 20 else a for a, m, c in zip(test["upset_mean_all"], test["upset_mean"], test["upset_count"])]
test["win_prob"] = [(1-upset_prob)if teamA<teamB else upset_prob if teamA>teamB else 0.5
for teamA, teamB, upset_prob in zip(test['TeamA_seed'], test['TeamB_seed'], test['upset_prob'])]
test.tail()<save_to_csv>
model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))
submit = pd.read_csv(".. /input/WSampleSubmissionStage2.csv")
 submit["Pred"] = test['win_prob']
submit.to_csv("submission_agg_all_manually_noclip.csv", index=False)
 submit.head()<save_to_csv>
pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)
 for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 print("F1 score at threshold {0} is {1}".format(thresh, metrics.f1_score(val_y,(pred_paragram_val_y>thresh ).astype(int))))
clipped_sub = np.clip(test['win_prob'], 0.05, 0.95)
 submit = pd.read_csv(".. /input/WSampleSubmissionStage2.csv")
 submit["Pred"] = clipped_sub
submit.to_csv("submission_agg_all_manually_cliped.csv", index=False)
 submit.head()<import_modules>
pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1 )
import numpy as np
import pandas as pd
from sklearn.metrics import log_loss
from sklearn.model_selection import RandomizedSearchCV
from sklearn.ensemble import ExtraTreesClassifier<load_from_csv>
del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x
time.sleep(10 )
teams = pd.read_csv('.. /input/wdatafiles/WTeams.csv')
 teams2 = pd.read_csv('.. /input/wdatafiles/WTeamSpellings.csv', encoding='latin-1')
 season_cresults = pd.read_csv('.. /input/wdatafiles/WRegularSeasonCompactResults.csv')
 season_dresults = pd.read_csv('.. /input/wdatafiles/WRegularSeasonDetailedResults.csv')
 tourney_cresults = pd.read_csv('.. /input/wdatafiles/WNCAATourneyCompactResults.csv')
 tourney_dresults = pd.read_csv('.. /input/wdatafiles/WNCAATourneyDetailedResults.csv')
 slots = pd.read_csv('.. /input/wdatafiles/WNCAATourneySlots.csv')
 seeds = pd.read_csv('.. /input/wdatafiles/WNCAATourneySeeds.csv')
 seeds = {'_'.join(map(str,[int(k1),k2])) :int(v[1:3])for k1, v, k2 in seeds[['Season', 'Seed', 'TeamID']].values}
seeds = {**seeds, **{k.replace('2018_','2019_'):seeds[k] for k in seeds if '2018_' in k}}
cities = pd.read_csv('.. /input/wdatafiles/WCities.csv')
 gcities = pd.read_csv('.. /input/wdatafiles/WGameCities.csv')
 seasons = pd.read_csv('.. /input/wdatafiles/WSeasons.csv')
 sub = pd.read_csv('.. /input/WSampleSubmissionStage2.csv' )<merge>
pred_val_y =(4 * pred_glove_val_y + pred_fasttext_val_y + 3 * pred_paragram_val_y + 2 * pred_cnn_val_y)/ 10.0
thresholds = []
for thresh in np.arange(0.1, 0.501, 0.01):
thresh = np.round(thresh, 2)
 res = metrics.f1_score(val_y,(pred_val_y > thresh ).astype(int))
thresholds.append([thresh, res])
 print("F1 score at threshold {0} is {1}".format(thresh, res))
thresholds.sort(key=lambda x: x[1], reverse=True)
 best_thresh = thresholds[0][0]
print("Best threshold: ", best_thresh )
<categorify><EOS> 
pred_test_y =(4 * pred_glove_test_y + pred_fasttext_test_y + 3 * pred_paragram_test_y + 2 * pred_cnn_test_y)/ 10.0
pred_test_y =(pred_test_y > best_thresh ).astype(int)
 out_df = pd.DataFrame({"qid":test_df["qid"].values})
 out_df['prediction'] = pred_test_y
out_df.to_csv("submission.csv", index=False )