Kaggle competition: Google QUEST Q&A Labeling
<SOS> metric: MCSpearmanR Kaggle data source: google-quest-q&a-labeling<load_from_csv>

 pyLDAvis.enable_notebook()
warnings.filterwarnings('ignore' )
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 print('Load in data successful!' )<define_variables>
sample = pd.read_csv('/kaggle/input/google-quest-challenge/sample_submission.csv')
 sample.head(3 )
SEED = 1111
np.random.seed(SEED)
 TRAINING_PGTS = False
VALIDATING = False
TRAINING = False
print(f'TRAINING_PGTS = {TRAINING_PGTS}
\
VALIDATING = {VALIDATING}
\
TRAINING = {TRAINING}' )<prepare_x_and_y>
train = pd.read_csv('/kaggle/input/google-quest-challenge/train.csv')
 train.head(3 )
train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
train.fillna(train.mean() , inplace=True)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 features = [c for c in train.columns if 'feature' in c]
f_mean = np.mean(train[features[1:]].values, axis=0)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
print(X_train.shape, y_train.shape)
 if VALIDATING:
X_train_v = train[train['date'] <= 416]
y_train_v = np.stack([(X_train_v[c] > 0 ).astype('int')for c in resp_cols] ).T
X_train_v = X_train_v.loc[:, features].values
X_test_v = train[train['date'] > 416]
y_test_v = np.stack([(X_test_v[c] > 0 ).astype('int')for c in resp_cols] ).T
X_test_v = X_test_v.loc[:, features].values
print(X_test_v.shape, y_test_v.shape)
 print('Done data preprocessing!' )<define_search_model>
test = pd.read_csv('/kaggle/input/google-quest-challenge/test.csv')
 test.head(3 )
def create_mlp(num_columns, num_labels, hidden_units,
dropout_rates, label_smoothing, learning_rate):
inp = Input(shape=(num_columns,))
x = BatchNormalization()(inp)
 x = Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = Dense(hidden_units[i] )(x)
 x = BatchNormalization()(x)
 x = Activation(tf.keras.activations.swish )(x)
 x = Dropout(dropout_rates[i+1] )(x)
 x = Dense(num_labels )(x)
 out = Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name='AUC')
)
 return model<import_modules>
targets = [
'question_asker_intent_understanding',
'question_body_critical',
'question_conversational',
'question_expect_short_answer',
'question_fact_seeking',
'question_has_commonly_accepted_answer',
'question_interestingness_others',
'question_interestingness_self',
'question_multi_intent',
'question_not_really_a_question',
'question_opinion_seeking',
'question_type_choice',
'question_type_compare',
'question_type_consequence',
'question_type_definition',
'question_type_entity',
'question_type_instructions',
'question_type_procedure',
'question_type_reason_explanation',
'question_type_spelling',
'question_well_written',
'answer_helpful',
'answer_level_of_information',
'answer_plausible',
'answer_relevance',
'answer_satisfaction',
'answer_type_instructions',
'answer_type_procedure',
'answer_type_reason_explanation',
'answer_well_written'
]
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<prepare_x_and_y>
stopwords=stopwords.words('english')
 train['que_stopwords']=train['question_body'].apply(lambda x : [x for x in x.split() if x in stopwords])
 train['ans_stopwords']=train['answer'].apply(lambda x: [x for x in x.split() if x in stopwords] )
n_samples = 2000
n_groups = 20
assert n_samples % n_groups == 0
idx = np.linspace(0, n_samples-1, num=n_samples)
 X_train_pgts = np.random.random(size=(n_samples, 5))
y_train_pgts = np.random.choice([0, 1], n_samples)
 groups = np.repeat(np.linspace(0, n_groups-1, num=n_groups), n_samples/n_groups)
 groups.shape<split>
def common_ngrams(col,common=10):
corpus=[]
for question in train[col].values:
words=[str(x[0]+' '+x[1])for x in ngrams(question.split() ,2)]
corpus.append(words)
 flatten=[x for one in corpus for x in one]
counter=Counter(flatten)
 most_common=counter.most_common(common)
 string,value=zip(*(most_common))
return string,value
FOLDS = 5
models = []
if TRAINING_PGTS:
gkf = PurgedGroupTimeSeriesSplit(n_splits=FOLDS, group_gap=20)
 splits = list(gkf.split(y_train, groups=train['date'].values))
for fold,(train_indices, test_indices)in tqdm(enumerate(splits)) :
X_train_pgts, X_test_pgts = X_train.iloc[train_indices, :], X_train.iloc[test_indices, :]
y_train_pgts, y_test_pgts = y_train[train_indices], y_train[test_indices]
tf.keras.backend.clear_session()
model = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 er = EarlyStopping(patience = 8,
restore_best_weights = True,
monitor = 'val_loss')
 ReduceLR = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_loss',
factor = 0.1,
patience = 8,
verbose = 1,
mode = 'min')
 model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
 filepath = f'.. /input/js-model-v1/model2_{SEED}_{fold}.hdf5',
save_weights_only = True,
verbose = 0,
monitor = 'val_loss',
save_best_only = True)
 nn_callbacks = [er, ReduceLR, model_checkpoint_callback]
model.fit(X_train_pgts, y_train_pgts,
validation_data =(X_test_pgts, y_test_pgts),
epochs = epochs[0],
batch_size = batch_size,
callbacks = nn_callbacks)
 models.append(model)
 else:
for fold in tqdm(range(FOLDS)) :
tf.keras.backend.clear_session()
model = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 model.load_weights(f'.. /input/js-model-v1/model2_{SEED}_{fold}.hdf5')
 models.append(model )<choose_model_class>
np.set_printoptions(suppress=True )
callbacks = EarlyStopping(monitor='AUC', patience=20, verbose=2, mode ='auto')
 if VALIDATING:
clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 history = clf.fit(
 X_train_v, y_train_v,
epochs = epochs,
batch_size = batch_size,
validation_data =(X_test_v, y_test_v),
callbacks = callbacks 
)
 del X_train_v, y_train_v, X_test_v, y_test_v
else:
if TRAINING:
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 clf.fit(X_train, y_train, epochs=epochs[1],
batch_size=batch_size, verbose=1, 
)
 clf.save_weights('.. /input/js-model-v1/js_model_v3-2_weights.h5')
 print('Training succeeded!
')
 else:
clf = create_mlp(len(features), 5, hidden_units,
dropout_rates, label_smoothing, learning_rate)
 clf.load_weights('.. /input/js-model-v1/js_model_v3-2_weights.h5')
 print('Loading succeeded!
')
 clf.summary()
models.append(clf )<import_modules>
PATH = '.. /input/google-quest-challenge/'
BERT_PATH = '.. /input/bert-base-from-tfhub/bert_en_uncased_L-12_H-768_A-12'
tokenizer = FullTokenizer(BERT_PATH+'/assets/vocab.txt', True)
 MAX_SEQUENCE_LENGTH = 512
df_train = pd.read_csv(PATH+'train.csv')
 df_test = pd.read_csv(PATH+'test.csv')
 df_sub = pd.read_csv(PATH+'sample_submission.csv')
 print('train shape =', df_train.shape)
 print('test shape =', df_test.shape)
 output_categories = list(df_train.columns[11:])
 input_categories = list(df_train.columns[[1,2,5]])
 print('
output categories:
\t', output_categories)
 print('
input categories:
\t', input_categories )
SEED = 1111
tf.random.set_seed(SEED)
 np.random.seed(SEED)
 train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train['weight'] != 0]
features_mean = []
features = [c for c in train.columns if 'feature' in c]
for i in features:
x = train[i].mean()
features_mean.append(x)
 train[i] = train[i].fillna(x)
 train['action'] =(( train['resp'].values)> 0 ).astype(int)
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X_train = train.loc[:, train.columns.str.contains('feature')]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.20, 0.20, 0.20, 0.20]
label_smoothing = 1e-2
learning_rate = 1e-3
def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer = tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics = tf.keras.metrics.AUC(name='AUC')
)
 return model
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(
 len(features), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=2)
 ')
 ", scores)
 models = []
models.append(clf)
 th = 0.503
f = np.median
f_mean = np.mean(train[features[1:]].values,axis=0)
 env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<set_options>
def _get_masks(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 return [1]*len(tokens)+ [0] *(max_seq_length - len(tokens))
def _get_segments(tokens, max_seq_length):

 if len(tokens)>max_seq_length:
raise IndexError("Token length more than max seq length!")
 segments = []
first_sep = True
current_segment_id = 0
for token in tokens:
segments.append(current_segment_id)
 if token == "[SEP]":
if first_sep:
first_sep = False
else:
current_segment_id = 1
return segments + [0] *(max_seq_length - len(tokens))
def _get_ids(tokens, tokenizer, max_seq_length):

 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 input_ids = token_ids + [0] *(max_seq_length-len(token_ids))
return input_ids
def _trim_input(title, question, answer, max_sequence_length,
t_max_len=30, q_max_len=239, a_max_len=239):
t = tokenizer.tokenize(title)
 q = tokenizer.tokenize(question)
 a = tokenizer.tokenize(answer)
 t_len = len(t)
 q_len = len(q)
 a_len = len(a)
 if(t_len+q_len+a_len+4)> max_sequence_length:
if t_max_len > t_len:
t_new_len = t_len
a_max_len = a_max_len + floor(( t_max_len - t_len)/2)
 q_max_len = q_max_len + ceil(( t_max_len - t_len)/2)
 else:
t_new_len = t_max_len
if a_max_len > a_len:
a_new_len = a_len
q_new_len = q_max_len +(a_max_len - a_len)
 elif q_max_len > q_len:
a_new_len = a_max_len +(q_max_len - q_len)
 q_new_len = q_len
else:
a_new_len = a_max_len
q_new_len = q_max_len
if t_new_len+a_new_len+q_new_len+4 != max_sequence_length:
raise ValueError("New sequence length should be %d, but is %d"
%(max_sequence_length,(t_new_len+a_new_len+q_new_len+4)))
 t = t[:t_new_len]
q = q[:q_new_len]
a = a[:a_new_len]
return t, q, a
def _convert_to_bert_inputs(title, question, answer, tokenizer, max_sequence_length):

 stoken = ["[CLS]"] + title + ["[SEP]"] + question + ["[SEP]"] + answer + ["[SEP]"]
input_ids = _get_ids(stoken, tokenizer, max_sequence_length)
 input_masks = _get_masks(stoken, max_sequence_length)
 input_segments = _get_segments(stoken, max_sequence_length)
 return [input_ids, input_masks, input_segments]
def compute_input_arays(df, columns, tokenizer, max_sequence_length):
input_ids, input_masks, input_segments = [], [], []
for _, instance in tqdm(df[columns].iterrows()):
t, q, a = instance.question_title, instance.question_body, instance.answer
t, q, a = _trim_input(t, q, a, max_sequence_length)
 ids, masks, segments = _convert_to_bert_inputs(t, q, a, tokenizer, max_sequence_length)
 input_ids.append(ids)
 input_masks.append(masks)
 input_segments.append(segments)
 return [np.asarray(input_ids, dtype=np.int32),
np.asarray(input_masks, dtype=np.int32),
np.asarray(input_segments, dtype=np.int32)]
def compute_output_arrays(df, columns):
return np.asarray(df[columns] )
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 100)
 pd.set_option('display.max_rows', 100)
 DATA_PATH = '.. /input/jane-street-market-prediction/'
NFOLDS = 5
TRAIN = False
CACHE_PATH = '.. /input/mlp012003weights'
def save_pickle(dic, save_path):
with open(save_path, 'wb')as f:
pickle.dump(dic, f)
 def load_pickle(load_path):
with open(load_path, 'rb')as f:
message_dict = pickle.load(f)
 return message_dict
feat_cols = [f'feature_{i}' for i in range(130)]
target_cols = ['action', 'action_1', 'action_2', 'action_3', 'action_4']
f_mean = np.load(f'{CACHE_PATH}/f_mean_online.npy')
 all_feat_cols = [col for col in feat_cols]
all_feat_cols.extend(['cross_41_42_43', 'cross_1_2'])
 class Model(nn.Module):
def __init__(self):
super(Model, self ).__init__()
self.batch_norm0 = nn.BatchNorm1d(len(all_feat_cols))
self.dropout0 = nn.Dropout(0.2)
 dropout_rate = 0.2
hidden_size = 256
self.dense1 = nn.Linear(len(all_feat_cols), hidden_size)
 self.batch_norm1 = nn.BatchNorm1d(hidden_size)
 self.dropout1 = nn.Dropout(dropout_rate)
 self.dense2 = nn.Linear(hidden_size+len(all_feat_cols), hidden_size)
 self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(dropout_rate)
 self.dense3 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(dropout_rate)
 self.dense4 = nn.Linear(hidden_size+hidden_size, hidden_size)
 self.batch_norm4 = nn.BatchNorm1d(hidden_size)
 self.dropout4 = nn.Dropout(dropout_rate)
 self.dense5 = nn.Linear(hidden_size+hidden_size, len(target_cols))
self.Relu = nn.ReLU(inplace=True)
 self.PReLU = nn.PReLU()
self.LeakyReLU = nn.LeakyReLU(negative_slope=0.01, inplace=True)
 self.RReLU = nn.RReLU()
def forward(self, x):
x = self.batch_norm0(x)
 x = self.dropout0(x)
 x1 = self.dense1(x)
 x1 = self.batch_norm1(x1)
 x1 = self.LeakyReLU(x1)
 x1 = self.dropout1(x1)
 x = torch.cat([x, x1], 1)
 x2 = self.dense2(x)
 x2 = self.batch_norm2(x2)
 x2 = self.LeakyReLU(x2)
 x2 = self.dropout2(x2)
 x = torch.cat([x1, x2], 1)
 x3 = self.dense3(x)
 x3 = self.batch_norm3(x3)
 x3 = self.LeakyReLU(x3)
 x3 = self.dropout3(x3)
 x = torch.cat([x2, x3], 1)
 x4 = self.dense4(x)
 x4 = self.batch_norm4(x4)
 x4 = self.LeakyReLU(x4)
 x4 = self.dropout4(x4)
 x = torch.cat([x3, x4], 1)
 x = self.dense5(x)
 return x
if True:
device = torch.device("cpu")
 model_list = []
tmp = np.zeros(len(feat_cols))
for _fold in range(NFOLDS):
torch.cuda.empty_cache()
model = Model()
model.to(device)
 model_weights = f"{CACHE_PATH}/online_model{_fold}.pth"
model.load_state_dict(torch.load(model_weights, map_location=torch.device('cpu')))
 model.eval()
model_list.append(model )<choose_model_class>
def compute_spearmanr(trues, preds):
rhos = []
for col_trues, col_pred in zip(trues.T, preds.T):
rhos.append(
 spearmanr(col_trues, col_pred + np.random.normal(0, 1e-7, col_pred.shape[0])).correlation)
 return np.nanmean(rhos)
 class CustomCallback(tf.keras.callbacks.Callback):
def __init__(self, valid_data, test_data, batch_size=16, fold=None):
self.valid_inputs = valid_data[0]
self.valid_outputs = valid_data[1]
self.test_inputs = test_data
self.batch_size = batch_size
self.fold = fold
def on_train_begin(self, logs={}):
self.valid_predictions = []
self.test_predictions = []
def on_epoch_end(self, epoch, logs={}):
self.valid_predictions.append(
 self.model.predict(self.valid_inputs, batch_size=self.batch_size))
rho_val = compute_spearmanr(
 self.valid_outputs, np.average(self.valid_predictions, axis=0))
print("
validation rho: %.4f" % rho_val)
 if self.fold is not None:
self.model.save_weights(f'bert-base-{fold}-{epoch}.h5py')
 self.test_predictions.append(
 self.model.predict(self.test_inputs, batch_size=self.batch_size)
)
 def bert_model() :
input_word_ids = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_word_ids')
 input_masks = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_masks')
 input_segments = tf.keras.layers.Input(
(MAX_SEQUENCE_LENGTH,), dtype=tf.int32, name='input_segments')
 bert_layer = hub.KerasLayer(BERT_PATH, trainable=True)
 _, sequence_output = bert_layer([input_word_ids, input_masks, input_segments])
 x = tf.keras.layers.GlobalAveragePooling1D()(sequence_output)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(30, activation="sigmoid", name="dense_output" )(x)
 model = tf.keras.models.Model(
 inputs=[input_word_ids, input_masks, input_segments], outputs=out)
 return model
def train_and_predict(model, train_data, valid_data, test_data,
learning_rate, epochs, batch_size, loss_function, fold):
custom_callback = CustomCallback(
 valid_data=(valid_data[0], valid_data[1]),
test_data=test_data,
batch_size=batch_size,
fold=None)
 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
 model.compile(loss=loss_function, optimizer=optimizer)
 model.fit(train_data[0], train_data[1], epochs=epochs,
batch_size=batch_size, callbacks=[custom_callback])
 return custom_callback
SEED = 1111
np.random.seed(SEED)
 def create_mlp(
 num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate 
):
inp = tf.keras.layers.Input(shape=(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i + 1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation("sigmoid" )(x)
 model = tf.keras.models.Model(inputs=inp, outputs=out)
 model.compile(
 optimizer=tfa.optimizers.RectifiedAdam(learning_rate=learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=label_smoothing),
metrics=tf.keras.metrics.AUC(name="AUC"), 
)
 return model
epochs = 200
batch_size = 4096
hidden_units = [160, 160, 160]
dropout_rates = [0.2, 0.2, 0.2, 0.2]
label_smoothing = 1e-2
learning_rate = 1e-3
tf.keras.backend.clear_session()
tf.random.set_seed(SEED)
 clf = create_mlp(
 len(feat_cols), 5, hidden_units, dropout_rates, label_smoothing, learning_rate 
)
 clf.load_weights('.. /input/jane-street-with-keras-nn-overfit/model.h5')
 tf_models = [clf]<choose_model_class>
gkf = GroupKFold(n_splits=10 ).split(X=df_train.question_body, groups=df_train.question_body)
 test_inputs = compute_input_arays(df_test, input_categories, tokenizer, MAX_SEQUENCE_LENGTH )
class FFN(nn.Module):
def __init__(self, inputCount=160, outputCount=0, hiddenLayerCounts=[80, 40, 20], drop_prob=0.2, nonlin=nn.SiLU()):
super(FFN, self ).__init__()
self.nonlin = nonlin
self.dropout = nn.Dropout(drop_prob)
 self.dense1 = nn.Linear(inputCount, hiddenLayerCounts[0])
 self.batchnorm1 = nn.BatchNorm1d(hiddenLayerCounts[0])
 self.dense2 = nn.Linear(hiddenLayerCounts[0], hiddenLayerCounts[1])
 self.batchnorm2 = nn.BatchNorm1d(hiddenLayerCounts[1])
 self.dense3 = nn.Linear(hiddenLayerCounts[1], hiddenLayerCounts[2])
 self.batchnorm3 = nn.BatchNorm1d(hiddenLayerCounts[2])
 self.outDense = None
if outputCount > 0:
self.outDense = nn.Linear(hiddenLayerCounts[-1], outputCount)
 self.outActivtn = None
if outputCount == 1 or outputCount == 2:
self.outActivtn = nn.Sigmoid()
elif outputCount > 0:
self.outActivtn = nn.Softmax(dim=-1)
 return
def forward(self, X, **kwargs):
X = self.dropout(self.nonlin(self.batchnorm1(self.dense1(X))))
X = self.dropout(self.nonlin(self.batchnorm2(self.dense2(X))))
X = self.dropout(self.nonlin(self.batchnorm3(self.dense3(X))))
if self.outActivtn:
X = self.outActivtn(self.outDense(X))
return X 
<load_from_csv>
models = []
for i in range(5):
model_path = f'.. /input/bertuned-f{i}/bertuned_f{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model)
N_FEAT_TAGS = 29
DEVICE = device
class Clf_Model(nn.Module):
def __init__(self, embed_dim=N_FEAT_TAGS, csv_file='.. /input/jane-street-market-prediction/features.csv'):
super(Clf_Model, self ).__init__()
global N_FEAT_TAGS
N_FEAT_TAGS = 29
dtype = {'tag_0' : 'int8'}
for i in range(1, 29):
k = 'tag_' + str(i)
 dtype[k] = 'int8'
t_df = pd.read_csv(csv_file, usecols=range(1,N_FEAT_TAGS+1), dtype=dtype)
 t_df['tag_29'] = np.array([1] +([0] *(t_df.shape[0]-1)) ).astype('int8')
 self.features_tag_matrix = torch.tensor(t_df.to_numpy())
 N_FEAT_TAGS += 1
self.embed_dim = embed_dim
self.tag_embedding = nn.Embedding(N_FEAT_TAGS+1, embed_dim)
 self.tag_weights = nn.Linear(N_FEAT_TAGS, 1)
 self.dropout = nn.Dropout(0.2)
 self.layer_normal = nn.LayerNorm(embed_dim)
 self.ffn = FFN(inputCount=(130+embed_dim), outputCount=0, hiddenLayerCounts=[(160+embed_dim),(100+embed_dim),(50+embed_dim)], drop_prob=0.25)
 self.outDense = nn.Linear(50+embed_dim, 5)
 self.criterion = nn.BCEWithLogitsLoss()
return
def features2emb(self):

 all_tag_idxs = torch.LongTensor(np.arange(N_FEAT_TAGS))
tag_bools = self.features_tag_matrix
f_emb = self.tag_embedding(all_tag_idxs ).repeat(130, 1, 1)
 f_emb = f_emb * tag_bools[:, :, None]
s = torch.sum(tag_bools, dim=1)
 f_emb = torch.sum(f_emb, dim=-2)/ s[:, None]
return f_emb
def forward(self, features, labels=None):

 features = features.view(-1, 130)
 if labels is not None:
labels = labels.view(-1, 5)
 f_emb = self.features2emb()
features_2 = torch.matmul(features, f_emb)
 features = torch.hstack(( features, features_2))
x = self.ffn(features)
 out_logits = self.outDense(x)
 if labels is None:
out_probs = F.sigmoid(out_logits)
 Probs = namedtuple('Probs',['logits'])
 out_probs = Probs(out_probs)
 return out_probs
batchLoss = self.criterion(out_logits, labels.float())
 out_probs = F.sigmoid(out_logits)
 Loss_Logits = namedtuple('Loss_Logits',['loss','logits'])
 loss_logits = Loss_Logits(batchLoss, out_probs)
 return loss_logits<load_pretrained>
model_path = f'.. /input/bertf1e15/Full-0.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
embNN_model = Clf_Model()
try:
embNN_model.load_state_dict(torch.load(".. /input/jane-emb-nn5-clf-end-noscal-160-100-50/jane_emb_nn5_clf_end_noscal_160_100_50.pt"))
except:
embNN_model.load_state_dict(torch.load(".. /input/jane-emb-nn5-clf-end-noscal-160-100-50/jane_emb_nn5_clf_end_noscal_160_100_50.pt", map_location='cpu'))
embNN_model = embNN_model.eval()<split>
for i in range(2):
model_path = f'.. /input/bertmodelpretrained/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
env = janestreet.make_env()
env_iter = env.iter_test()<concatenate>
for i in range(7):
model_path = f'.. /input/pretrained-bert/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
if True:
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, feat_cols].values
if np.isnan(x_tt.sum()):
x_tt = np.nan_to_num(x_tt)+ np.isnan(x_tt)* f_mean
cross_41_42_43 = x_tt[:, 41] + x_tt[:, 42] + x_tt[:, 43]
cross_1_2 = x_tt[:, 1] /(x_tt[:, 2] + 1e-5)
 feature_inp = np.concatenate((
x_tt,
np.array(cross_41_42_43 ).reshape(x_tt.shape[0], 1),
np.array(cross_1_2 ).reshape(x_tt.shape[0], 1), 
), axis=1)
 torch_pred = np.zeros(( 1, len(target_cols)))
 for model in model_list:
torch_pred += model(torch.tensor(feature_inp, dtype=torch.float ).to(device)).sigmoid().detach().cpu().numpy() / NFOLDS
torch_pred = np.median(torch_pred)
 tf_pred = np.median(np.mean([model(x_tt, training = False ).numpy() for model in tf_models],axis=0))
x_tt = torch.tensor(x_tt ).float().view(-1, 130)
 embnn_p = np.median(embNN_model(x_tt ).logits.detach().cpu().numpy().reshape(( -1, 5)) , axis=1)
 pred_pr = torch_pred*0.35 + tf_pred*0.35 + embnn_p*0.30
pred_df.action = np.where(pred_pr >= 0.5, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<train_model>
for i in range(7):
model_path = f'.. /input/bert-base-tf2-0-training/bert-{i}.h5'
model = bert_model()
model.load_weights(model_path)
 models.append(model )
print("Done !" )<define_variables>
test_predictions = []
BASE = "/kaggle/input/jane-street-market-prediction/"
train = 'train.csv'
ex_test = 'example_test.csv'
features = 'features.csv' 
<load_from_csv>
for model in models:
test_predictions.append(model.predict(test_inputs, batch_size=8))
%%time
train = pd.read_csv(BASE + train)
 ex_test = pd.read_csv(BASE + ex_test)
 features = pd.read_csv(BASE + features )<set_options>
final_predictions = np.mean(test_predictions, axis=0 )
warnings.filterwarnings('ignore')
 tf.random.set_seed(42)
<load_from_csv>
n=df_test['url'].apply(lambda x:(( 'ell.stackexchange.com' in x)or('english.stackexchange.com' in x)) ).tolist()
spelling=[]
for x in n:
if x:
spelling.append(0.5)
 else:
spelling.append(0.)
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', nrows = 3)
 features = [c for c in train.columns if 'feature' in c]
print('Filling...')
 f_mean = train[features[1:]].mean()
train = train.query('weight > 0' ).reset_index(drop = True)
 train[features[1:]] = train[features[1:]].fillna(f_mean)
 train['action'] =(train['resp'] > 0 ).astype('int')
 print('Converting...')
<choose_model_class>
df_sub['question_type_spelling']=spelling
df_sub.iloc[:, 1:] = final_predictions
df_sub['answer_relevance'] = df_sub['answer_relevance'].apply(lambda x : 0.33333334326744 if x < 0.7 else x)
 df_sub.to_csv('submission.csv', index=False )