Kaggle competition: Don't Overfit! II
tf.random.set_seed(42)
 features_columns = ["feature_%d" % i for i in range(130)]
columns_dtypes = {}
for column in features_columns:
columns_dtypes[column] = "float32"
columns_dtypes["resp_1"] = "float32"
columns_dtypes["resp_2"] = "float32"
columns_dtypes["resp_3"] = "float32"
columns_dtypes["resp_4"] = "float32"
columns_dtypes["resp"] = "float32"
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv', dtype=columns_dtypes)
 features = [c for c in train.columns if "feature" in c]<data_type_conversions>
warnings.filterwarnings('ignore')
 pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', -1)
 pd.set_option('display.expand_frame_repr', False)
 pd.options.display.max_columns = None
f_mean = train[features].median()
train = train.query('date > 85' ).reset_index(drop = True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train[features] = train[features].fillna(f_mean)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
np.save('f_mean.npy', f_mean )<prepare_x_and_y>
train_data = pd.read_csv('.. /input/train.csv')
 test_data = pd.read_csv('.. /input/test.csv')
 train_label = train_data.pop('target')
 test_id = test_data.pop('id')
 train_data.drop('id', axis=1, inplace=True)
 stats_data = pd.concat([train_data, test_data])
 print('Train rows:', train_data.shape)
 print('Test rows:', test_data.shape)
 print('Stats rows:', stats_data.shape )
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<init_hyperparams>
def normDist_WoE(frame, target, nrOfBins):
ranges = []
for i in range(0, nrOfBins + 1):
ranges.append(norm.ppf(i / nrOfBins, loc=0, scale=1))
df = pd.DataFrame(frame)
 df.columns = ['val']
df['target'] = target.astype(int)
 df['bin'] = 0
for i in range(1, nrOfBins + 1):
df.loc[(df['val'] >= ranges[i-1])&(df['val'] < ranges[i]), 'bin'] = i
result = []
for i in range(1, nrOfBins + 1):
dic = {}
dic.update(BinNr=i, Min_Value=ranges[i-1], Max_Value=ranges[i], Count=len(df[df['bin'] == i]),
Event=len(df[(df['bin'] == i)&(df['target'] == 1)]),
NonEvent=len(df[(df['bin'] == i)&(df['target'] != 1)]))
result.append(dic)
 result = pd.DataFrame(result)
 result = result.reindex(columns=(['BinNr', 'Min_Value', 'Max_Value', 'Count', 'Event', 'NonEvent']))
result['Event_Rate'] = result['Event'] / result['Count']
result['NonEvent_Rate'] = result['NonEvent'] / result['Count']
result['Dist_Event'] = result['Event'] / sum(result['Event'])
 result['Dist_NonEvent'] = result['NonEvent'] / sum(result['NonEvent'])
 result['WoE'] = np.log(result['Dist_Event'] / result['Dist_NonEvent'])
 result['IV'] =(result['Dist_Event'] - result['Dist_NonEvent'])* np.log(result['Dist_Event'] /result['Dist_NonEvent'])
 return(sum(result['IV']), result )
params = {
'colsample_bytree': 0.4,
'learning_rate': 0.001,
'max_depth': 12,
'subsample': 0.8,
'seed': 42,
'tree_method': 'gpu_hist'
}
params["objective"] = 'binary:logistic'
params["eval_metric"] = 'logloss'<train_model>
BIN_SIZE = 5
woe_table = pd.DataFrame()
iv_table = []
for i in train_data.columns:
curr_iv, curr_woe = normDist_WoE(train_data[i], train_label, BIN_SIZE)
 iv_table.append([i, curr_iv])
 curr_woe['Index'] = i
curr_woe = curr_woe.reindex(columns=['Index'] + curr_woe.columns[:-1].tolist())
 curr_woe = pd.DataFrame(curr_woe)
 woe_table = pd.concat([woe_table, curr_woe])
 iv_table = pd.DataFrame(iv_table)
 iv_table.columns = ['Index', 'IV']
iv_table = iv_table.sort_values(by='IV', ascending=False)
 display(iv_table.head())
 plt.plot(iv_table['IV'].values)
 plt.ylabel('IV')
 plt.show()
print("len(IV >= 0.2)", len(iv_table[iv_table['IV'] >= 0.2]))
print("len(IV >= 0.15)", len(iv_table[iv_table['IV'] >= 0.15]))
print("len(IV >= 0.1)", len(iv_table[iv_table['IV'] >= 0.1]))
iv_list = iv_table['Index'].values
display("Top 15 Indices in order:", iv_list[0:15] )
y_tr = train['action'].values
xgb_path = 'xgb_best.model'
dtrain = xgb.DMatrix(X, label=y_tr)
 bst = xgb.train(params, dtrain, 100, [(dtrain, 'train')])
 bst.save_model(xgb_path )<choose_model_class>
train_data_filtered = train_data[[*three_top_15_lists[three_top_15_lists['Count'] == 3].Index.values,
*three_top_15_lists[three_top_15_lists['Count'] == 2].Index.values]]
test_data_filtered = test_data[[*three_top_15_lists[three_top_15_lists['Count'] == 3].Index.values,
*three_top_15_lists[three_top_15_lists['Count'] == 2].Index.values]]
train_label_int = train_label.astype(int)
 display(train_data_filtered.head())
 display(test_data_filtered.head())
 train_X, val_X, train_y, val_y = train_test_split(train_data_filtered, train_label_int, test_size=0.1, random_state = None, shuffle = True )
def create_mlp(num_columns, num_labels, hidden_units, dropout_rates, label_smoothing, learning_rate):
inp = tf.keras.layers.Input(shape =(num_columns,))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(dropout_rates[0] )(x)
 for i in range(len(hidden_units)) :
x = tf.keras.layers.Dense(hidden_units[i] )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Activation(tf.keras.activations.swish )(x)
 x = tf.keras.layers.Dropout(dropout_rates[i+1] )(x)
 x = tf.keras.layers.Dense(num_labels )(x)
 out = tf.keras.layers.Activation('sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate = learning_rate),
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = label_smoothing),
metrics = tf.keras.metrics.AUC(name = 'AUC'), 
)
 return model<define_search_space>
logit = LogisticRegressionCV(solver='liblinear', cv=5, dual=False, penalty='l1', multi_class='ovr')
 logit.fit(train_X, train_y)
 score = logit.score(val_X, val_y)
 print("Logistic Regression:", score)
 print(logit.predict(val_X))
display([p[1] for p in logit.predict_proba(val_X)])
 clf = KNeighborsClassifier(n_neighbors=3, weights='distance', algorithm='auto', p=2)
 clf.fit(train_X, train_y)
 score = clf.score(val_X, val_y)
 print("KNN:", score)
 for k in [RBF() , Matern() , RationalQuadratic() , DotProduct() ]:
gpc = GaussianProcessClassifier(kernel=k, n_restarts_optimizer=5, max_iter_predict=200)
 gpc.fit(train_X, train_y)
 score = gpc.score(val_X, val_y)
 print("GaussianProcess", k, score)
 nb = GaussianNB()
nb.fit(train_X, train_y)
 score = nb.score(val_X, val_y)
 print("Naive Bayes:", score)
 print(nb.predict(val_X))
display([p[1] for p in nb.predict_proba(val_X)] )
batch_size = 5000
hidden_units = [160, 160, 160]
dropout_rates = [
0.25,
0.25,
0.25,
0.25
]
label_smoothing = 1e-2
learning_rate = 1e-3
num_models = 2<train_model>
<load_pretrained><EOS> 
logit = LogisticRegressionCV(solver='liblinear', cv=5, dual=False, penalty='l1', multi_class='ovr')
 logit.fit(train_data_filtered, train_label_int)
 logit_prediction = [p[1] for p in logit.predict_proba(test_data_filtered)]
nb = GaussianNB()
nb.fit(train_data_filtered, train_label_int)
 nb_prediction = [p[1] for p in nb.predict_proba(test_data_filtered)]
sub_preds = [(i + j)/ 2 for i, j in zip(logit_prediction, nb_prediction)]
submission = [[a, b] for a, b in zip(test_id.values, sub_preds)]
submission = pd.DataFrame(submission, columns=['id', 'target'])
 display(submission.head())
 print("len(submission)", len(submission))
print("len(target ~ 1)", len(submission[submission.target >= 0.5]))
print("len(target ~ 0)", len(submission[submission.target < 0.5]))
sns.distplot(submission['target'], hist=True, kde=True, color = 'blue', bins = 25 ).set_title('Predictions')
 plt.show()
submission.to_csv("submission.csv", index=False )