Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<load_from_csv>
%matplotlib inline
train = pd.read_csv('.. /input/train.csv')
 print('Shape of train set {}'.format(train.shape))
test = pd.read_csv('.. /input/test.csv')
 print('Shape of test set {}'.format(test.shape))
dataDirPath = "/kaggle/input/jane-street-market-prediction/"
train = dt.fread(dataDirPath + 'train.csv', na_strings = ['NA','NULL','NaN','\\N'])
 train = train[(f.date>85)&(f.weight>0),:]
train[:,["action1","action2","action3","action4","action"]] = train[:, [f.resp_1>0, f.resp_2>0, f.resp_3>0, f.resp_4>0, f.resp>0]]
train[["action1","action2","action3","action4","action"]] = dt.int32
features = [c for c in train.names if 'feature' in c]
resps = [c for c in train.names if 'resp' in c]
actions = [c for c in train.names if 'action' in c]
meanImputeValues = train[:, 8:137].mean().to_numpy() [0,:]
train = train.to_pandas()
train = train.astype({c: np.float32 for c, t in train.dtypes.items() if t == np.float64})
 train = train.fillna(train.mean())
 train.loc[train['feature_0'] ==-1, 'feature_0'] = 0<split>
def onehot_encode(cat_feat, data):

 encoded = pd.get_dummies(data[cat_feat], prefix=cat_feat, dummy_na=True)
 res = pd.concat([data.drop(columns=[cat_feat]), encoded], axis='columns')
 return res
def add_quality(df, componente='pared', component='wall'):
for i in [1,2,3]:
i_quality =(df['e{}{}'.format(componente, i)] == 1)
 df.loc[i_quality, '{}_quality'.format(component)] = i
return df
def to_english(df, sp_pre='pared', eng_pre='wall_', translate=None):

 for sp in translate.keys() :
spanish_name = sp_pre + '{}'.format(sp)
 english_name = eng_pre + '{}'.format(translate[sp])
 df.rename(columns={spanish_name: english_name}, inplace=True)
 return df
nSplits = 5
groupGap = 5
cv = PurgedGroupTimeSeriesSplit(
 n_splits=5,
max_train_group_size=np.inf,
group_gap=0,
max_test_group_size=np.inf 
)
 foldList = []
trainXList = []; trainYList = []; trainWList = []
validXList = []; validYList = []; validWList =[];validDList = [];
for fold,(tr, te)in enumerate(cv.split(train['resp'].values, train['resp'].values, train['date'].values)) :
if fold != 4: continue
subTrain, subValid = train.loc[tr,], train.loc[te,]
trainX, trainY, trainW = subTrain.loc[tr, features].values, subTrain.loc[tr, resps].values, subTrain.loc[tr, 'weight'].values
validX, validY, validW,validD = subValid.loc[te, features].values, subValid.loc[te, resps].values, subValid.loc[te, 'weight'].values, subValid.loc[te, 'date'].values
foldList.append(fold)
 trainXList.append(trainX); trainYList.append(trainY); trainWList.append(trainW)
 validXList.append(validX); validYList.append(validY); validWList.append(validW); validDList.append(validD )<choose_model_class>
test['Target'] = np.nan
data_all = pd.concat([train, test])
 print('Shape of all data: {}'.format(data_all.shape))
n_house = data_all['idhogar'].nunique()
print('
hiddenList = [128, 64,64]
dropoutList = [0.2, 0.3,0.3]
baseMLP = MLPModel(hiddenList,dropoutList)
 modelPrefix = 'MLP_{}d{}l.params'.format(hiddenList[0], len(hiddenList))
modelCtx = mx.gpu() if mx.context.num_gpus() else mx.cpu()
dataCtx = mx.gpu() if mx.context.num_gpus() else mx.cpu()
mlpTrainer = MLPTrainer(baseMLP, dataCtx, modelCtx)

epochs = 50
esEpochs = 3
evalCriteria = 'max'
batchSize = 128*8
learningRate = 0.001
sampleRate = 0.8
smoothingAlpha=0.005
initializer = mx.init.Xavier(magnitude=2.24)
 optimizer = 'adam';
lossFunc = SigmoidBinaryCrossEntropyLoss(from_sigmoid=True)
 trainerParamsList = {'epochs': epochs, 'esEpochs': esEpochs, 'evalCriteria': evalCriteria,
'batchSize': batchSize, 'learningRate': learningRate, 'sampleRate': sampleRate, 'smoothingAlpha':smoothingAlpha,
'initializer': initializer, 'optimizer':optimizer, 'lossFunc': lossFunc} 
<train_model>
n_row = data_all.shape[0]
n_null = data_all.drop('Target', axis='columns' ).isnull().values.sum(axis=0)
 columns = list(data_all.columns)
 columns.remove('Target')
 pd.DataFrame({'column': columns,
'n_null': n_null} ).sort_values('n_null', ascending=False ).head(10 )
modelList = []
for fold in foldList:
fold=0
subMark = "fold"+str(fold)
 trainX, trainY, trainW = np.array(trainXList[fold]), np.array(trainYList[fold]),np.array(trainWList[fold])
 testX, testY, testW,testD = np.array(validXList[fold]), np.array(validYList[fold]), np.array(validWList[fold]), np.array(validDList[fold])
 subFit = mlpTrainer.fit(subMark, trainerParamsList, trainX, trainY, trainW, testX, testY, testW,testD)
 modelList.append(subFit)
 break<train_model>
def mk_derived_feats(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 return df
def mk_agg_feats(df):
aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['sum', 'mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
return df
def drop_redundant(df):
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['hhsize', 'area2'], axis=1, inplace=True)
 return df
env = janestreet.make_env()
envIter = env.iter_test()
optThreshold = 0.5
for(testDf, predDf)in tqdm(envIter):
if testDf['weight'].item() > 0:
testDf.loc[testDf['feature_0'] == -1, 'feature_0'] = 0
xTT = testDf.loc[:, features].values
if np.isnan(xTT[:, 1:].sum()):
xTT[:, 1:] = np.nan_to_num(xTT[:, 1:])+ np.isnan(xTT[:, 1:])* meanImputeValues
pred = 0.
 pred = subFit(nd.array(xTT,ctx=dataCtx)) [0].asnumpy()
predDf.action = np.where(pred >= optThreshold, 1, 0 ).astype(int)
 else:
predDf.action = 0
env.predict(predDf)
<import_modules>
yes_no_map = {'no': 0, 'yes': 1}
data_all['dependency'] = data_all['dependency'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefe'] = data_all['edjefe'].replace(yes_no_map ).astype(np.float32)
 data_all['edjefa'] = data_all['edjefa'].replace(yes_no_map ).astype(np.float32)
 data_all['idhogar'] = LabelEncoder().fit_transform(data_all['idhogar'])
 data_all = mk_derived_feats(data_all)
 data_all = mk_agg_feats(data_all)
 data_all = drop_redundant(data_all )
import os, gc
import numpy as np
from numba import njit
import datatable as dtable
import pandas as pd
import janestreet
import xgboost as xgb
from hyperopt import hp, fmin, tpe, Trials
from hyperopt.pyll.base import scope
from sklearn.metrics import roc_auc_score, roc_curve
from sklearn.model_selection import GroupKFold
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from tqdm.notebook import tqdm
from joblib import dump, load<groupby>
fe_feats = [ff for ff in data_all.columns if ff.startswith('fe_')]
agg_feats = [ff for ff in data_all.columns if ff.startswith('agg')]
def weighted_average(a):
w = []
n = len(a)
 for j in range(1, n + 1):
j = 2 if j == 1 else j
w.append(1 /(2**(n + 1 - j)))
 return np.average(a, weights = w )<load_from_csv>
basic_feats = ['dependency']
%%time
print('Loading...')
 train = dtable.fread('/kaggle/input/jane-street-market-prediction/train.csv' ).to_pandas()
features = [c for c in train.columns if 'feature' in c]
print('Filling...')
 train = train.query('weight > 0' ).reset_index(drop = True)
 train[features] = train[features].fillna(method = 'ffill' ).fillna(0)
 train['action'] =(train['resp'] > 0 ).astype('int')
 print('Finish.' )<define_variables>
head_df.loc[head_df['male'] == 1, 'head_gender'] = 'male'
head_df.loc[head_df['female'] == 1, 'head_gender'] = 'female'
print('Shape of head_df: {}'.format(head_df.shape))
head_df = onehot_encode('head_gender', head_df)
 head_gender_feats = [cc for cc in head_df.columns if 'head_gender' in cc]
head_gender_feats
n_splits = 5
group_gap = 31<import_modules>
for i in range(1, 10):
head_df.loc[head_df['instlevel{}'.format(i)] == 1, 'head_edu_level'] = i
head_df = head_df.rename(columns={'escolari': 'head_school_years'} )
class GroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_size=None 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_size = max_train_size
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = n_groups // n_folds
group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
for train_group_idx in unique_groups[:group_test_start]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
if self.max_train_size and self.max_train_size < train_end:
train_array = train_array[train_end -
self.max_train_size:train_end]
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 yield [int(i)for i in train_array], [int(i)for i in test_array]<import_modules>
cols = ['idhogar', 'head_school_years', 'head_edu_level'] + head_gender_feats
data_all = pd.merge(data_all, head_df[cols], how='left', on='idhogar')
 print(data_all.shape )
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<choose_model_class>
house_head_feats = ['head_school_years', 'head_edu_level'] + head_gender_feats
def optimise(params):
print(params)
 p = {'learning_rate': params['learning_rate'],
'max_depth': params['max_depth'],
'gamma': params['gamma'],
'min_child_weight': params['min_child_weight'],
'subsample': params['subsample'],
'colsample_bytree': params['colsample_bytree'],
'verbosity': 0,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'tree_method': 'gpu_hist',
'random_state': 42,
}
scores = []
gkf = PurgedGroupTimeSeriesSplit(n_splits = n_splits, group_gap = group_gap)
 for fold,(tr, te)in enumerate(gkf.split(train['action'].values, train['action'].values, train['date'].values)) :
X_tr, X_val = train.loc[tr, features].values, train.loc[te, features].values
y_tr, y_val = train.loc[tr, 'action'].values, train.loc[te, 'action'].values
d_tr = xgb.DMatrix(X_tr, y_tr)
 d_val = xgb.DMatrix(X_val, y_val)
 clf = xgb.train(p, d_tr, params['n_round'], [(d_val, 'eval')], verbose_eval = False)
 val_pred = clf.predict(d_val)
 score = roc_auc_score(y_val, val_pred)
 scores.append(score)
 del clf, val_pred, d_tr, d_val, X_tr, X_val, y_tr, y_val, score
rubbish = gc.collect()
score_avg = weighted_average(scores)
 print(score_avg)
 return - score_avg<prepare_x_and_y>
data_all = add_quality(data_all, componente='pared', component='wall')
 data_all = add_quality(data_all, componente='techo', component='roof')
 data_all = add_quality(data_all, componente='viv', component='floor')
 print(data_all.shape )
%%time
p_best = {'learning_rate': 0.014106988708201764,
'max_depth': 8,
'gamma': 9.800749651802157,
'min_child_weight': 0.3032862674190433,
'subsample': 0.4648851101943981,
'colsample_bytree': 0.994909039539885,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'tree_method': 'gpu_hist',
}
X_tr, y_tr = train.loc[train['date'] > 85, features].values, train.loc[train['date'] > 85, 'action'].values
d_tr = xgb.DMatrix(X_tr, y_tr)
 models = []
for seed in range(5):
p_best['random_state'] = seed
clf = xgb.train(p_best, d_tr, 950)
 models.append(clf)
 rubbish = gc.collect()<categorify>
translate = {'blolad': 'block',
'zocalo': 'socket',
'preb': 'cement',
'des': 'waste',
'mad': 'wood',
'zinc': 'zink',
'fibras': 'natural_fibers',
'other': 'other'}
data_all = to_english(data_all, sp_pre='pared', eng_pre='wall_',
translate=translate)
 wall_feats = [cc for cc in data_all.columns if 'wall_' in cc]
translate = {
'moscer': 'mosaic',
'cemento': 'cement',
'other': 'other',
'natur': 'natural',
'notiene': 'no_floor',
'madera': 'wood'
}
data_all = to_english(data_all, sp_pre='piso', eng_pre='floor_', translate=translate)
 floor_feats = [cc for cc in data_all.columns if 'floor_' in cc]
translate = {
'zinc': 'zinc',
'entrepiso': 'fiber cement',
'cane': 'natural fibers',
'otro': 'other'
}
data_all = to_english(data_all, sp_pre='techo', eng_pre='roof_', translate=translate)
 roof_feats = [cc for cc in data_all.columns if 'roof_' in cc]
@njit
def fast_fillna(array, values):
if np.isnan(array.sum()):
array = np.where(np.isnan(array), values, array)
 return array
train.loc[0, features[1:]] = fast_fillna(train.loc[0, features[1:]].values, 0 )<split>
material_feats = roof_feats + wall_feats + floor_feats
env = janestreet.make_env()
env_iter = env.iter_test()<predict_on_test>
translate = {
'guadentro': 'inside_house',
'guafuera': 'outside_house',
'guano': 'no'
}
data_all = to_english(data_all, sp_pre='abasta', eng_pre='water_provision_',
translate=translate)
 water_feats = [cc for cc in data_all.columns if 'water_provision_' in cc]
translate = {
'public': 'public',
'planpri': 'private_plan',
'noelec': 'no',
'coopele': 'cooperate'
}
data_all = to_english(data_all, sp_pre='', eng_pre='electric_', translate=translate)
 elec_feats = [cc for cc in data_all.columns if 'electric_' in cc]
translate = {
'cinar1': 'no',
'cinar2': 'electricity',
'cinar3': 'gas',
'cinar4': 'charcoal'
}
data_all = to_english(data_all, sp_pre='energco', eng_pre='energy_', translate=translate)
 energy_feats = [cc for cc in data_all.columns if 'energy_' in cc]
translate = {
'1': 'no',
'2': 'sewer',
'3': 'septic_tank',
'5': 'black hole',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='sanitario', eng_pre='toilet_', translate=translate)
 toilet_feats = [cc for cc in data_all.columns if 'toilet_' in cc]
translate = {
'1': 'tanker truck',
'2': 'buried',
'3': 'burning',
'4': 'throw empty place',
'5': 'throw to river',
'6': 'other'
}
data_all = to_english(data_all, sp_pre='elimbasu', eng_pre='rubbish_', translate=translate)
 rubbish_feats = [cc for cc in data_all.columns if 'rubbish_' in cc]
opt_th = 0.503
tmp = np.zeros(len(features))
for(test_df, pred_df)in tqdm(env_iter):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
x_tt[0, :] = fast_fillna(x_tt[0, :], tmp)
 tmp = x_tt[0, :]
d_tt = xgb.DMatrix(x_tt)
 pred = 0.
 for clf in models:
pred += clf.predict(d_tt)/ len(models)
 pred_df.action = np.where(pred >= opt_th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
facility_feats = water_feats + elec_feats + energy_feats + toilet_feats + rubbish_feats
import pandas as pd
import gc
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf<load_from_csv>
translate = {
'1': 'own_fully_paid',
'2': 'own_pay_installment',
'3': 'rented',
'4': 'precarious',
'5': 'other'
}
data_all = to_english(data_all, sp_pre='tipovivi', eng_pre='living_type_',
translate=translate )
train = pd.read_csv('/kaggle/input/jane-street-market-prediction/train.csv')
 train = reduce_mem_usage(train)
 features = [c for c in train.columns if 'feature' in c]
NAN_VALUE = -999<feature_engineering>
live_feats = [cc for cc in data_all.columns if 'living_type_' in cc]
train = train.astype({c: np.float32 for c in train.select_dtypes(include='float16' ).columns})
 train = train.fillna(train.mean())
 f_mean = np.mean(train[features[1:]].values,axis=0)
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train[train.weight != 0]
n_folds = 5
seed = 2020
skf = StratifiedKFold(n_splits=n_folds, shuffle=False)
 resp_cols = ['resp', 'resp_1', 'resp_2', 'resp_3', 'resp_4']
X_train = train[train['date'] < 400][features]
X_test = train[train['date'] >= 400][features]
y_train = np.stack([(train[train['date'] < 400][c] > 0 ).astype('int')for c in resp_cols] ).T
y_test = np.stack([(train[train['date'] >= 400][c] > 0 ).astype('int')for c in resp_cols] ).T
X_train = train[features]
y_train = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T<choose_model_class>
head_df = data_all.query('parentesco1 == 1')
 train = head_df.loc[head_df['Target'].notnull() , :]
test = head_df.loc[head_df['Target'].isnull() , :]
TUNNING = False
def create_model(hp,input_dim,output_dim):
inputs = tf.keras.layers.Input(input_dim)
 x = tf.keras.layers.BatchNormalization()(inputs)
 x = tf.keras.layers.GaussianNoise(hp.Choice('noise',[0.0,0.03,0.05]))(x)
 x = tf.keras.layers.Dropout(hp.Choice('init_dropout',[0.0,0.3,0.5]))(x)
 x = tf.keras.layers.Dense(hp.Int('num_units_1', 128, 2048, 64), activation=hp.Choice('activation_1', ['tanh','relu','swish']))(x)
 x = tf.keras.layers.Dropout(hp.Choice(f'dropout_1',[0.0,0.3,0.5]))(x)
 x = tf.keras.layers.Dense(hp.Int('num_units_2', 128, 1024, 32), activation=hp.Choice('activation_2', ['tanh','relu','swish']))(x)
 x = tf.keras.layers.Dropout(hp.Choice(f'dropout_2',[0.0,0.3,0.5]))(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs,outputs=x)
 model.compile(optimizer=tf.keras.optimizers.Adam(hp.Choice('lr',[1e-2, 1e-3, 1e-5])) ,loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=hp.Choice('label_smoothing',[0.0, 0.01, 0.1])) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model
model = tf.keras.Sequential([
tf.keras.Input(shape = len(features)) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.GaussianNoise(0.05),
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, activation='tanh'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(64, activation='tanh'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(256, activation='tanh'),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tf.keras.layers.Dense(5, activation = 'sigmoid')
 ])
 EPOCHS = 500
BATCH_SIZE = 4096
if TUNNING:
EPOCHS = 50
MAX_TRIAL = 20
model_fn = lambda hp: create_model(hp, X_train.shape[-1], y_train.shape[-1])
 tuner = kt.tuners.BayesianOptimization(model_fn, kt.Objective('val_auc', direction='max'), MAX_TRIAL, seed = 2020)
 tuner.search(X_train, y_train, epochs=EPOCHS, validation_data=(X_test, y_test),callbacks=[tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience = 10, restore_best_weights=True)])
 model = tuner.get_best_models() [0]
else:
callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=30, restore_best_weights=True)
 optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
 loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-2)
 model.compile(loss = loss, optimizer=optimizer, metrics=[tf.keras.metrics.AUC() ])
 history = model.fit(X_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS)
<feature_engineering>
features = basic_feats + house_head_feats + material_feats + facility_feats + live_feats + fe_feats + agg_feats
print('
env = janestreet.make_env()
iter_test = env.iter_test()
for(test_df, sample_prediction_df)in tqdm(iter_test):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
action = np.mean(model(x_tt, training = False ).numpy() [0])
 if(action > 0.5):
sample_prediction_df.action = 1
else:
sample_prediction_df.action = 0
else:
sample_prediction_df.action = 0
env.predict(sample_prediction_df )<import_modules>
X, y = train[['idhogar', 'Id'] + features], train['Target']
X_train, X_valid, y_train, y_valid = train_test_split(X[features], y,
test_size=0.1, random_state=0 )
from tensorflow.keras.layers import Input, Dense, BatchNormalization, Dropout, Concatenate, Lambda, GaussianNoise, Activation
from tensorflow.keras.models import Model, Sequential
from tensorflow.keras.losses import BinaryCrossentropy
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau
from tensorflow.keras.layers.experimental.preprocessing import Normalization
import tensorflow as tf
import numpy as np
import pandas as pd
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices
import random
import kerastuner as kt<import_modules>
gbm = lgb.LGBMClassifier(n_jobs=4, random_state=0, class_weight='balanced')
 param_grid = {'num_leaves': np.arange(10, 50, 10),
'learning_rate': np.arange(0.05, 0.2, 0.05),
'n_estimators': np.arange(10, 25, 5)}
scoring = {'f1_macro': get_scorer('f1_macro')}
metric = 'f1_macro'
class PurgedGroupTimeSeriesSplit(_BaseKFold):
@_deprecate_positional_args
def __init__(self, n_splits = 5, *, max_train_group_size=np.inf, max_test_group_size=np.inf, group_gap=None, verbose=False):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def data_split(self, X, y=None, groups=None):
if groups is None:
raise ValueError("The 'groups' should not be void")
 X, y, groups = indexable(X,Y,groups)
 n_sample = n_sample(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_gap = self.max_test_group_gap
max_train_group_gap = self.max_train_group_gap
n_folds = n_splits + 1
group_dictionary = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array] 
<train_on_grid>
gs = GridSearchCV(gbm,
param_grid=param_grid,
scoring=scoring,
cv=5,
refit=metric,
verbose=True,
n_jobs=4, 
)
 gs.fit(X_train, y_train)
 best_estimator = gs.best_estimator_
class CVTuner(kt.engine.tuner.Tuner):
def run_trial(self, trial, X, y, splits, batch_size=64, epochs=5,callbacks=None):
val_losses = []
for train_indices, test_indices in splits:
X_train, X_test = [x[train_indices] for x in X], [x[test_indices] for x in X]
y_train, y_test = [a[train_indices] for a in y], [a[test_indices] for a in y]
if len(X_train)< 2:
X_train = X_train[0]
X_test = X_test[0]
if len(y_train)< 2:
y_train = y_train[0]
y_test = y_test[0]
model = self.hypermodel.build(trial.hyperparameters)
 hist = model.fit(X_train,y_train, validation_data=(X_test,y_test), epochs=epochs, batch_size=batch_size, callbacks=callbacks)
 val_losses.append([hist.history[k][-1] for k in hist.history])
 val_losses = np.asarray(val_losses)
 self.oracle.update_trial(trial.trial_id, {k:np.mean(val_losses[:,i])for i,k in enumerate(hist.history.keys())})
 self.save_model(trial.trial_id, model )<load_from_csv>
from sklearn.metrics import f1_score
TRAINING = True
USE_FINETUNE = True
FOLDS = 5
SEED = 123
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
y_pred = best_estimator.predict(X_valid)
 macro_f1 = f1_score(y_valid, y_pred, average='macro')
 print(macro_f1 )
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.summary()
autoencoder.compile(optimizer=Adam(0.001),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
X_test = test[['Id', 'idhogar'] + features]
y_pred = best_estimator.predict(X_test[features])
 pred_for_heads = pd.DataFrame({'Id': X_test['Id'], 'idhogar': X_test['idhogar'],
'Target': y_pred})
 pred_for_heads['Target'] = pred_for_heads['Target'].apply(int )
def create_model(hp,input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(hp.Float('init_dropout',0.0,0.5))(x)
 for i in range(hp.Int('num_layers',1,3)) :
x = Dense(hp.Int('num_units_{i}',64,256))(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.swish )(x)
 x = Dropout(hp.Float(f'dropout_{i}',0.0,0.5))(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(hp.Float('lr',0.00001,0.1,default=0.001)) ,loss=BinaryCrossentropy(label_smoothing=hp.Float('label_smoothing',0.0,0.1)) ,metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<categorify>
test_all = data_all.loc[data_all['Target'].isnull() , ['Id', 'idhogar']]
submit = pd.merge(test_all, pred_for_heads[['idhogar', 'Target']], how='left' )
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights(f'.. /input/janestreetencoder/encoder.hdf5')
 encoder.trainable = False<compute_test_metric>
sum(submit['Target'].isnull() )
def utility_score_bincount(date, weight, resp, action):
count_i = len(np.unique(date))
Pi = np.bincount(date, weight * resp * action)
 t = np.sum(Pi)/ np.sqrt(np.sum(Pi ** 2)) * np.sqrt(250 / count_i)
 u = np.clip(t, 0, 6)* np.sum(Pi)
 return u<categorify>
submit.fillna(1, inplace=True)
 submit['Target'] = submit['Target'].astype(np.int16)
 submit.head()
<load_from_csv><EOS> 
submit[['Id', 'Target']].to_csv('submisssion.csv', index=False )