Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<define_variables>
sns.set(font_scale=1.4)
 warnings.filterwarnings("ignore")
num_cols_to_shrink_all = detect_num_cols_to_shrink(num_all_cols, all_data)
 convert_to_int8 = num_cols_to_shrink_all[0]
convert_to_int16 = num_cols_to_shrink_all[1]
convert_to_int32 = num_cols_to_shrink_all[2]
convert_to_float16 = num_cols_to_shrink_all[3]
convert_to_float32 = num_cols_to_shrink_all[4]<data_type_conversions>
def root_mean_squared_error(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_pred - y_true)) )
print("starting with converting process.... ")
 for col in convert_to_int16:
all_data[col] = all_data[col].astype('int16')
 for col in convert_to_int32:
all_data[col] = all_data[col].astype('int32')
 for col in convert_to_float16:
all_data[col] = all_data[col].astype('float16')
 for col in convert_to_float32:
all_data[col] = all_data[col].astype('float32')
 print("successfully converted!" )<define_variables>
def add_pca(train_df, test_df, cols, n_comp=20, fit_test = True, prefix='pca_', fit_test_first=False):
pca = PCA(n_components=n_comp, random_state=42)
 pca_titles = [prefix+'_pca_'+str(x)for x in range(n_comp)]
temp_train = train_df.copy()
temp_test = test_df.copy()
for c in cols:
fv = temp_train[c].mean()
temp_train[c] = temp_train[c].fillna(value=fv)
 fv = temp_test[c].mean()
temp_test[c] = temp_test[c].fillna(value=fv)
 for p in pca_titles:
train_df[p] = 0.0
test_df[p] = 0.0
if fit_test==True:
pca_data = pd.concat([temp_train[cols], temp_test[cols]], axis=0)
 if fit_test_first==True:
pca.fit(pca_data[len(train_df):])
 pca_data = pca.transform(pca_data)
 else:
pca_data = pca.fit_transform(pca_data)
 train_df.loc[:, pca_titles] = pca_data[0:len(train_df)]
test_df.loc[:, pca_titles] = pca_data[len(train_df):]
return train_df, test_df, pca_titles
def split_distributions(train_df, test_df, cols, target, n_comp=20, prefix='gm', add_labels=False):
gm = GaussianMixture(n_components=n_comp,covariance_type='full', tol=0.001, reg_covar=1e-06, max_iter=100, n_init=1,
init_params='kmeans', weights_init=None, means_init=None, precisions_init=None,
random_state=42, warm_start=False, verbose=0, verbose_interval=10)
 temp_train = train_df.copy()
temp_test = test_df.copy()
gm_data = pd.concat([temp_train[[cols]], temp_test[[cols]]], axis=0 ).reset_index(drop=True)
 gm.fit(gm_data)
 gm_data['labels'] = gm.predict(gm_data)
 if add_labels:
gm_titles = [prefix+str(x)for x in range(n_comp)]
train_df[gm_titles]=0
test_df[gm_titles]=0
dummies = pd.get_dummies(gm_data['labels'], prefix=prefix)
 train_df.loc[:, gm_titles] = dummies[0:len(train_df)].values
test_df.loc[:, gm_titles] = dummies[len(train_df):].values
else:
gm_titles = []
train_df[prefix+'_label'] = gm.labels_[0:len(train_df)]
test_df[prefix+'_label'] = gm.labels_[len(train_df):]
means = train_df.groupby([prefix+'_label'])[target].mean()
train_df['menc_'+prefix+'_label'] = train_df[prefix+'_label'].map(means)
 test_df['menc_'+prefix+'_label'] = test_df[prefix+'_label'].map(means)
 gm_titles+=[prefix+'_label', 'menc_'+prefix+'_label']
return train_df, test_df, gm_titles
def assign_fold(df, label_column,fold_column, NFOLDS=5):
skf = StratifiedKFold(n_splits=NFOLDS)
 df[fold_column]=0
f=0
for trn_idx, val_idx in skf.split(df, df[label_column]):
df.loc[val_idx, 'fold']=f
f+=1
df[fold_column].value_counts()
return df
def st_scale(train_df, test_df, cols):
StSc = StandardScaler()
train_df[cols] = StSc.fit_transform(train_df[cols])
 test_df[cols] = StSc.transform(test_df[cols])
 return train_df, test_df
def seed_everything(seed=1234):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed )
v = [1, 3, 4, 6, 8, 11]
v += [13, 14, 17, 20, 23, 26, 27, 30]
v += [36, 37, 40, 41, 44, 47, 48]
v += [54, 56, 59, 62, 65, 67, 68, 70]
v += [76, 78, 80, 82, 86, 88, 89, 91]
v += [96, 98, 99, 104]
v += [107, 108, 111, 115, 117, 120, 121, 123]
v += [124, 127, 129, 130, 136]
v += [138, 139, 142, 147, 156, 162]
v += [165, 160, 166]
v += [178, 176, 173, 182]
v += [187, 203, 205, 207, 215]
v += [169, 171, 175, 180, 185, 188, 198, 210, 209]
v += [218, 223, 224, 226, 228, 229, 235]
v += [240, 258, 257, 253, 252, 260, 261]
v += [264, 266, 267, 274, 277]
v += [220, 221, 234, 238, 250, 271]
v += [294, 284, 285, 286, 291, 297]
v += [303, 305, 307, 309, 310, 320]
v += [281, 283, 289, 296, 301, 314]
v += [332, 325, 335, 338]<drop_column>
PATH = '/kaggle/input/tabular-playground-series-jan-2021/'
train = pd.read_csv(PATH+'train.csv')
 test = pd.read_csv(PATH+'test.csv')
 submission = pd.read_csv(PATH+'sample_submission.csv')
 FT_COLS = [x for x in train.columns if 'cont' in x]
TARGET='target'
print(train.shape)
 train.head(10 )
cols = ['V'+str(x)for x in v]
for i in all_data_cols:
if(i.startswith("V")) and i not in cols:
all_data = all_data.drop(columns=[i])
 all_data<drop_column>
original_feature_dict = {'cont1': 5,
'cont2': 10,
'cont3': 8,
'cont4': 8,
'cont5': 5,
'cont6': 6,
'cont7': 4,
'cont8': 3,
'cont9': 7,
'cont10':3,
'cont11': 4,
'cont12': 2,
'cont13': 3,
'cont14': 5,}
pca_dict = {
'extra_pca_0': 4,
'extra_pca_1': 1,
'extra_pca_2': 1,
'extra_pca_3': 2,
'extra_pca_4': 1,
'extra_pca_5': 1 }
MULTIPLIER = 3.0
original_feature_dict = {a:int(b*MULTIPLIER)for(a,b)in original_feature_dict.items() }
ft_dict = {**original_feature_dict, **pca_dict}
ft_dict
for c in ['C3','M5','id_08','id_33']:
cols.remove(c)
 all_data = all_data.drop(columns=[i])
 for c in ['card4','id_07','id_14','id_21','id_30','id_32','id_34']:
cols.remove(c)
 all_data = all_data.drop(columns=[i])
 for c in ['id_'+str(x)for x in range(22,28)]:
cols.remove(c)
 all_data = all_data.drop(columns=[i])
 for c in ['D6','D7','D8','D9','D12','D13','D14']:
cols.remove(c)
 all_data = all_data.drop(columns=[i] )<define_variables>
mixture_title_cols=[]
for f in FT_COLS+pca_titles2:
train, test, titles = split_distributions(train, test,f,TARGET, n_comp=ft_dict[f], prefix=f+'_dim', add_labels=True)
 mixture_title_cols+=titles
mixture_value_cols = []
for count,f in enumerate(FT_COLS+pca_titles2):
for d in range(ft_dict[f]):
t_median = train[f][train[f+'_dim'+str(d)]==1].median()
t_std = train[f][train[f+'_dim'+str(d)]==1].std()
train[f+'_dim'+str(d)+'_dist'] = np.where(train[f+'_dim'+str(d)]==1, 
(train[f] - t_median)/t_std, np.nan)
 test[f+'_dim'+str(d)+'_dist'] = np.where(test[f+'_dim'+str(d)]==1, 
(test[f] - t_median)/t_std, np.nan)
 mixture_value_cols+=[f+'_dim'+str(d)+'_dist']
numerical = all_data.select_dtypes(include='number')
 categorical = all_data.select_dtypes(exclude = 'number')
 numeric_transformer = Pipeline(steps=[('mean',SimpleImputer(strategy='constant',fill_value=-1)) ])
 categorical_transformer = Pipeline(steps=[('constant', SimpleImputer(strategy='constant',fill_value=-1)) ] )<split>
print('Total Original Features', len(FT_COLS))
print('Total Submixtures Labels', len(mixture_title_cols))
print('Total Submixtures Values', len(mixture_value_cols))
print('Total Feature Columns', len(FT_COLS)+len(mixture_title_cols)+len(mixture_value_cols))
final_train = all_data[ : ntrain]
final_test = all_data[ntrain : ]<drop_column>
NAN_VALUE = 0.0
for d in mixture_value_cols:
train[d] = train[d].fillna(value=NAN_VALUE)
 test[d] = test[d].fillna(value=NAN_VALUE )
del all_data<data_type_conversions>
SCALE = True
if SCALE:
train, test = st_scale(train, test, FT_COLS)
 for f in FT_COLS:
train[f] = np.clip(train[f], -2, 2)
 test[f] = np.clip(test[f], -2, 2)
 SCALE_DISTS = True
if SCALE_DISTS:
for d in mixture_value_cols:
TEMP_MAX = np.abs(train[d] ).max()
train[d] = train[d] / TEMP_MAX
test[d] = test[d] / TEMP_MAX
for i,f in enumerate(final_train.columns):
if(np.str(final_train[f].dtype)=='category')|(final_train[f].dtype=='object'):
df_comb = pd.concat([final_train[f],final_test[f]],axis=0)
 df_comb,_ = df_comb.factorize(sort=True)
 if df_comb.max() >32000: print(f,'needs int32')
 final_train[f] = df_comb[:len(final_train)].astype('int16')
 final_test[f] = df_comb[len(final_train):].astype('int16')
 elif f not in ['TransactionAmt','TransactionDT']:
mn = np.min(( final_train[f].min() ,final_test[f].min()))
final_train[f] -= np.float32(mn)
 final_test[f] -= np.float32(mn)
 final_train[f].fillna(-999,inplace=True)
 final_test[f].fillna(-999,inplace=True )<count_values>
train['outlier_filter'] = np.where(train[TARGET]<4, True, False)
 print('
y.value_counts()<choose_model_class>
print('CV Benchmark - Random Guess of Median')
 cv_bm = np.sqrt(mse(train[TARGET], np.full(( train[TARGET].shape), train[TARGET].median())))
 cv_bm
model = xgb.XGBClassifier(n_estimators=2000,
max_depth=12,
learning_rate=0.02,
subsample=0.8,
colsample_bytree=0.4,
missing=-1,
eval_metric='auc',
tree_method='hist'
)<prepare_x_and_y>
def run_training(model, train_df, test_df,sample_submission, fold_col,
orig_features, mixture_val_cols,mixture_label_cols, target_col,
benchmark,
outlier_col=None, nn=False, epochs=10,batch_size=32, verbose=False,
dense=70, dout=0.15, dense_reg = 0.000001,act='elu'):
FOLD_VALUES = sorted([x for x in train_df[fold_col].unique() ])
 oof = np.zeros(( len(train_df),))
test_predictions = np.zeros(( len(sample_submission),))
fig,axes=plt.subplots(figsize=(15,6))
axes.set_ylim(0.69,0.75)
 RANDOM_SEEDS = [0,42,100,1000]
for rs in RANDOM_SEEDS:
seed_everything(seed=rs)
 for fold in FOLD_VALUES:
print(' --- ')
 print('running random seed', rs, 'fold', fold)
 if outlier_col:
trn_idx =(train_df[fold_col]!=fold)&(~train_df[outlier_col])
 else:
trn_idx = train_df[fold_col]!=fold
val_idx = train_df[fold_col]==fold
X_train_orig = train_df.loc[trn_idx,orig_features].values
X_train = train_df.loc[trn_idx,mixture_val_cols].values
X_train_mask = train_df.loc[trn_idx,mixture_label_cols].values
y_train = train_df.loc[trn_idx, target_col].values
X_val_orig = train_df.loc[val_idx,orig_features].values
X_val = train_df.loc[val_idx,mixture_val_cols].values
X_val_mask = train_df.loc[val_idx,mixture_label_cols].values
y_val = train_df.loc[val_idx, target_col].values
model = keras_model(orig_features, mixture_val_cols,mixture_label_cols,dout=dout,
dense=dense,act=act, dense_reg = dense_reg, descend_fraction = 0.9)
 K.clear_session()
back_count = 5
epoch_list=list(range(epochs-back_count, epochs))
class CustomModelCheckpoint(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
if epoch in epoch_list:
self.model.save_weights('model_fold_'+str(fold)+'_epoch_'+str(epoch)+'.h5', overwrite=True)
 else:
pass
cbk = CustomModelCheckpoint()
history = model.fit([X_train_orig, X_train,X_train_mask ], y_train, epochs=epochs, batch_size=batch_size,shuffle=True,
validation_data=([X_val_orig, X_val,X_val_mask], y_val), verbose=verbose,
callbacks=[cbk])
 print('Fold Last Epoch Train Error', history.history['root_mean_squared_error'][-1])
 print('Fold Last Epoch Valid Error', history.history['val_root_mean_squared_error'][-1])
 model.save_weights('model_final_'+str(fold)+'.h5')
 sns.lineplot(x=range(epochs), y=history.history['loss'], color='Blue')
 sns.lineplot(x=range(epochs), y=history.history['val_loss'], color='Red')
 val_preds = np.zeros(( len(X_val),))
for e in epoch_list:
model = keras_model(orig_features, mixture_val_cols,mixture_label_cols,dout=dout,
dense=dense,act=act, dense_reg = dense_reg, descend_fraction = 0.9)
 model.load_weights('model_fold_'+str(fold)+'_epoch_'+str(e)+'.h5')
 val_preds += model.predict([X_val_orig, X_val,X_val_mask] ).mean(axis=1)
 test_predictions += model.predict([test_df[orig_features].values,test_df[mixture_val_cols].values,
test_df[mixture_label_cols].values] ).mean(axis=1)
 val_preds = val_preds / len(epoch_list)
 oof[val_idx]+=val_preds
val_error = np.sqrt(mse(y_val, val_preds))
print('Fold multi epoch weight prediction error', val_error)
 oof = oof / len(RANDOM_SEEDS)
 total_val_error = np.sqrt(mse(train_df[target_col], oof))
print('final OOF MSE', total_val_error)
 fold_errors = []
for fold in train_df[fold_col].unique() :
val_idx = train_df[fold_col]==fold
fold_errors+=[np.sqrt(mse(train_df.loc[val_idx, target_col].values, oof[val_idx])) ]
test_predictions = test_predictions/(NFOLDS * len(RANDOM_SEEDS)* len(epoch_list))
return oof, test_predictions, fold_errors
X_train = final_train
y_train = y
X_test = final_test<drop_column>
def keras_model(ft_orig, mixture_values, mixture_labels, n_layer=3,bnorm=True,dout=0.2,
dense=20,act='elu', dense_reg = 0.000001, descend_fraction = 0.9):
input1 = L.Input(shape=(len(ft_orig)) , name='input_orig')
 input1_do = L.Dropout(0.1 )(input1)
 XA = L.Dense(dense, activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg), name='dense_orig' )(input1_do)
 if bnorm:
XA = L.BatchNormalization(name='bn_1' )(XA)
 XA1 = L.Dropout(dout, name='do_orig' )(XA)
 input2 = L.Input(shape=(len(mixture_values)) , name='input_new')
 input3 = L.Input(shape=(len(mixture_labels)) , name='input_new2')
 all_input_combo = tf.keras.layers.Concatenate(axis=1, name='concat_i2' )([input2, input3])
 input2_do = L.Dropout(0.2 )(all_input_combo)
 XB = L.Dense(dense, activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg), name='dense_new' )(input2_do)
 if bnorm:
XB = L.BatchNormalization(name='bn_2' )(XB)
 XB1 = L.Dropout(dout, name='do_new' )(XB)
 all_input_combo = tf.keras.layers.Concatenate(axis=1, name='concat_new' )([XA1, XB1])
 X = L.Dense(int(dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg))(all_input_combo)
 if bnorm:
X = L.BatchNormalization(name='bn_3' )(X)
 X2 = L.Dropout(dout, name='do_3' )(X)
 X2 = tf.keras.layers.concatenate([all_input_combo, X2], axis=1)
 X = L.Dense(int(( descend_fraction**2)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg))(X2)
 if bnorm:
X = L.BatchNormalization()(X)
 X3 = L.Dropout(dout )(X)
 X3 = tf.keras.layers.concatenate([X2, X3], axis=1)
 X = L.Dense(int(( descend_fraction**3)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg *2))(X3)
 if bnorm:
X = L.BatchNormalization()(X)
 X4 = L.Dropout(dout )(X)
 X4 = tf.keras.layers.concatenate([X3, X4],axis=1)
 X = L.Dense(int(( descend_fraction**4)*dense), activation=act, activity_regularizer=tf.keras.regularizers.L2(dense_reg*4))(X4)
 if bnorm:
X = L.BatchNormalization()(X)
 X5 = L.Dropout(dout )(X)
 X5 = tf.keras.layers.concatenate([X4, X5], axis=1)
 X = L.Dense(5, activation=act, activity_regularizer=tf.keras.regularizers.L2(0.0001))(X5)
 output1 = L.Dense(1, activation='linear' )(X)
 model = tf.keras.Model(inputs=[input1, input2, input3], outputs=output1)
 model.compile(loss=root_mean_squared_error,
optimizer=tfa.optimizers.AdamW(learning_rate=0.000809028893821181, weight_decay=9.83479875802558E-06),
metrics=tf.keras.metrics.RootMeanSquaredError())
 return model
K.clear_session()
model = keras_model(FT_COLS, mixture_title_cols, mixture_value_cols,
dense=70,
dout=0.15,
dense_reg = 0.000001,
act='elu',
descend_fraction = 0.9)
 model.summary()
del final_train
del y
del final_test<drop_column>
oof, test_predictions, fold_errors = run_training(model, train, test,submission, 'fold', FT_COLS,
mixture_value_cols, mixture_title_cols, TARGET, cv_bm,
outlier_col='outlier_filter',
epochs=25,
batch_size=256,
verbose=False,
dense=70,
dout=0.15,
dense_reg = 0.000001,
act='elu', )
final_col = X_train['TransactionID']
X_train = X_train.drop(columns=['TransactionID'] )<feature_engineering>
print('Save Out of Fold Predictions')
 oof = pd.DataFrame(columns=['oof_prediction'], index=train['id'], data=oof + TARGET_MEAN)
 oof.to_csv('oof_predictions.csv', index=True)
 oof.head(10 )
START_DATE = datetime.datetime.strptime('2017-11-30', '%Y-%m-%d')
 X_train['DT_M'] = X_train['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_train['DT_M'] =(X_train['DT_M'].dt.year-2017)*12 + X_train['DT_M'].dt.month
X_test['DT_M'] = X_test['TransactionDT'].apply(lambda x:(START_DATE + datetime.timedelta(seconds = x)))
 X_test['DT_M'] =(X_test['DT_M'].dt.year-2017)*12 + X_test['DT_M'].dt.month<drop_column>
print('fold errors', fold_errors)
 print('fold error std', np.array(fold_errors ).std() )
cols = list(X_train.columns)
 cols.remove('TransactionDT' )<define_variables>
submission['target'] = test_predictions + TARGET_MEAN
submission.to_csv('submission.csv', index=False)
 submission.head(5 )