Kaggle competition: Categorical Feature Encoding Challenge
op = multi_baseweek_frame(
 augment_train_cosine(train, display_sample=False, n_similar=5, threshold=.5)
 )<prepare_x_and_y>
train_df = pd.read_csv('/kaggle/input/cat-in-the-dat/train.csv')
 test_df = pd.read_csv('/kaggle/input/cat-in-the-dat/test.csv')
 sample_sub_df = pd.read_csv('/kaggle/input/cat-in-the-dat/sample_submission.csv' )
cat_cols = ['Sex', 'SmokingStatus']
to_drop = ['FVC', 'Percent', 'Weeks', 'factor', 'Base_Percent']
num_cols = ['Weeks', 'Week_Offset', 'Base_Week', 'Age', 'Base_FVC', 'Percent', 'Base_Percent']
cat_method = 'ord'
math = []
age_bins = 5
folds = 7
total_patients = train.Patient.unique()
np.random.shuffle(total_patients)
 val_len = len(total_patients)// folds
temp = pd.DataFrame()
preds = pd.DataFrame()
X, Y = get_model_data(
 op, factor=True, num_cols=num_cols, cat_cols=cat_cols,
age_bins=age_bins, to_drop=to_drop, display_stats=False,
cat_method=cat_method, transform_stats=stats, math=math)
 X_VAL = base_shift(train, q=0)
 Y_VAL = X_VAL['FVC'].dropna()
X_VAL['Percent'] = X_VAL['Base_Percent']
X_VAL = get_model_data(
 X_VAL, num_cols=num_cols, cat_cols=cat_cols, to_drop=to_drop, factor=True,
cat_method=cat_method, train=False, age_bins=age_bins, math=math)
 x_test = sub[['Patient', 'Weeks']].merge(
 test.rename({"Weeks": "Base_Week",
"FVC": "Base_FVC",
"Percent": "Base_Percent"}, axis=1),
on='Patient')
 x_test['Week_Offset'] = x_test['Weeks'] - x_test['Base_Week']
x_test['Percent'] = x_test['Base_Percent']
x_test = get_model_data(
 x_test, cat_cols=cat_cols, num_cols=num_cols, to_drop=to_drop, math=math,
factor=True, train=False, cat_method=cat_method,
age_bins=age_bins ).drop("Patient", 1)
 for i in range(folds):
val_patients = total_patients[(i)*val_len:(i+1)*val_len]
train_patients = np.setdiff1d(total_patients, val_patients)
 assert len(np.intersect1d(val_patients, train_patients)) == 0
x, y, =(X[X.Patient.isin(train_patients)].drop("Patient", 1), Y[X.Patient.isin(train_patients)])
 x_val, y_val =(X_VAL[X_VAL.Patient.isin(val_patients)].drop("Patient", 1),
Y_VAL[X_VAL.Patient.isin(val_patients)])
 model = GBR(alpha=0.75)
 model.fit(x, y)
 y_middle, y_upper, y_lower = model.predict_forecast(x_val, return_bounds=True)
 y_middle_pred, y_test_conf = model.predict_forecast(x_test)
 print("For Fold
i+1, - laplace_log_likelihood(y_middle, y_val),
- laplace_log_likelihood(y_middle, y_val, y_upper - y_lower)))
 temp = temp.append(pd.DataFrame(
 data=np.stack([y_upper, y_lower, y_middle, y_val], axis=1),
columns=['upper', 'lower', 'pred', 'actual'] 
))
preds = preds.append(pd.DataFrame(
 data=np.stack([y_middle_pred, y_test_conf], axis=1)/ folds,
columns=['pred', 'Confidence'] 
))
preds = preds.groupby(preds.index ).sum()
temp['Confidence'] = temp['upper'] - temp['lower']<compute_train_metric>
Y_train = train_df['target']
X_train = train_df.drop(['target', 'id'], axis=1)
 X_test = test_df.drop(['id'], axis=1 )
pat_scores =(
 temp.reset_index(drop=True)
.groupby(train.Patient)
.apply(lambda x: -laplace_log_likelihood(x['actual'], x['pred'], x['Confidence'] ).numpy())
 ).rename('scores' ).reset_index().sort_values("scores", ascending=False)
 pat_scores = pat_scores.head(8)
 print("Worst Patient-Mean-Score: {:.2f}".format(pat_scores.scores.mean()))
pat_scores = pat_scores.Patient.values<compute_train_metric>
%%time
combined_data = pd.concat([X_train, X_test], axis=0, sort=False)
 combined_data = pd.get_dummies(combined_data, columns=combined_data.columns, drop_first=True, sparse=True)
 X_train = combined_data.iloc[: len(train_df)]
X_test = combined_data.iloc[ len(train_df): ]
print("
|================== Summary ==================|
\
Score on Total Dataset: {:.3f} @ 70 Confidence
\
Score on Total Dataset: {:.3f} @ 225 Confidence
\
Score on Total Dataset: {:.3f} @ Pred Confidence".format(
 -laplace_log_likelihood(temp['actual'], temp['pred'], 70),
-laplace_log_likelihood(temp['actual'], temp['pred'], 225),
-laplace_log_likelihood(temp['actual'], temp['pred'], temp['Confidence'])
))
f, ax = plt.subplots(figsize=(40, 40), nrows=4, ncols=2)
 ax = ax.ravel()
for i, pat in enumerate(pat_scores): 
(temp.reset_index(drop=True ).loc[train.Patient == pat] 
.drop(["Confidence"], 1 ).plot(ax=ax[i], legend=False))
f.suptitle("Model's Worst Predictions", size=30)
 f.tight_layout(rect=[0, 0.03, 1, 0.95]);<save_to_csv>
X_train = X_train.sparse.to_coo().tocsr()
X_test = X_test.sparse.to_coo().tocsr()
sub['FVC'] = preds['pred']
sub['Confidence'] = preds['Confidence']
for i in range(len(test)) :
sub.loc[sub['Patient_Week']==test.Patient[i]+'_'+str(test.Weeks[i]), 'FVC'] = test.FVC[i]
sub.loc[sub['Patient_Week']==test.Patient[i]+'_'+str(test.Weeks[i]), 'Confidence'] = 70
sub[['Patient_Week', 'FVC', 'Confidence']].to_csv("quant_submission.csv", index=False)
 sub.head()<count_unique_values>
class ModelHelper(object):
def __init__(self, params, model):
self.model = model(**params)
 def train(self, X_train, Y_train):
self.model.fit(X_train, Y_train)
 def predict(self, X_test):
return self.model.predict_proba(X_test)[:, 1]
def evaluate(self, Y_true, Y_preds):
return roc_auc_score(Y_true, Y_preds )
cat_cols = ['Sex', 'SmokingStatus']
num_cols = ['Weeks', 'Week_Offset', 'Base_Week', 'Age', 'Base_FVC', 'Percent', 'Base_Percent']
to_drop = ["FVC", 'Percent', 'Weeks', 'Base_Week', 'Age', 'Base_Percent', 'factor']
math = []
multi_method = {}
multi_data = {}
folds = 7
methods = ['cubic', 'quadratic', 'cubicspline',
'pchip', 'akima', 'nearest', 'zero',
'slinear', 'linear', 'SIMILARITY_AUG']
total_patients = train.Patient.unique()
val_len = len(total_patients)// folds
for method in methods:
if method == 'SIMILARITY_AUG':
temp = augment_train_cosine(train, n_similar=3, threshold=0.25, display_sample=False)
 else:
temp = augment_train_naive(data=train, method=method, steps=15, noise=0., display_sample=False)
 temp = multi_baseweek_frame(temp, display=False)
 multi_data[method] = temp
X, Y = get_model_data(temp, num_cols=num_cols, cat_cols=cat_cols, to_drop=to_drop,
display_stats=False, cat_method='1h', math=math, factor=True)
 X_VAL = base_shift(train, q=0)
 Y_VAL = X_VAL['FVC'].dropna()
X_VAL['Percent'] = X_VAL['Base_Percent']
X_VAL = get_model_data(
 X_VAL, num_cols=num_cols, cat_cols=cat_cols, factor=True,
to_drop=to_drop, train=False, cat_method='1h', math=math)
 np.random.shuffle(total_patients)
 scores = {}
for i in range(folds):
val_patients = total_patients[(i)*val_len:(i+1)*val_len]
train_patients = np.setdiff1d(total_patients, val_patients)
 x, y = X[X.Patient.isin(train_patients)], Y[X.Patient.isin(train_patients)]
x_val, y_val = X_VAL[X_VAL.Patient.isin(val_patients)], Y_VAL[X_VAL.Patient.isin(val_patients)]
assert len(np.intersect1d(x_val.Patient.unique() , x.Patient.unique())) == 0
scores['Train'] = scores.get('Train', [])+ [cross_val_score(
 GBR() , x.drop("Patient", 1), y,
scoring=l1(70), cv=GroupKFold(5), groups=x.Patient ).mean() ]
lr = GBR().fit(x.drop("Patient", 1), y)
 temp = lr.predict_forecast(x_val.drop("Patient", 1))
temp = pd.DataFrame(np.stack(temp, 1), columns=['pred', 'conf'])
 temp['actual'] = y_val.reset_index(drop=True)
 scores['Val'] = scores.get('Val', [])+ [-laplace_log_likelihood(
 temp['actual'], temp['pred'], 70
).numpy() ]
scores['ValC'] = scores.get('ValC', [])+ [-laplace_log_likelihood(
 temp['actual'], temp['pred'], temp['conf']
).numpy() ]
scores['ValW'] = scores.get('ValW', [])+ [temp.apply(lambda x: -laplace_log_likelihood(
 x['actual'], x['pred'], x['conf'] ).numpy() , 1 ).nlargest(25 ).mean() ]
print("Method: {}
Train score: {:5.2f} @ {:.2f} Variance \
Val Score: {:7.2f} @ {:.2f} Variance\
ValC Score: {:6.2f} @ {:.2f} Variance\
Worst Score: {:5.2f} @ {:.2f} Variance
{}
".format(
 method.upper() , np.mean(scores['Train']), np.std(scores['Train']),
np.mean(scores['Val']), np.std(scores['Val']), np.mean(scores['ValC']),
np.std(scores['ValC']), np.mean(scores['ValW']), np.std(scores['ValW']), "=" * 35 
))
multi_method[method] = scores<compute_test_metric>
SPLITS = 5
kfold = KFold(n_splits=SPLITS, shuffle=True, random_state=666)
 lr_params = {
'verbose': 100,
'max_iter': 250,
'C': 0.1,
'solver': 'lbfgs'
}
model_helper = ModelHelper(lr_params, LogisticRegression)
 scores = []
predictions = np.zeros(( SPLITS, X_test.shape[0]))
for i,(train_index, test_index)in enumerate(kfold.split(X_train)) :
X_dev, X_val = X_train[train_index], X_train[test_index]
Y_dev,Y_val = Y_train[train_index], Y_train[test_index]
model_helper.train(X_dev, Y_dev)
 preds = model_helper.predict(X_val)
 roc_score = model_helper.evaluate(Y_val, preds)
 full_test_preds = model_helper.predict(X_test)
 scores.append(roc_score)
 predictions[i, :] = full_test_preds
print(scores)
 print(predictions )
val_scores = [np.mean(multi_method[i]['ValW'])for i in methods]
cut_off = np.percentile(val_scores, 50)
 print("At {:.1f} cutoff, the Worst Score would be {:.3f} @ Pred Conf".format(
 cut_off, np.mean(list(filter(lambda x: x < cut_off, val_scores)))) )<feature_engineering>
sample_sub_df['target'] = predictions[np.argmax(scores)]
sample_sub_df.to_csv('submission.csv', index=False)
 sample_sub_df
x_test = sub[['Patient', 'Weeks']].merge(
 test.rename({"Weeks": "Base_Week",
"FVC": "Base_FVC",
"Percent": "Base_Percent"}, axis=1),
on='Patient')
 x_test['Week_Offset'] = x_test['Weeks'] - x_test['Base_Week']
x_test['Percent'] = x_test['Base_Percent']
preds = {i: None for i in methods}
for method in methods:
if np.mean(multi_method[method]['ValW'])>= cut_off:
preds.pop(method)
 continue
x, y = get_model_data(
 multi_data[method], num_cols=num_cols, cat_cols=cat_cols,
to_drop=to_drop, train=True, cat_method='1h',
display_stats=False, factor=True, 
)
 lr = GBR().fit(x.drop("Patient", 1), y)
 preds[method] = lr.predict_forecast(get_model_data(
 x_test, cat_cols=cat_cols, num_cols=num_cols, to_drop=to_drop, train=False, factor=True,
).drop("Patient", 1), return_bounds=True)
 preds.keys()<feature_engineering>
FileLink('submission.csv' )