Kaggle competition: Categorical Feature Encoding Challenge II
def augment_df(df):
for feature in features:
df[f'sq_{feature}'] = df[feature]**2
df[f'repo_{feature}'] = df[feature].apply(lambda x: 0 if x==0 else 1/x)
 df['min'] = df[features].min(axis=1)
 df['mean'] = df[features].mean(axis=1)
 df['max'] = df[features].max(axis=1)
 df['median'] = df[features].median(axis=1)
 df['std'] = df[features].std(axis=1)
 df['var'] = df[features].var(axis=1)
 df['abs_mean'] = df[features].abs().mean(axis=1)
 df['abs_median'] = df[features].abs().median(axis=1)
 df['abs_std'] = df[features].abs().std(axis=1)
 df['skew'] = df[features].skew(axis=1)
 df['kurt'] = df[features].kurt(axis=1)
 df['sq_kurt'] = df[[f'sq_{feature}' for feature in features]].kurt(axis=1 )<define_variables>
df = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/train.csv", index_col="id")
 df_test = pd.read_csv("/kaggle/input/cat-in-the-dat-ii/test.csv", index_col="id")
 y = df["target"]
D = df.drop(columns="target")
 features = D.columns
test_ids = df_test.index
D_all = pd.concat([D, df_test])
 num_train = len(D)
 print(f"D_all.shape = {D_all.shape}" )
features = features + [f'sq_{feature}' for feature in features] + [f'repo_{feature}' for feature in features]
num_features = len(features )<compute_train_metric>
ord_maps = {
"ord_0": {val: i for i, val in enumerate([1, 2, 3])},
"ord_1": {
val: i
for i, val in enumerate(
 ["Novice", "Contributor", "Expert", "Master", "Grandmaster"] 
)
 },
"ord_2": {
val: i
for i, val in enumerate(
 ["Freezing", "Cold", "Warm", "Hot", "Boiling Hot", "Lava Hot"] 
)
 },
**{col: {val: i for i, val in enumerate(sorted(D_all[col].dropna().unique())) } for col in ["ord_3", "ord_4", "ord_5", "day", "month"]},
}
class roc(Callback):
def on_epoch_begin(self, **kwargs):
self.total = 0
self.batch_count = 0
def on_batch_end(self, last_output, last_target, **kwargs):
preds = F.softmax(last_output, dim=1)
 try:
roc_score = roc_auc_score(to_np(last_target), to_np(preds[:, -1]))
self.total+=roc_score
self.batch_count+=1
except:
pass
def on_epoch_end(self, num_batch, **kwargs):
self.metric = self.total/self.batch_count<train_on_grid>
oh_cols = D_all.columns.difference(ord_maps.keys() - {"day", "month"})
 print(f"OneHot encoding {len(oh_cols)} columns")
 one_hot = pd.get_dummies(
 D_all[oh_cols],
columns=oh_cols,
drop_first=True,
dummy_na=True,
sparse=True,
dtype="int8",
).sparse.to_coo()
def train_and_eval_tabular_learner(train_df, train_features, valid_idx, add_noise=False, lr=0.02, epochs=1, layers=[200, 100], ps=[0.5, 0.2], name='learner'):
data = TabularDataBunch.from_df(path='.', df=train_df, dep_var='target', valid_idx=valid_idx,
cat_names=[], cont_names=train_features, bs=bs, procs=[FillMissing, Normalize], test_df=test_df)
 learn = tabular_learner(data, layers=layers, ps=ps, metrics=[roc() ])
 if add_noise:
learn.data = None
data = None
noise = np.random.normal(summary[features].loc['mean'].values, summary[features].loc['std'].values,(len(train_df), num_features)) /100
train_df[features]+=noise
data = TabularDataBunch.from_df(path='.', df=train_df, dep_var='target', valid_idx=valid_idx,
cat_names=[], cont_names=train_features, bs=bs, procs=[FillMissing, Normalize], test_df=test_df)
 learn.data = data
learn.fit_one_cycle(epochs, lr)
 train_df[features]-=noise
noise = None
learn.fit_one_cycle(epochs, lr)
 learn.save(name, with_opt=False)
 valid_preds, _ = learn.get_preds(ds_type=DatasetType.Valid)
 valid_probs = np.array(valid_preds[:, -1])
 valid_targets = train_df.loc[valid_idx].target.values
valid_score = roc_auc_score(valid_targets, valid_probs)
 test_preds, _ = learn.get_preds(ds_type=DatasetType.Test)
 test_probs = to_np(test_preds[:, -1])
 del(data)
 del(learn)
 gc.collect()
return valid_score, valid_probs, test_probs<define_variables>
ord_cols = pd.concat([D_all[col].map(ord_map ).fillna(max(ord_map.values())//2 ).astype("float32")for col, ord_map in ord_maps.items() ], axis=1)
 ord_cols /= ord_cols.max()
ord_cols_sqr = 4*(ord_cols - 0.5)**2
sub_features = []
valid_scores = []
valid_preds = []
preds = []
num_epochs = 10
cv_counts = len(df)//num_epochs
saved_model_prefix = 'learner'<define_variables>
X = scipy.sparse.hstack([one_hot, ord_cols, ord_cols_sqr] ).tocsr()
print(f"X.shape = {X.shape}")
 X_train, X_test, y_train, y_test = train_test_split(X[:num_train], y, test_size=0.1, random_state=42, shuffle=False)
 X_train = X_train[:10000]
y_train = y_train[:10000]
X_test = X_test[:2000]
y_test = y_test[:2000]
augmented_features = ['min', 'mean', 'max', 'median', 'std', 'abs_mean', 'abs_median', 'abs_std', 'skew', 'kurt', 'sq_kurt']<define_variables>
log = LogisticRegression(C=0.05, solver="lbfgs", max_iter=5000)
 dtree = DecisionTreeClassifier(random_state=4)
 rtree = RandomForestClassifier(n_estimators=100, random_state=4)
 svm = SVC(random_state=4, probability=True)
 nb = GaussianNB()
gbc = GradientBoostingClassifier()
knn = KNeighborsClassifier(n_neighbors=400)
 adaboost = AdaBoostClassifier(n_estimators=500, learning_rate=0.01, random_state=42,
base_estimator=DecisionTreeClassifier(max_depth=8,
min_samples_leaf=10, random_state=42))
labels = [0,1]
def model_algorithm(clf, X_train, y_train, X_test, y_test, name, labels, features):
clf.fit(X_train, y_train)
 y_probas = clf.predict_proba(X_test)
 y_pred = clf.predict(X_test)
 wandb.init(anonymous='allow', project="kaggle-feature-encoding", name=name, reinit=True)
 wandb.termlog('
Plotting %s.'%name)
 wandb.sklearn.plot_learning_curve(clf, X_train, y_train)
 wandb.termlog('Logged learning curve.')
 wandb.sklearn.plot_confusion_matrix(y_test, y_pred, labels)
 wandb.termlog('Logged confusion matrix.')
 wandb.sklearn.plot_summary_metrics(clf, X=X_train, y=y_train, X_test=X_test, y_test=y_test)
 wandb.termlog('Logged summary metrics.')
 wandb.sklearn.plot_class_proportions(y_train, y_test, labels)
 wandb.termlog('Logged class proportions.')
 if(not isinstance(clf, naive_bayes.MultinomialNB)) :
wandb.sklearn.plot_calibration_curve(clf, X_train, y_train, name)
 wandb.termlog('Logged calibration curve.')
 wandb.sklearn.plot_roc(y_test, y_probas, labels)
 wandb.termlog('Logged roc curve.')
 wandb.sklearn.plot_precision_recall(y_test, y_probas, labels)
 wandb.termlog('Logged precision recall curve.')
 csv_name = "submission_"+name+".csv"
bs = 2048<import_modules>
model_algorithm(log, X_train, y_train, X_test, y_test, 'LogisticRegression', labels, features )
import gc<set_options>
model_algorithm(svm, X_train, y_train, X_test, y_test, 'SVM', labels, features )
gc.collect()<import_modules>
model_algorithm(knn, X_train, y_train, X_test, y_test, 'KNearestNeighbor', labels, features )
from sklearn.metrics import roc_auc_score<train_model>
model_algorithm(adaboost, X_train, y_train, X_test, y_test, 'AdaBoost', labels, features )
for i in range(num_epochs):
print('Training model: ', i)
 sub_features.append(random.sample(list(features), int(num_features*0.75)) + augmented_features)
 name = f'{saved_model_prefix}_{i}'
score, valid_probs, test_probs = train_and_eval_tabular_learner(df, sub_features[-1], valid_idx,
add_noise=True, epochs=3, lr=0.02, name=name)
 valid_scores.append(score)
 valid_preds.append(valid_probs)
 preds.append(test_probs )<import_modules>
model_algorithm(gbc, X_train, y_train, X_test, y_test, 'GradientBoosting', labels, features )
import lightgbm as lgb
from sklearn.model_selection import KFold, StratifiedKFold<define_variables>
model_algorithm(dtree, X_train, y_train, X_test, y_test, 'DecisionTree', labels, None )
target = df['target']<import_modules>
model_algorithm(rtree, X_train, y_train, X_test, y_test, 'RandomForest', labels, features )
<init_hyperparams><EOS> 
clf=LogisticRegression(C=0.05, solver="lbfgs", max_iter=5000)
 clf.fit(X_train, y_train)
 pred = clf.predict_proba(X_test)[:, 1]
pd.DataFrame({"id": test_ids, "target": pred} ).to_csv("submission_lr.csv", index=False )