Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<load_from_csv>
from fastai.tabular import *
train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 test.insert(1, 'target', 0 )<categorify>
sample_submission = pd.read_csv(".. /input/cat-in-the-dat-ii/sample_submission.csv")
 X_test = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv" )
features = [_f for _f in train if _f not in ['id', 'target']]
def factor_encoding(train, test):
assert sorted(train.columns)== sorted(test.columns)
 full = pd.concat([train, test], axis=0, sort=False)
 for f in full:
full[f], _ = pd.factorize(full[f])
 full[f] += 1
return full.iloc[:train.shape[0]], full.iloc[train.shape[0]:]
train_f, test_f = factor_encoding(train[features], test[features] )<categorify>
procs = [FillMissing, Categorify, Normalize]
FillMissing.FillStrategy='MEAN'
dep_var = 'target'
cat_names = ['bin_0', 'bin_1', 'bin_2', 'bin_3', 'bin_4', 'nom_0', 'nom_1', 'nom_2',
'nom_3', 'nom_4', 'nom_5', 'nom_6', 'nom_7', 'nom_8', 'nom_9',
'ord_0', 'ord_1', 'ord_2', 'ord_3', 'ord_4', 'ord_5', 'day', 'month']
cont_names = []
X=train[cont_names]
X=train[cat_names]
class LibFFMEncoder(object):
def __init__(self):
self.encoder = 1
self.encoding = {}
def encode_for_libffm(self, row):
txt = f"{row[0]}"
for i, r in enumerate(row[1:]):
try:
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
except KeyError:
self.encoding[(i, r)] = self.encoder
self.encoder += 1
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
return txt
fold_ids = [
[trn_, val_] for(trn_, val_)in StratifiedKFold(N_Splits,True,SEED ).split(train, train['target'])
 ]
for fold_,(trn_, val_)in enumerate(fold_ids):
encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'].iloc[trn_], train_f.iloc[trn_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_val = pd.concat([train['target'].iloc[val_], train_f.iloc[val_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_tst = pd.concat([test['target'], test_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 print(train['target'].iloc[trn_].shape, train['target'].iloc[val_].shape, libffm_format_tst.shape)
 libffm_format_trn.to_csv(f'libffm_trn_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_val.to_csv(f'libffm_val_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_tst.to_csv(f'libffm_tst_fold_{fold_+1}.txt', index=False, header=False)
<load_pretrained>
PATH = Path('/kaggle/input/cat-in-the-dat-ii/')
 test = TabularList.from_df(X_test, path=PATH, cat_names=cat_names, cont_names=cont_names )
!./ffm-train -p libffm_val_fold_1.txt -r 0.05 -l 0.0002 -k 50 --auto-stop libffm_trn_fold_1.txt libffm_fold_1_model
!./ffm-predict libffm_val_fold_1.txt libffm_fold_1_model val_preds_fold_1.txt
!./ffm-predict libffm_tst_fold_1.txt libffm_fold_1_model tst_preds_fold_1.txt
os.remove('libffm_val_fold_1.txt')
 os.remove('libffm_trn_fold_1.txt')
 os.remove('libffm_fold_1_model')
 os.remove('libffm_tst_fold_1.txt')
(
 log_loss(train['target'].iloc[fold_ids[0][1]], pd.read_csv('val_preds_fold_1.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[0][1]], pd.read_csv('val_preds_fold_1.txt', header=None ).values[:,0])
)
<load_from_csv>
data =(TabularList.from_df(train, path=PATH,
cat_names=cat_names,
cont_names=cont_names,
procs=procs)
.split_by_idx(valid_idx = range(len(train)-50000, len(train)))
.label_from_df(cols=dep_var)
.add_test(test)
.databunch())
 print(data.train_ds.cont_names)
 print(data.train_ds.cat_names )
!./ffm-train -p libffm_val_fold_2.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_2.txt libffm_fold_2_model
!./ffm-predict libffm_val_fold_2.txt libffm_fold_2_model val_preds_fold_2.txt
!./ffm-predict libffm_tst_fold_2.txt libffm_fold_2_model tst_preds_fold_2.txt
os.remove('libffm_val_fold_2.txt')
 os.remove('libffm_trn_fold_2.txt')
 os.remove('libffm_fold_2_model')
 os.remove('libffm_tst_fold_2.txt')
(
 log_loss(train['target'].iloc[fold_ids[1][1]], pd.read_csv('val_preds_fold_2.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[1][1]], pd.read_csv('val_preds_fold_2.txt', header=None ).values[:,0])
)
<load_from_csv>
data.show_batch()
!./ffm-train -p libffm_val_fold_3.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_3.txt libffm_fold_3_model
!./ffm-predict libffm_val_fold_3.txt libffm_fold_3_model val_preds_fold_3.txt
!./ffm-predict libffm_tst_fold_3.txt libffm_fold_3_model tst_preds_fold_3.txt
os.remove('libffm_val_fold_3.txt')
 os.remove('libffm_trn_fold_3.txt')
 os.remove('libffm_fold_3_model')
 os.remove('libffm_tst_fold_3.txt')
(
 log_loss(train['target'].iloc[fold_ids[2][1]], pd.read_csv('val_preds_fold_3.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[2][1]], pd.read_csv('val_preds_fold_3.txt', header=None ).values[:,0])
)
<load_from_csv>
learn = tabular_learner(data, layers=[200,100], metrics=accuracy,model_dir="/tmp/model/", ps=0.15 )
!./ffm-train -p libffm_val_fold_4.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_4.txt libffm_fold_4_model
!./ffm-predict libffm_val_fold_4.txt libffm_fold_4_model val_preds_fold_4.txt
!./ffm-predict libffm_tst_fold_4.txt libffm_fold_4_model tst_preds_fold_4.txt
os.remove('libffm_val_fold_4.txt')
 os.remove('libffm_trn_fold_4.txt')
 os.remove('libffm_fold_4_model')
 os.remove('libffm_tst_fold_4.txt')
(
 log_loss(train['target'].iloc[fold_ids[3][1]], pd.read_csv('val_preds_fold_4.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[3][1]], pd.read_csv('val_preds_fold_4.txt', header=None ).values[:,0])
)
<load_from_csv>
learn.fit(1, lr=1e-3 )
!./ffm-train -p libffm_val_fold_5.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_5.txt libffm_fold_5_model
!./ffm-predict libffm_val_fold_5.txt libffm_fold_5_model val_preds_fold_5.txt
!./ffm-predict libffm_tst_fold_5.txt libffm_fold_5_model tst_preds_fold_5.txt
os.remove('libffm_val_fold_5.txt')
 os.remove('libffm_trn_fold_5.txt')
 os.remove('libffm_fold_5_model')
 os.remove('libffm_tst_fold_5.txt')
(
 log_loss(train['target'].iloc[fold_ids[4][1]], pd.read_csv('val_preds_fold_5.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[4][1]], pd.read_csv('val_preds_fold_5.txt', header=None ).values[:,0])
)
<load_from_csv>
lr=1e-03
!./ffm-train -p libffm_val_fold_6.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_6.txt libffm_fold_6_model
!./ffm-predict libffm_val_fold_6.txt libffm_fold_6_model val_preds_fold_6.txt
!./ffm-predict libffm_tst_fold_6.txt libffm_fold_6_model tst_preds_fold_6.txt
os.remove('libffm_val_fold_6.txt')
 os.remove('libffm_trn_fold_6.txt')
 os.remove('libffm_fold_6_model')
 os.remove('libffm_tst_fold_6.txt')
(
 log_loss(train['target'].iloc[fold_ids[5][1]], pd.read_csv('val_preds_fold_6.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[5][1]], pd.read_csv('val_preds_fold_6.txt', header=None ).values[:,0])
)
<load_from_csv>
learn.fit_one_cycle(1, slice(lr))
!./ffm-train -p libffm_val_fold_7.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_7.txt libffm_fold_7_model
!./ffm-predict libffm_val_fold_7.txt libffm_fold_7_model val_preds_fold_7.txt
!./ffm-predict libffm_tst_fold_7.txt libffm_fold_7_model tst_preds_fold_7.txt
os.remove('libffm_val_fold_7.txt')
 os.remove('libffm_trn_fold_7.txt')
 os.remove('libffm_fold_7_model')
 os.remove('libffm_tst_fold_7.txt')
(
 log_loss(train['target'].iloc[fold_ids[6][1]], pd.read_csv('val_preds_fold_7.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[6][1]], pd.read_csv('val_preds_fold_7.txt', header=None ).values[:,0])
)
<load_from_csv>
lr=1e-03
!./ffm-train -p libffm_val_fold_8.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_8.txt libffm_fold_8_model
!./ffm-predict libffm_val_fold_8.txt libffm_fold_8_model val_preds_fold_8.txt
!./ffm-predict libffm_tst_fold_8.txt libffm_fold_8_model tst_preds_fold_8.txt
os.remove('libffm_val_fold_8.txt')
 os.remove('libffm_trn_fold_8.txt')
 os.remove('libffm_fold_8_model')
 os.remove('libffm_tst_fold_8.txt')
(
 log_loss(train['target'].iloc[fold_ids[7][1]], pd.read_csv('val_preds_fold_8.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[7][1]], pd.read_csv('val_preds_fold_8.txt', header=None ).values[:,0])
)
<load_from_csv>
learn.fit_one_cycle(2, slice(lr),wd=0.3 )
!./ffm-train -p libffm_val_fold_9.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_9.txt libffm_fold_9_model
!./ffm-predict libffm_val_fold_9.txt libffm_fold_9_model val_preds_fold_9.txt
!./ffm-predict libffm_tst_fold_9.txt libffm_fold_9_model tst_preds_fold_9.txt
os.remove('libffm_val_fold_9.txt')
 os.remove('libffm_trn_fold_9.txt')
 os.remove('libffm_fold_9_model')
 os.remove('libffm_tst_fold_9.txt')
(
 log_loss(train['target'].iloc[fold_ids[8][1]], pd.read_csv('val_preds_fold_9.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[8][1]], pd.read_csv('val_preds_fold_9.txt', header=None ).values[:,0])
)
<load_from_csv>
learn.save('model_1' )
!./ffm-train -p libffm_val_fold_10.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_10.txt libffm_fold_10_model
!./ffm-predict libffm_val_fold_10.txt libffm_fold_10_model val_preds_fold_10.txt
!./ffm-predict libffm_tst_fold_10.txt libffm_fold_10_model tst_preds_fold_10.txt
os.remove('libffm_val_fold_10.txt')
 os.remove('libffm_trn_fold_10.txt')
 os.remove('libffm_fold_10_model')
 os.remove('libffm_tst_fold_10.txt')
(
 log_loss(train['target'].iloc[fold_ids[9][1]], pd.read_csv('val_preds_fold_10.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[9][1]], pd.read_csv('val_preds_fold_10.txt', header=None ).values[:,0])
)
<load_from_csv>
preds = learn.get_preds(ds_type=DatasetType.Test)[0][:,1].numpy()
!./ffm-train -p libffm_val_fold_11.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_11.txt libffm_fold_11_model
!./ffm-predict libffm_val_fold_11.txt libffm_fold_11_model val_preds_fold_11.txt
!./ffm-predict libffm_tst_fold_11.txt libffm_fold_11_model tst_preds_fold_11.txt
os.remove('libffm_val_fold_11.txt')
 os.remove('libffm_trn_fold_11.txt')
 os.remove('libffm_fold_11_model')
 os.remove('libffm_tst_fold_11.txt')
(
 log_loss(train['target'].iloc[fold_ids[10][1]], pd.read_csv('val_preds_fold_11.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[10][1]], pd.read_csv('val_preds_fold_11.txt', header=None ).values[:,0])
)
<load_from_csv>
submission = pd.DataFrame({'id':sample_submission['id'],'target':preds})
 submission.to_csv('my_submission.csv', header=True, index=False )