Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<save_to_csv>
!pip install --no-warn-conflicts -q deepctr
clf = linear_model.LogisticRegression()
clf.fit(X, train.target.values)
 preds = clf.predict_proba(X_test)[:, 1]
clf.score(X, train.target.values)
 sample.loc[:, "target"] = preds
sample.to_csv("submission.csv", index=False )<compute_test_metric>
warnings.simplefilter('ignore' )
clf.score(X, train.target.values )<load_from_csv>
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )
df_train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 df_test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 df_sub = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/sample_submission.csv' )<sort_values>
test["target"] = -1
df_train.sort_index(inplace=True )<prepare_x_and_y>
data = pd.concat([train, test] ).reset_index(drop=True )
y_train = df_train['target']
test_id = df_test['id']
df_train.drop(['target', 'id'], axis=1, inplace=True)
 df_test.drop('id', axis=1, inplace=True )<categorify>
data['null'] = data.isna().sum(axis=1 )
cat_feat_to_encode = df_train.columns.tolist()
smoothing=0.20
oof = pd.DataFrame([])
 for tr_idx, oof_idx in StratifiedKFold(n_splits=5, random_state= 1024, shuffle=True ).split(df_train, y_train):
ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(df_train.iloc[tr_idx, :], y_train.iloc[tr_idx])
 oof = oof.append(ce_target_encoder.transform(df_train.iloc[oof_idx, :]), ignore_index=False)
 ce_target_encoder = ce.TargetEncoder(cols = cat_feat_to_encode, smoothing=smoothing)
 ce_target_encoder.fit(df_train, y_train)
 df_train = oof.sort_index()
df_test = ce_target_encoder.transform(df_test )<prepare_x_and_y>
sparse_features = [feat for feat in train.columns if feat not in ['id','target']]
data[sparse_features] = data[sparse_features].fillna('-1', )
x_train = df_train.iloc[:,:].values
x_test = df_test.iloc[:,:].values<statistical_test>
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat].fillna("-1" ).astype(str ).values )
cw = class_weight.compute_class_weight('balanced',np.unique(y_train),y_train )<import_modules>
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True )
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.callbacks import EarlyStopping 
<choose_model_class>
fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique())for feat in sparse_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns )
classifier = Sequential()
classifier.add(Dense(units = 512, kernel_initializer = 'uniform', activation = 'relu', input_dim = 23))
classifier.add(Dense(units = 256, kernel_initializer = 'uniform', activation = 'relu'))
classifier.add(Dense(units = 128, kernel_initializer = 'uniform', activation = 'relu'))
classifier.add(Dense(units = 64, kernel_initializer = 'uniform', activation = 'relu'))
classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
classifier.compile(loss=keras.losses.binary_crossentropy,
optimizer=keras.optimizers.SGD(lr=0.01, momentum=0.9, nesterov=True)) 
<choose_model_class>
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1)
<train_model>
class CyclicLR(keras.callbacks.Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma **(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(
 self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr())
classifier.fit(x_train, y_train, batch_size = 1000, epochs = 100, verbose=0,callbacks=[es],class_weight=cw)
<predict_on_test>
target = ['target']
N_Splits = 20
Epochs = 10
SEED = 2020
y_pred = classifier.predict_proba(x_test)[:,0] 
<save_to_csv>
oof_pred_deepfm = np.zeros(( len(train),))
y_pred_deepfm = np.zeros(( len(test),))
skf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)
 for fold,(tr_ind, val_ind)in enumerate(skf.split(train, train[target])) :
X_train, X_val = train[sparse_features].iloc[tr_ind], train[sparse_features].iloc[val_ind]
y_train, y_val = train[target].iloc[tr_ind], train[target].iloc[val_ind]
train_model_input = {name:X_train[name] for name in feature_names}
val_model_input = {name:X_val[name] for name in feature_names}
test_model_input = {name:test[name] for name in feature_names}
model = NFFM(linear_feature_columns, dnn_feature_columns)
 model.compile("adam", "binary_crossentropy", metrics=[auc],)
 es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.0001, patience=2, verbose=1, mode='max', baseline=None, restore_best_weights=True)
 sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=0)
 clr = CyclicLR(base_lr=0.00001 / 100, max_lr = 0.0001,
step_size= int(1.0*(test.shape[0])/1024), mode='exp_range',
gamma=1., scale_fn=None, scale_mode='cycle')
 history = model.fit(train_model_input, y_train,
validation_data=(val_model_input, y_val),
batch_size=512, epochs=Epochs, verbose=1,
callbacks=[es, sb, clr],)
 model.load_weights('./nn_model.w8')
 val_pred = model.predict(val_model_input, batch_size=512)
 print(f"validation AUC fold {fold+1} : {round(roc_auc_score(y_val, val_pred), 5)}")
 oof_pred_deepfm[val_ind] = val_pred.ravel()
y_pred_deepfm += model.predict(test_model_input, batch_size=512 ).ravel() /(N_Splits)
 K.clear_session()
df_sub = pd.DataFrame()
df_sub['id'] = test_id
df_sub['target'] = y_pred
df_sub.to_csv('submission.csv', index=False )<import_modules>
print(f"OOF AUC : {round(roc_auc_score(train.target.values, oof_pred_deepfm), 5)}" )
from fastai.tabular import *<load_from_csv>
test_idx = test.id.values
submission = pd.DataFrame.from_dict({
'id': test_idx,
'target': y_pred_deepfm
})
 submission.to_csv("submission.csv", index=False)
 print("Submission file saved!" )
sample_submission = pd.read_csv(".. /input/cat-in-the-dat-ii/sample_submission.csv")
 X_test = pd.read_csv(".. /input/cat-in-the-dat-ii/test.csv")
 train = pd.read_csv(".. /input/cat-in-the-dat-ii/train.csv" )<prepare_x_and_y>
np.save('oof_pred_deepfm.npy',oof_pred_deepfm)
 np.save('y_pred_deepfm.npy', y_pred_deepfm )