Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<import_modules>
!pip install --no-warn-conflicts -q deepctr
from keras.models import load_model<load_pretrained>
warnings.simplefilter('ignore' )
saved_model = load_model('best_model.h5' )<predict_on_test>
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )
predictions = saved_model.predict_classes(X_test )<concatenate>
test['target'] = -1
test.head()
submission = pd.concat([pd.Series(range(1,28001), name ="ImageId"), submission], axis = 1 )<save_to_csv>
data = pd.concat([train, test] ).reset_index(drop=True )
submission.to_csv("submission.csv", index=False )<save_to_csv>
data['null'] = data.isna().sum(axis=1 )
submission.to_csv("submission.csv", index=False )<import_modules>
sparse_features = [feat for feat in train.columns if feat not in ['id','target']]
data[sparse_features] = data[sparse_features].fillna('-1', )
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt<load_from_csv>
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat].fillna('-1' ).astype(str ).values )
test_data=pd.read_csv('.. /input/digit-recognizer/test.csv')
 train_data=pd.read_csv('.. /input/digit-recognizer/train.csv' )<prepare_x_and_y>
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True )
training=np.array(train_data,dtype='float32')
 testing=np.array(test_data,dtype='float32' )<prepare_x_and_y>
fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique())for feat in sparse_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns )
x_train=training[:,1:]/255
y_train=training[:,0]<prepare_x_and_y>
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )
x_test=testing[:,0:]/255<categorify>
def focal_loss(gamma=2., alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(K.epsilon() +pt_1)) -K.mean(( 1-alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0 + K.epsilon()))
return focal_loss_fixed
get_custom_objects().update({'focal_loss_fn': focal_loss() } )
y_train_cat=to_categorical(y_train )<choose_model_class>
def custom_gelu(x):
return 0.5 * x *(1 + tf.tanh(tf.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3))))
get_custom_objects().update({'custom_gelu': Activation(custom_gelu)} )
model =Sequential()
model.add(Conv2D(64,kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1)))
 model.add(Conv2D(64,kernel_size=(3, 3),activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 128, kernel_size =(3,3),activation ='relu'))
model.add(Conv2D(filters = 128, kernel_size =(3,3),activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(BatchNormalization())
 model.add(Conv2D(filters = 256, kernel_size =(3,3),activation ='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(BatchNormalization())
 model.add(Dense(512, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
class WarmUpLearningRateScheduler(tf.keras.callbacks.Callback):

 def __init__(self, warmup_batches, init_lr, verbose=0):

 super(WarmUpLearningRateScheduler, self ).__init__()
self.warmup_batches = warmup_batches
self.init_lr = init_lr
self.verbose = verbose
self.batch_count = 0
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.batch_count = self.batch_count + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
if self.batch_count <= self.warmup_batches:
lr = self.batch_count*self.init_lr/self.warmup_batches
K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: WarmUpLearningRateScheduler setting learning '
'rate to %s.' %(self.batch_count + 1, lr))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.RMSprop() ,
metrics=['accuracy'])
 learning_rate_reduction = ReduceLROnPlateau(monitor='accuracy',
patience=3,
verbose=1,
factor=0.3,
min_lr=0.0001 )<train_model>
class CyclicLR(keras.callbacks.Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma **(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(
 self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr())
epochs_range = 60
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
 model.fit(X_train,y_train_cat,batch_size=256,epochs=epochs_range )<predict_on_test>
target = ['target']
N_Splits = 50
Verbose = 0
Epochs = 10
SEED = 2020
Batch_S_T = 128
Batch_S_P = 512
y_pred=model.predict_classes(X_test )<load_from_csv>
oof_pred_deepfm = np.zeros(( len(train),))
y_pred_deepfm = np.zeros(( len(test),))
skf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)
 for fold,(tr_ind, val_ind)in enumerate(skf.split(train, train[target])) :
X_train, X_val = train[sparse_features].iloc[tr_ind], train[sparse_features].iloc[val_ind]
y_train, y_val = train[target].iloc[tr_ind], train[target].iloc[val_ind]
train_model_input = {name:X_train[name] for name in feature_names}
val_model_input = {name:X_val[name] for name in feature_names}
test_model_input = {name:test[name] for name in feature_names}
model = DeepFM(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 256), dnn_dropout=0.0, dnn_activation='Mish', dnn_use_bn=False, task='binary')
 model.compile('adam', loss = 'binary_crossentropy', metrics=[auc],)
 es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=3, verbose=Verbose, mode='max', baseline=None, restore_best_weights=True)
 sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=Verbose)
 clr = CyclicLR(base_lr=1e-7, max_lr = 1e-4, step_size= int(1.0*(test.shape[0])/(Batch_S_T*4)) , mode='exp_range', gamma=1.0, scale_fn=None, scale_mode='cycle')
 history = model.fit(train_model_input, y_train,
validation_data=(val_model_input, y_val),
batch_size=Batch_S_T, epochs=Epochs, verbose=Verbose,
callbacks=[es, sb, clr],)
 model.load_weights('./nn_model.w8')
 val_pred = model.predict(val_model_input, batch_size=Batch_S_P)
 print(f'validation AUC fold {fold+1} : {round(roc_auc_score(y_val, val_pred), 5)}')
 oof_pred_deepfm[val_ind] = val_pred.ravel()
y_pred_deepfm += model.predict(test_model_input, batch_size=Batch_S_P ).ravel() /(N_Splits)
 K.clear_session()
sample=pd.read_csv('.. /input/digit-recognizer/sample_submission.csv' )<create_dataframe>
print(f'OOF AUC : {round(roc_auc_score(train.target.values, oof_pred_deepfm), 5)}' )
temp=pd.DataFrame({'ImageId':sample['ImageId'],'Label':y_pred} )<save_to_csv>
test_idx = test.id.values
submission = pd.DataFrame.from_dict({
'id': test_idx,
'target': y_pred_deepfm
})
 submission.to_csv('submission.csv', index=False)
 print('Submission file saved!' )
csv_data=temp.to_csv('pred.csv',index=False )<set_options>
np.save('oof_pred_deepfm.npy',oof_pred_deepfm)
 np.save('y_pred_deepfm.npy', y_pred_deepfm )