Kaggle competition: Categorical Feature Encoding Challenge II
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge-ii<normalization>
!pip install --no-warn-conflicts -q deepctr
def augment(x,y,t=2):
if t==0:
return x, y
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 del x1
gc.collect()
for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 del x1
gc.collect()
print("The sizes of x, xn, and xs are {}, {}, {}, respectively.".format(sys.getsizeof(x),
sys.getsizeof(xn),
sys.getsizeof(xs)
)
)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 print("The sizes of x, xn, and xs are {}, {}, {}, respectively.".format(sys.getsizeof(x)/1024**3,
sys.getsizeof(xn),
sys.getsizeof(xs)
)
)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 y = np.concatenate([y,ys,yn])
 print("The sizes of y, yn, and ys are {}, {}, {}, respectively.".format(sys.getsizeof(y),
sys.getsizeof(yn),
sys.getsizeof(ys)
)
)
 gc.collect()
return np.vstack([x,xs, xn]), y<split>
warnings.simplefilter('ignore' )
for fold_,(trn_, val_)in enumerate(folds.split(y, y)) :
print("Current Fold: {}".format(fold_))
trn_x, trn_y = X[trn_, :], y[trn_]
val_x, val_y = X[val_, :], y[val_]
NAUGMENTATIONS=1
NSHUFFLES=0
val_pred, test_fold_pred = 0, 0
for i in range(NAUGMENTATIONS):
print("
Fold {}, Augmentation {}".format(fold_, i+1))
trn_aug_x, trn_aug_y = augment(trn_x, trn_y, NSHUFFLES)
 trn_aug_x = pd.DataFrame(trn_aug_x)
 trn_aug_x = trn_aug_x.add_prefix('var_')
 clf = Pipeline([ 
('lr_clf', LogisticRegression(solver='lbfgs', max_iter=1500, C=10))
])
 clf.fit(trn_aug_x, trn_aug_y)
 print("Making predictions for the validation data")
 val_pred += clf.predict_proba(val_x)[:,1]
print("Making predictions for the test data")
 test_fold_pred += clf.predict_proba(X_test)[:,1]
val_pred /= NAUGMENTATIONS
test_fold_pred /= NAUGMENTATIONS
roc_cv.append(roc_auc_score(val_y, val_pred))
print("AUC = {}".format(roc_auc_score(val_y, val_pred)))
 oof_preds[val_, :] = val_pred.reshape(( -1, 1))
test_preds += test_fold_pred.reshape(( -1, 1))<feature_engineering>
train = pd.read_csv('.. /input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('.. /input/cat-in-the-dat-ii/test.csv' )
test_preds /= NFOLDS<compute_test_metric>
test['target'] = -1
roc_score_1 = round(roc_auc_score(y, oof_preds.ravel()), 5)
 roc_score = round(sum(roc_cv)/len(roc_cv), 5)
 st_dev = round(np.array(roc_cv ).std() , 5)
 print("Average of the folds' AUCs = {}".format(roc_score))
print("Combined folds' AUC = {}".format(roc_score_1))
print("The standard deviation = {}".format(st_dev))<save_to_csv>
data = pd.concat([train, test] ).reset_index(drop=True )
print("Saving submission file")
 sample = pd.read_csv('.. /input/sample_submission.csv')
 sample.target = test_preds.astype(float)
 sample.ID_code = test_ids
sample.to_csv('submission.csv', index=False )<set_options>
data['null'] = data.isna().sum(axis=1 )
warnings.filterwarnings('ignore' )<load_from_csv>
sparse_features = [feat for feat in train.columns if feat not in ['id','target']]
data[sparse_features] = data[sparse_features].fillna('-1', )
%%time
train_df = pd.read_csv(PATH+"train.csv")
 test_df = pd.read_csv(PATH+"test.csv" )<feature_engineering>
for feat in sparse_features:
lbe = LabelEncoder()
data[feat] = lbe.fit_transform(data[feat].fillna('-1' ).astype(str ).values )
def missing_data(data):
total = data.isnull().sum()
percent =(data.isnull().sum() /data.isnull().count() *100)
 tt = pd.concat([total,percent],axis=1,keys=['Total','Percent'])
 types = []
for col in data.columns:
dtype = str(data[col].dtype)
 types.append(dtype)
 tt['Types'] = types
return(np.transpose(tt))<count_missing_values>
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True )
%%time
missing_data(train_df )<count_missing_values>
fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique())for feat in sparse_features]
dnn_feature_columns = fixlen_feature_columns
linear_feature_columns = fixlen_feature_columns
feature_names = get_feature_names(linear_feature_columns + dnn_feature_columns )
%%time
missing_data(test_df )<count_values>
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )
print("There are {}% target values with 1".format(100 * train_df["target"].value_counts() [1]/train_df.shape[0]))<categorify>
def focal_loss(gamma=2., alpha=.25):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(K.epsilon() +pt_1)) -K.mean(( 1-alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0 + K.epsilon()))
return focal_loss_fixed
get_custom_objects().update({'focal_loss_fn': focal_loss() } )
correlations = train_df[features].corr().where(np.triu(np.ones(train_df[features].corr().shape),k=1 ).astype(np.bool))
correlations_df = correlations.abs().unstack().dropna().sort_values().reset_index()
correlations_df.shape<count_unique_values>
def custom_gelu(x):
return 0.5 * x *(1 + tf.tanh(tf.sqrt(2 / np.pi)*(x + 0.044715 * tf.pow(x, 3))))
get_custom_objects().update({'custom_gelu': Activation(custom_gelu)} )
%%time
features = train_df.columns.values[2:202]
unique_max_train = []
unique_max_test = []
for feature in features:
values = train_df[feature].value_counts()
unique_max_train.append([feature,values.max() ,values.idxmax() ])
 values = test_df[feature].value_counts()
unique_max_test.append([feature,values.max() ,values.idxmax() ] )<feature_engineering>
class WarmUpLearningRateScheduler(tf.keras.callbacks.Callback):

 def __init__(self, warmup_batches, init_lr, verbose=0):

 super(WarmUpLearningRateScheduler, self ).__init__()
self.warmup_batches = warmup_batches
self.init_lr = init_lr
self.verbose = verbose
self.batch_count = 0
self.learning_rates = []
def on_batch_end(self, batch, logs=None):
self.batch_count = self.batch_count + 1
lr = K.get_value(self.model.optimizer.lr)
 self.learning_rates.append(lr)
 def on_batch_begin(self, batch, logs=None):
if self.batch_count <= self.warmup_batches:
lr = self.batch_count*self.init_lr/self.warmup_batches
K.set_value(self.model.optimizer.lr, lr)
 if self.verbose > 0:
print('
Batch %05d: WarmUpLearningRateScheduler setting learning '
'rate to %s.' %(self.batch_count + 1, lr))
%%time
i = 1
for df in [test_df, train_df]:
idx = df.columns.values[i:i+200]
df['sum'] = df[idx].sum(axis=1)
 df['min'] = df[idx].min(axis=1)
 df['max'] = df[idx].max(axis=1)
 df['mean'] = df[idx].mean(axis=1)
 df['std'] = df[idx].std(axis=1)
 df['skew'] = df[idx].skew(axis=1)
 df['kurt'] = df[idx].kurtosis(axis=1)
 df['med'] = df[idx].median(axis=1)
 df['range'] = df['max']-df['min']
i = i + 1<define_variables>
class CyclicLR(keras.callbacks.Callback):
def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',
gamma=1., scale_fn=None, scale_mode='cycle'):
super(CyclicLR, self ).__init__()
self.base_lr = base_lr
self.max_lr = max_lr
self.step_size = step_size
self.mode = mode
self.gamma = gamma
if scale_fn == None:
if self.mode == 'triangular':
self.scale_fn = lambda x: 1.
 self.scale_mode = 'cycle'
elif self.mode == 'triangular2':
self.scale_fn = lambda x: 1 /(2.**(x - 1))
self.scale_mode = 'cycle'
elif self.mode == 'exp_range':
self.scale_fn = lambda x: gamma **(x)
 self.scale_mode = 'iterations'
else:
self.scale_fn = scale_fn
self.scale_mode = scale_mode
self.clr_iterations = 0.
 self.trn_iterations = 0.
 self.history = {}
self._reset()
def _reset(self, new_base_lr=None, new_max_lr=None,
new_step_size=None):

 if new_base_lr != None:
self.base_lr = new_base_lr
if new_max_lr != None:
self.max_lr = new_max_lr
if new_step_size != None:
self.step_size = new_step_size
self.clr_iterations = 0.
 def clr(self):
cycle = np.floor(1 + self.clr_iterations /(2 * self.step_size))
x = np.abs(self.clr_iterations / self.step_size - 2 * cycle + 1)
 if self.scale_mode == 'cycle':
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(cycle)
 else:
return self.base_lr +(self.max_lr - self.base_lr)* np.maximum(0,(1 - x)) * self.scale_fn(
 self.clr_iterations)
 def on_train_begin(self, logs={}):
logs = logs or {}
if self.clr_iterations == 0:
K.set_value(self.model.optimizer.lr, self.base_lr)
 else:
K.set_value(self.model.optimizer.lr, self.clr())
 def on_batch_end(self, epoch, logs=None):
logs = logs or {}
self.trn_iterations += 1
self.clr_iterations += 1
K.set_value(self.model.optimizer.lr, self.clr())
drop_cols = [col for col in correlations.columns if any(abs(correlations[col])>0.95)]<drop_column>
target = ['target']
N_Splits = 50
Verbose = 0
Epochs = 10
SEED = 2020
Batch_S_T = 128
Batch_S_P = 512
train_df = train_df.drop(columns=drop_cols)
 test_df = test_df.drop(columns=drop_cols )<train_model>
oof_pred_deepfm = np.zeros(( len(train),))
y_pred_deepfm = np.zeros(( len(test),))
skf = StratifiedKFold(n_splits=N_Splits, shuffle=True, random_state=SEED)
 for fold,(tr_ind, val_ind)in enumerate(skf.split(train, train[target])) :
X_train, X_val = train[sparse_features].iloc[tr_ind], train[sparse_features].iloc[val_ind]
y_train, y_val = train[target].iloc[tr_ind], train[target].iloc[val_ind]
train_model_input = {name:X_train[name] for name in feature_names}
val_model_input = {name:X_val[name] for name in feature_names}
test_model_input = {name:test[name] for name in feature_names}
model = DeepFM(linear_feature_columns, dnn_feature_columns, dnn_hidden_units=(256, 256), dnn_dropout=0.0, dnn_activation='Mish', dnn_use_bn=False, task='binary')
 model.compile('adam', loss = 'binary_crossentropy', metrics=[auc],)
 es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=3, verbose=Verbose, mode='max', baseline=None, restore_best_weights=True)
 sb = callbacks.ModelCheckpoint('./nn_model.w8', save_weights_only=True, save_best_only=True, verbose=Verbose)
 clr = CyclicLR(base_lr=1e-7, max_lr = 1e-4, step_size= int(1.0*(test.shape[0])/(Batch_S_T*4)) , mode='exp_range', gamma=1.0, scale_fn=None, scale_mode='cycle')
 history = model.fit(train_model_input, y_train,
validation_data=(val_model_input, y_val),
batch_size=Batch_S_T, epochs=Epochs, verbose=Verbose,
callbacks=[es, sb, clr],)
 model.load_weights('./nn_model.w8')
 val_pred = model.predict(val_model_input, batch_size=Batch_S_P)
 print(f'validation AUC fold {fold+1} : {round(roc_auc_score(y_val, val_pred), 5)}')
 oof_pred_deepfm[val_ind] = val_pred.ravel()
y_pred_deepfm += model.predict(test_model_input, batch_size=Batch_S_P ).ravel() /(N_Splits)
 K.clear_session()
print('Train and test columns: {} {}'.format(len(train_df.columns), len(test_df.columns)) )<drop_column>
print(f'OOF AUC : {round(roc_auc_score(train.target.values, oof_pred_deepfm), 5)}' )
train = train_df.drop(columns=['ID_code','target'])
 train_label = train_df['target']
test = test_df.drop(columns='ID_code')
 test_ids = test_df['ID_code']<string_transform>
test_idx = test.id.values
submission = pd.DataFrame.from_dict({
'id': test_idx,
'target': y_pred_deepfm
})
 submission.to_csv('submission.csv', index=False)
 print('Submission file saved!' )
def del_punct(one_list):
return_list = []
regex = re.compile('['+re.escape("'")+']')
 for element in one_list:
return_list.append(regex.sub(" ",element ).strip())
 return return_list<string_transform>
np.save('oof_pred_deepfm.npy',oof_pred_deepfm)
 np.save('y_pred_deepfm.npy', y_pred_deepfm )