Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<feature_engineering>
warnings.filterwarnings("ignore" )
data_clean=data['text'].apply(lambda text:clean_tweet(text))<define_variables>
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 labels = train.pop('target')
 labels = labels.values
train_id = train.pop("id")
 test_id = test.pop("id" )
data_labels=data.target.values<choose_model_class>
def ord_to_num(df, col):
keys=np.sort(df[col].unique())
 values=np.arange(len(keys))
map = dict(zip(keys, values))
df[col] = df[col].replace(map)
 ord_to_num(train, 'ord_4')
 ord_to_num(test,'ord_4')
 ord_to_num(train, 'ord_5')
 ord_to_num(test,'ord_5')
 train['ord_4_band'] = pd.qcut(train['ord_4'], 6)
 bands=train.ord_4_band.unique()
keys_bands=np.sort(bands)
 values_bands=np.arange(len(keys_bands))
map_bands = dict(zip(keys_bands, values_bands))
train['ord_4_band'] = train['ord_4_band'].replace(map_bands)
 test['ord_4_band']=pd.cut(test.ord_4,pd.IntervalIndex(keys_bands))
test['ord_4_band'] = test['ord_4_band'].replace(map_bands)
 train['ord_5_band'] = pd.qcut(train['ord_5'], 6)
 bands=train.ord_5_band.unique()
keys_bands=np.sort(bands)
 values_bands=np.arange(len(keys_bands))
map_bands = dict(zip(keys_bands, values_bands))
train['ord_5_band'] = train['ord_5_band'].replace(map_bands)
 test['ord_5_band']=pd.cut(test.ord_5,pd.IntervalIndex(keys_bands))
test['ord_5_band'] = test['ord_5_band'].replace(map_bands)
 for col in ["nom_7", "nom_8", "nom_9"]:
train_vals = set(train[col].unique())
 test_vals = set(test[col].unique())
 ex=train_vals ^ test_vals
if ex:
train.loc[train[col].isin(ex), col]="x"
test.loc[test[col].isin(ex), col]="x"
fulltokenizer=bert.bert_tokenization.FullTokenizer
bert_layer=hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1' )<data_type_conversions>
acc = []
data = pd.concat([train, test])
 numerics = ['int8', 'int16', 'int32', 'int64', 'float16', 'float32', 'float64']
categorical_columns = []
features = data.columns.values.tolist()
for col in features:
if data[col].dtype in numerics: continue
categorical_columns.append(col)
 data2 = data
for col in categorical_columns:
if col in data.columns:
le = LabelEncoder()
le.fit(list(data[col].astype(str ).values))
data2[col] = le.transform(list(data[col].astype(str ).values))
train = data2.iloc[:train.shape[0], :]
test = data2.iloc[train.shape[0]:, :]
vocab_file=bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case=bert_layer.resolved_object.do_lower_case.numpy()
tokenizer=fulltokenizer(vocab_file,do_lower_case )<categorify>
Xtrain, Xval, Ztrain, Zval = train_test_split(train, labels, test_size=0.2, random_state=0)
 train_set = lgbm.Dataset(Xtrain, Ztrain, silent=False)
 valid_set = lgbm.Dataset(Xval, Zval, silent=False)
 params = {
'boosting_type':'gbdt',
'objective': 'regression',
'num_leaves': 31,
'learning_rate': 0.05,
'max_depth': -1,
'subsample': 0.8,
'bagging_fraction' : 1,
'max_bin' : 5000 ,
'bagging_freq': 20,
'colsample_bytree': 0.6,
'metric': 'rmse',
'min_split_gain': 0.5,
'min_child_weight': 1,
'min_child_samples': 10,
'scale_pos_weight':1,
'zero_as_missing': True,
'seed':0,
}
modelL = lgbm.train(params, train_set = train_set, num_boost_round=1000,
early_stopping_rounds=50, verbose_eval=50, valid_sets=valid_set)
 feature_score = pd.DataFrame(train.columns, columns = ['feature'])
 pred_train = pd.DataFrame()
pred_test = pd.DataFrame()
feature_score['score_lgb'] = modelL.feature_importance()
pred_train['lgb_pred'] = modelL.predict(train)
 pred_test['lgb_pred'] = modelL.predict(test)
 acc.append(round(r2_score(labels, pred_train['lgb_pred'])* 100, 2))
fig = plt.figure(figsize =(15,15))
axes = fig.add_subplot(111)
 lgbm.plot_importance(modelL,ax = axes,height = 0.5)
 plt.show() ;plt.close()
def encode_sentence(sent):
return ['[CLS]']+tokenizer.tokenize(sent)+['[SEP]']<define_variables>
data_tr = xgb.DMatrix(Xtrain, label=Ztrain)
 data_cv = xgb.DMatrix(Xval , label=Zval)
 data_train = xgb.DMatrix(train)
 data_test = xgb.DMatrix(test)
 evallist = [(data_tr, 'train'),(data_cv, 'valid')]
parms = {'max_depth':8,
'objective':'reg:logistic',
'eta' :0.3,
'subsample':0.8,
'lambda ' :4,
'colsample_bytree ':0.9,
'colsample_bylevel':1,
'min_child_weight': 10}
modelx = xgb.train(parms, data_tr, num_boost_round=200, evals = evallist,
early_stopping_rounds=30, maximize=False,
verbose_eval=10)
 feature_score['score_xgb'] = feature_score['feature'].map(modelx.get_score(importance_type='weight'))
pred_train['xgb_pred'] = modelx.predict(data_train)
 pred_test['xgb_pred'] = modelx.predict(data_test)
 acc.append(round(r2_score(labels, pred_train['xgb_pred'])* 100, 2))
fig = plt.figure(figsize =(15,15))
axes = fig.add_subplot(111)
 xgb.plot_importance(modelx,ax = axes,height = 0.5)
 plt.show() ;plt.close()
data_inputs=[encode_sentence(sent)for sent in data_clean]<categorify>
train2 = pd.DataFrame(preprocessing.MinMaxScaler().fit_transform(train),
columns=train.columns, index=train.index)
 logreg = LogisticRegression()
logreg.fit(train2, labels)
 coeff_logreg = pd.DataFrame(train2.columns.delete(0))
coeff_logreg.columns = ['feature']
coeff_logreg["score_logreg"] = pd.Series(logreg.coef_[0])
 pred_train['log_pred'] = logreg.predict(train)
 pred_test['log_pred'] = logreg.predict(test)
 acc.append(round(r2_score(labels, pred_train['log_pred'])* 100, 2))
coeff_logreg.sort_values(by='score_logreg', ascending=False )
def get_ids(token):
return tokenizer.convert_tokens_to_ids(token)
 def get_masks(token):
return np.char.not_equal(token,['PAD'] ).astype('int')
 def get_segments(token):
seg_ids=[]
current_id=0
for tok in token:
seg_ids.append(current_id)
 if tok=='[SEP]':
current_id=1-current_id
return seg_ids<randomize_order>
coeff_logreg["score_logreg"] = coeff_logreg["score_logreg"].abs()
feature_score = pd.merge(feature_score, coeff_logreg, on='feature' )
data_with_len = [[sent, data_labels[i], len(sent)]
for i, sent in enumerate(data_inputs)]
random.shuffle(data_with_len)
 data_with_len.sort(key=lambda x: x[2])
 sorted_all = [([get_ids(sent_lab[0]),
get_masks(sent_lab[0]),
get_segments(sent_lab[0])],
sent_lab[1])
 for sent_lab in data_with_len if sent_lab[2] > 7]<create_dataframe>
coeff_linreg["score_linreg"] = coeff_linreg["score_linreg"].abs()
feature_score = pd.merge(feature_score, coeff_linreg, on='feature' )
all_dataset = tf.data.Dataset.from_generator(lambda: sorted_all,
output_types=(tf.int32, tf.int32))<define_variables>
feature_score = feature_score.fillna(0)
 feature_score = feature_score.set_index('feature')
 feature_score = pd.DataFrame(preprocessing.MinMaxScaler().fit_transform(feature_score),
columns=feature_score.columns, index=feature_score.index)
 feature_score['mean'] = feature_score.mean(axis=1)
 pred_train['mean'] = pred_train.mean(axis=1)
 pred_test['mean'] = pred_test.mean(axis=1)
 acc.append(round(r2_score(labels, pred_train['mean'])* 100, 2))
fL = 0.2
flog = 0.5
flin = 0.15
fx = 1-fL-flog-flin
feature_score['total'] = fL*feature_score['score_lgb'] + fx*feature_score['score_xgb'] \
+ flog*feature_score['score_logreg'] + flin*feature_score['score_linreg']
pred_train['total'] = fL*pred_train['lgb_pred'] + fx*pred_train['xgb_pred'] + \
+ flog*pred_train['log_pred'] + flin*pred_train['lin_pred']
pred_test['total'] = fL*pred_test['lgb_pred'] + fx*pred_test['xgb_pred'] + \
+ flog*pred_test['log_pred'] + flin*pred_test['lin_pred']
acc.append(round(r2_score(labels, pred_train['total'])* 100, 2))
feature_score.sort_values('total', ascending=False ).plot(kind='bar', figsize=(20, 10))
BATCH_SIZE = 32
all_batched = all_dataset.padded_batch(BATCH_SIZE,
padded_shapes=(( 3, None),()),
padding_values=(0, 0))<define_variables>
feature_score.sort_values('score_logreg', ascending=False )
NB_BATCHES = math.ceil(len(sorted_all)/ BATCH_SIZE)
 NB_BATCHES_TEST = NB_BATCHES // 10
all_batched.shuffle(NB_BATCHES)
 test_dataset = all_batched.take(NB_BATCHES_TEST)
 train_dataset = all_batched.skip(NB_BATCHES_TEST )<choose_model_class>
columns = [i for i in data.columns]
dummies = pd.get_dummies(data,
columns=columns,
drop_first=True,
sparse=True)
 del data
class DCNNBERTEmbedding(tf.keras.Model):
def __init__(self,
nb_filters=50,
FFN_units=512,
nb_classes=2,
dropout_rate=0.1,
name="dcnn"):
super(DCNNBERTEmbedding, self ).__init__(name=name)
 self.bert_layer = hub.KerasLayer(
 "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1",
trainable=False)
 self.bigram = layers.Conv1D(filters=nb_filters,
kernel_size=2,
padding="valid",
activation="relu")
 self.trigram = layers.Conv1D(filters=nb_filters,
kernel_size=3,
padding="valid",
activation="relu")
 self.fourgram = layers.Conv1D(filters=nb_filters,
kernel_size=4,
padding="valid",
activation="relu")
 self.pool = layers.GlobalMaxPool1D()
self.dense_1 = layers.Dense(units=FFN_units, activation="relu")
 self.dropout = layers.Dropout(rate=dropout_rate)
 if nb_classes == 2:
self.last_dense = layers.Dense(units=1,
activation="sigmoid")
 else:
self.last_dense = layers.Dense(units=nb_classes,
activation="softmax")
 def embed_with_bert(self, all_tokens):
_, embs = self.bert_layer([all_tokens[:, 0, :],
all_tokens[:, 1, :],
all_tokens[:, 2, :]])
 return embs
def call(self, inputs, training):
x = self.embed_with_bert(inputs)
 print(x.shape)
 x_1 = self.bigram(x)
 x_1 = self.pool(x_1)
 x_2 = self.trigram(x)
 x_2 = self.pool(x_2)
 x_3 = self.fourgram(x)
 x_3 = self.pool(x_3)
 merged = tf.concat([x_1, x_2, x_3], axis=-1)
 merged = self.dense_1(merged)
 merged = self.dropout(merged, training)
 output = self.last_dense(merged)
 return output<define_variables>
train = dummies.iloc[:train.shape[0], :]
test = dummies.iloc[train.shape[0]:, :]
del dummies
NB_FILTERS = 100
FFN_UNITS = 256
NB_CLASSES = 2
DROPOUT_RATE = 0.2
BATCH_SIZE = 32
NB_EPOCHS = 3<choose_model_class>
train = train.sparse.to_coo().tocsr()
test = test.sparse.to_coo().tocsr()
train = train.astype("float32")
 test = test.astype("float32" )
Dcnn = DCNNBERTEmbedding(nb_filters=NB_FILTERS,
FFN_units=FFN_UNITS,
nb_classes=NB_CLASSES,
dropout_rate=DROPOUT_RATE )<choose_model_class>
lr = LogisticRegression(C=0.12,
solver="lbfgs",
tol=0.002,
max_iter=10000)
 lr.fit(train, labels)
 lr_pred = lr.predict_proba(train)[:, 1]
score = roc_auc_score(labels, lr_pred)
 print("score: ", score )
if NB_CLASSES == 2:
Dcnn.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
 else:
Dcnn.compile(loss="sparse_categorical_crossentropy",
optimizer="adam",
metrics=["sparse_categorical_accuracy"] )<train_model>
submission["id"] = test_id
submission["target"] = lr.predict_proba(test)[:, 1]
class MyCustomCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, logs=None):
ckpt_manager.save()
print("Checkpoint saved at {}.".format(checkpoint_path))<train_model>
submission.to_csv("submission.csv", index=False )
<categorify><EOS> 
submission.to_csv("submission.csv", index=False )