Kaggle competition: Categorical Feature Encoding Challenge
<SOS> metric: AUC Kaggle data source: categorical-feature-encoding-challenge<set_options>
!pip install gast==0.2.2
del(train_orig)
 gc.collect()<feature_engineering>
warnings.filterwarnings("ignore")
train['target'] = np.where(train['target'] >= 0.5, True, False )<split>
from tensorflow.keras.utils import get_custom_objects
train_df, validate_df = model_selection.train_test_split(train, test_size=0.1)
 print('%d train comments, %d validate comments' %(len(train_df), len(validate_df)) )<string_transform>
def auc(y_true, y_pred):
def fallback_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except:
return 0.5
return tf.py_function(fallback_auc,(y_true, y_pred), tf.double )
MAX_NUM_WORDS = 100000
TOXICITY_COLUMN = 'target'
TEXT_COLUMN = 'comment_text'
tokenizer = Tokenizer(num_words=MAX_NUM_WORDS)
 tokenizer.fit_on_texts(train_df[TEXT_COLUMN])
 MAX_SEQUENCE_LENGTH = 256
def pad_text(texts, tokenizer):
return pad_sequences(tokenizer.texts_to_sequences(texts), maxlen=MAX_SEQUENCE_LENGTH )<set_options>
def create_model(data, catcols):
inputs = []
outputs = []
for c in catcols:
num_unique_values = int(data[c].nunique())
 embed_dim = int(min(np.ceil(( num_unique_values)/2), 50))
inp = layers.Input(shape=(1,))
out = layers.Embedding(num_unique_values + 1, embed_dim, name=c )(inp)
 out = layers.SpatialDropout1D(0.3 )(out)
 out = layers.Reshape(target_shape=(embed_dim,))(out)
 inputs.append(inp)
 outputs.append(out)
 x = layers.Concatenate()(outputs)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300 )(x)
 x = Activation('Mish', name="act1" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 x = layers.Dense(300 )(x)
 x = Activation('Mish', name="act2" )(x)
 x = layers.Dropout(0.3 )(x)
 x = layers.BatchNormalization()(x)
 y = layers.Dense(2, activation="softmax" )(x)
 model = Model(inputs=inputs, outputs=y)
 return model
gc.collect()<define_variables>
train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 sample = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv" )
EMBEDDINGS_DIMENSION = 300
embedding_matrix = np.zeros(( len(tokenizer.word_index)+ 1,EMBEDDINGS_DIMENSION))<feature_engineering>
test["target"] = -1
data = pd.concat([train, test] ).reset_index(drop=True )
num_words_in_embedding = 0
for word, i in tokenizer.word_index.items() :
if word in embeddings_index.vocab:
embedding_vector = embeddings_index[word]
embedding_matrix[i] = embedding_vector
num_words_in_embedding += 1<categorify>
features = [x for x in train.columns if x not in ["id", "target"]]
train_text = pad_text(train_df[TEXT_COLUMN], tokenizer)
 train_labels = train_df[TOXICITY_COLUMN]
validate_text = pad_text(validate_df[TEXT_COLUMN], tokenizer)
 validate_labels = validate_df[TOXICITY_COLUMN]<set_options>
for feat in features:
lbl_enc = preprocessing.LabelEncoder()
data[feat] = lbl_enc.fit_transform(data[feat].values )
gc.collect()<choose_model_class>
train = data[data.target != -1].reset_index(drop=True)
 test = data[data.target == -1].reset_index(drop=True)
 test_data = [test.loc[:, features].values[:, k] for k in range(test.loc[:, features].values.shape[1])]
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')
 embedding_layer = Embedding(len(tokenizer.word_index)+ 1,
EMBEDDINGS_DIMENSION,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False)
 x = embedding_layer(sequence_input)
 x = SpatialDropout1D(0.2 )(x)
 x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)
 x = Conv1D(64, kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(x)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 x = concatenate([avg_pool1, max_pool1])
 preds = Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 model.summary() 
<choose_model_class>
oof_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test)))
 skf = StratifiedKFold(n_splits=5)
 for train_index, test_index in skf.split(train, train.target.values):
X_train, X_test = train.iloc[train_index, :], train.iloc[test_index, :]
X_train = X_train.reset_index(drop=True)
 X_test = X_test.reset_index(drop=True)
 y_train, y_test = X_train.target.values, X_test.target.values
model = create_model(data, features)
 model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[auc])
 X_train = [X_train.loc[:, features].values[:, k] for k in range(X_train.loc[:, features].values.shape[1])]
X_test = [X_test.loc[:, features].values[:, k] for k in range(X_test.loc[:, features].values.shape[1])]
es = callbacks.EarlyStopping(monitor='val_auc', min_delta=0.001, patience=5,
verbose=1, mode='max', baseline=None, restore_best_weights=True)
 rlr = callbacks.ReduceLROnPlateau(monitor='val_auc', factor=0.5,
patience=3, min_lr=1e-6, mode='max', verbose=1)
 model.fit(X_train,
utils.to_categorical(y_train),
validation_data=(X_test, utils.to_categorical(y_test)) ,
verbose=1,
batch_size=1024,
callbacks=[es, rlr],
epochs=100 
)
 valid_fold_preds = model.predict(X_test)[:, 1]
test_fold_preds = model.predict(test_data)[:, 1]
oof_preds[test_index] = valid_fold_preds.ravel()
test_preds += test_fold_preds.ravel()
print(metrics.roc_auc_score(y_test, valid_fold_preds))
K.clear_session()
model.compile(loss='binary_crossentropy',
optimizer=Adam() ,
metrics=['acc'] )<define_variables>
print("Overall AUC={}".format(metrics.roc_auc_score(train.target.values, oof_preds)) )
<train_model><EOS> 
test_preds /= 5
test_ids = test.id.values
print("Saving submission file")
 submission = pd.DataFrame.from_dict({
'id': test_ids,
'target': test_preds
})
 submission.to_csv("submission.csv", index=False )