Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<categorify>
COMPLETE_RUN = False
def fast_encode_xlm(texts, tokenizer, chunk_size=256, maxlen=512):

 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tokenizer.batch_encode_plus(text_chunk, pad_to_max_length = True, max_length = maxlen)
 all_ids.extend(np.array(encs.input_ids))
return np.array(all_ids )<prepare_x_and_y>
os.environ['PYTHONHASHSEED'] = '10000'
np.random.seed(10001)
 random.seed(10002)
 tf.set_random_seed(10003)
 session_conf = tf.ConfigProto(intra_op_parallelism_threads=6, inter_op_parallelism_threads=5)
 backend.set_session(tf.Session(graph=tf.get_default_graph() , config=session_conf))
x_train = fast_encode_xlm(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_valid = fast_encode_xlm(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 x_test = fast_encode_xlm(test.content.astype(str), tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<categorify>
max_size = 150000
max_features = 100000
maxlen = 500
embed_size = 300
embedding_file = '.. /input/donorschooseorg-preprocessed-data/embeddings_small.vec'
if COMPLETE_RUN == False:
max_size = 150
max_features = 100
maxlen = 5
embedding_file = '.. /input/donorschooseorg-preprocessed-data/embeddings_small.vec'
olid_train_encode = fast_encode_xlm(olid_train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_1k_encode = fast_encode_xlm(olid_train_1k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_2k_encode = fast_encode_xlm(olid_train_2k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_train_5k_encode = fast_encode_xlm(olid_train_5k.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_valid_encode = fast_encode_xlm(olid_valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)
 olid_test_encode = fast_encode_xlm(olid_test.tweet.astype(str), tokenizer, maxlen = MAX_LEN)
<prepare_x_and_y>
base = ".. /input/donorschoose-application-screening/"
train = pd.read_csv(base + "train.csv")
 test = pd.read_csv(base + "test.csv")
 resources = pd.read_csv(base + "resources.csv")
 train = train.sort_values(by="project_submitted_datetime" )
y_olid_train = olid_train.toxic.values
y_olid_train_1k = olid_train_1k.toxic.values
y_olid_train_2k = olid_train_2k.toxic.values
y_olid_train_5k = olid_train_5k.toxic.values
y_olid_valid = olid_valid.toxic.values
y_olid_test =(olid_test_label.label == 'OFF' ).astype(int ).values<define_variables>
if COMPLETE_RUN == False:
train = train[:200]
test = test[:200]
resources = resources[:100]
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<categorify>
char_cols = ['project_title', 'project_essay_1', 'project_essay_2',
'project_essay_3', 'project_essay_4', 'project_resource_summary']
olid_test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(olid_test_encode)
.batch(BATCH_SIZE)
)
 olid_valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_valid_encode, y_olid_valid)) 
.batch(BATCH_SIZE)
)
 olid_train_1k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_1k_encode, y_olid_train_1k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_2k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_2k_encode, y_olid_train_2k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_5k_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_5k_encode, y_olid_train_5k)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 olid_train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( olid_train_encode, y_olid_train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
<choose_model_class>
resources['total_price'] = resources.quantity * resources.price
mean_total_price = pd.DataFrame(resources.groupby('id' ).total_price.mean())
 sum_total_price = pd.DataFrame(resources.groupby('id' ).total_price.sum())
 count_total_price = pd.DataFrame(resources.groupby('id' ).total_price.count())
 mean_total_price['id'] = mean_total_price.index
sum_total_price['id'] = mean_total_price.index
count_total_price['id'] = mean_total_price.index
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<load_pretrained>
def create_features(df):
df = pd.merge(df, mean_total_price, on='id')
 df = pd.merge(df, sum_total_price, on='id')
 df = pd.merge(df, count_total_price, on='id')
 df['year'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[0])
 df['month'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[1])
 for col in char_cols:
df[col] = df[col].fillna("NA")
 df['text'] = df.apply(lambda x: " ".join(x[col] for col in char_cols), axis=1)
 return df
%%time
with strategy.scope() :
transformer_layer =(
 transformers.TFAutoModelWithLMHead.from_pretrained(model_name)
)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary() 
<train_model>
train = create_features(train)
 test = create_features(test )
n_steps = x_train.shape[0] // BATCH_SIZE
model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
cat_features = ["teacher_prefix", "school_state", "year", "month", "project_grade_category",
"project_subject_categories", "project_subject_subcategories"]
num_features = ["teacher_number_of_previously_posted_projects", "total_price_x",
"total_price_y", "total_price"]
cat_features_hash = [col+"_hash" for col in cat_features]
n_steps = x_valid.shape[0] // BATCH_SIZE
model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
def feature_hash(df, max_size=max_size):
for col in cat_features:
df[col+"_hash"] = df[col].apply(lambda x: hash(x)%max_size)
 return df
train = feature_hash(train)
 test = feature_hash(test )
sub['toxic'] = model.predict(test_dataset, verbose=0)
 sub.to_csv('submission.csv', index=False )<import_modules>
scaler = StandardScaler()
X_train_num = scaler.fit_transform(train[num_features])
 X_test_num = scaler.transform(test[num_features])
 X_train_cat = np.array(train[cat_features_hash], dtype=np.int)
 X_test_cat = np.array(test[cat_features_hash], dtype=np.int)
 tokenizer = text.Tokenizer(num_words=max_features )
from sklearn.metrics import roc_auc_score<save_to_csv>
def preprocess(string):

 string = re.sub(r'(")', ' ', string)
 string = re.sub(r'(\r)', ' ', string)
 string = re.sub(r'(
)', ' ', string)
 string = re.sub(r'(\r 
)', ' ', string)
 string = re.sub(r'(\\)', ' ', string)
 string = re.sub(r'\t', ' ', string)
 string = re.sub(r'\:', ' ', string)
 string = re.sub(r'
olid_test.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test.to_csv('olid_test_0shot.csv', index=False)
 roc_auc_score(y_true = olid_test.toxic, y_score = olid_test.toxic_predict )<compute_train_metric>
tokenizer.fit_on_texts(train["text"].tolist() +test["text"].tolist())
 list_tokenized_train = tokenizer.texts_to_sequences(train["text"].tolist())
 list_tokenized_test = tokenizer.texts_to_sequences(test["text"].tolist())
 X_train_words = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_test_words = sequence.pad_sequences(list_tokenized_test, maxlen=maxlen)
 X_train_target = train.project_is_approved
def Find_Optimal_Cutoff(target, predicted):

 fpr, tpr, threshold = roc_curve(target, predicted)
 i = np.arange(len(tpr))
roc = pd.DataFrame({'tf' : pd.Series(tpr-(1-fpr), index=i), 'threshold' : pd.Series(threshold, index=i)})
 roc_t = roc.iloc[(roc.tf-0 ).abs().argsort() [:1]]
return list(roc_t['threshold'])
 threshold = Find_Optimal_Cutoff(olid_test.toxic, olid_test.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test.toxic_predict]<compute_test_metric>
embeddings_index = {}
with open(embedding_file, encoding='utf8')as f:
for line in f:
values = line.rstrip().rsplit(' ')
 word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
 embeddings_index[word] = coefs
f1_score(y_true = olid_test.toxic, y_pred = olid_test.toxic_predict_binary)
<load_pretrained>
word_index = tokenizer.word_index
num_words = min(max_features, len(word_index)+ 1)
 embedding_matrix = np.zeros(( num_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
model.save_weights("/kaggle/working/ckpt.h5")
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
GlobalMaxPool1D, GlobalAveragePooling1D, SpatialDropout1D, CuDNNGRU, Bidirectional, GRU, BatchNormalization
with strategy.scope() :
model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 n_steps = olid_train_1k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_1k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test1k = olid_test
olid_test1k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test1k.to_csv('olid_test_1k.csv', index=False)
 print('1k roc is ' + str(roc_auc_score(y_true = olid_test1k.toxic, y_score = olid_test1k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test1k.toxic, olid_test1k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test1k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test1k.toxic_predict]
print('1k f1-score is ' + str(f1_score(y_true = olid_test1k.toxic, y_pred = olid_test1k.toxic_predict_binary)))
<load_pretrained>
recurrent_units = 96
convolution_filters = 128
dense_units = [256, 128, 64]
dropout_rate = 0.3
learning_rate = 5e-3
if COMPLETE_RUN == False:
recurrent_units = 8
convolution_filters = 8
dense_units = [8, 4, 4]
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
def get_model() :
input_cat = Input(( len(cat_features_hash),))
input_num = Input(( len(num_features),))
input_words = Input(( maxlen,))
x_cat = Embedding(max_size, 20 )(input_cat)
 x_cat = SpatialDropout1D(dropout_rate )(x_cat)
 x_cat = Flatten()(x_cat)
 x_words = Embedding(max_features, 300,
weights=[embedding_matrix],
trainable=False )(input_words)
 x_words = SpatialDropout1D(dropout_rate )(x_words)
 x_words1 = Bidirectional(GRU(recurrent_units, return_sequences=True))(x_words)
 x_words1 = Convolution1D(convolution_filters, 3, activation="relu" )(x_words1)
 x_words1_1 = GlobalMaxPool1D()(x_words1)
 x_words1_2 = GlobalAveragePooling1D()(x_words1)
 x_words2 = Convolution1D(convolution_filters, 2, activation="relu" )(x_words)
 x_words2 = Convolution1D(convolution_filters, 2, activation="relu" )(x_words2)
 x_words2_1 = GlobalMaxPool1D()(x_words2)
 x_words2_2 = GlobalAveragePooling1D()(x_words2)
 x_num = input_num
x = concatenate([x_words1_1, x_words1_2, x_words2_1, x_words2_2, x_cat, x_num])
 x = BatchNormalization()(x)
 x = Dense(dense_units[0], activation="relu" )(x)
 x = Dense(dense_units[1], activation="relu" )(x)
 x = concatenate([x, x_num])
 x = Dense(dense_units[2], activation="relu" )(x)
 predictions = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_cat, input_num, input_words], outputs=predictions)
 model.compile(optimizer=optimizers.Adam(learning_rate, decay=1e-6),
loss='binary_crossentropy',
metrics=['accuracy'])
 return model
n_steps = olid_train_2k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_2k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test2k = olid_test
olid_test2k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test2k.to_csv('olid_test_2k.csv', index=False)
 print('2k roc is ' + str(roc_auc_score(y_true = olid_test2k.toxic, y_score = olid_test2k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test2k.toxic, olid_test2k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test2k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test2k.toxic_predict]
print('2k f1-score is ' + str(f1_score(y_true = olid_test2k.toxic, y_pred = olid_test2k.toxic_predict_binary)))
<load_pretrained>
model = get_model()
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
file_path = 'best.h5'
n_steps = olid_train_5k.shape[0] // BATCH_SIZE
model.fit(
 olid_train_5k_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test5k = olid_test
olid_test5k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test5k.to_csv('olid_test_5k.csv', index=False)
 print('5k roc is ' + str(roc_auc_score(y_true = olid_test5k.toxic, y_score = olid_test5k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test5k.toxic, olid_test5k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test5k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test5k.toxic_predict]
print('5k f1-score is ' + str(f1_score(y_true = olid_test5k.toxic, y_pred = olid_test5k.toxic_predict_binary)) )<load_pretrained>
checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1, save_best_only=True,
save_weights_only=True, mode='min')
 early = EarlyStopping(monitor="val_loss", mode="min", patience=4)
 lr_reduced = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, verbose=1,
epsilon=1e-4, mode='min')
 callbacks_list = [checkpoint, early, lr_reduced]
history = model.fit([X_train_cat, X_train_num, X_train_words], X_train_target, validation_split=0.1,
verbose=1, callbacks=callbacks_list, epochs=5, batch_size=256 )
tf.tpu.experimental.initialize_tpu_system(tpu)
 with strategy.scope() :
model.load_weights("/kaggle/working/ckpt.h5")
 model.compile(Adam(lr=2e-6), loss='binary_crossentropy', metrics=['accuracy'])
 model.predict(olid_test_dataset, verbose=0)[0:5]<save_to_csv>
model.load_weights(file_path)
 pred_test = model.predict([X_test_cat, X_test_num, X_test_words], batch_size=1024, verbose=1)
 test["project_is_approved"] = pred_test
if COMPLETE_RUN:
test[['id', 'project_is_approved']].to_csv("submission.csv", index=False )
n_steps = olid_train.shape[0] // BATCH_SIZE
model.fit(
 olid_train_dataset.repeat() ,
steps_per_epoch=n_steps,
validation_data=olid_valid_dataset,
callbacks = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True),
epochs=10 
)
 olid_test10k = olid_test
olid_test10k.toxic_predict = model.predict(olid_test_dataset, verbose=0)
 olid_test10k.to_csv('olid_test_10k.csv', index=False)
 print('10k roc is ' + str(roc_auc_score(y_true = olid_test10k.toxic, y_score = olid_test10k.toxic_predict)))
 threshold = Find_Optimal_Cutoff(olid_test10k.toxic, olid_test10k.toxic_predict)
 print("the optimal threshold is " + str(threshold[0]))
olid_test10k.toxic_predict_binary = [1 if p > threshold[0] else 0 for p in olid_test10k.toxic_predict]
print('10k f1-score is ' + str(f1_score(y_true = olid_test10k.toxic, y_pred = olid_test10k.toxic_predict_binary)))
<import_modules>
LABELS = ["project_is_approved"]
base = ".. /input/the-all-in-one-model-prediction-files/"
predict_list = []
for j in range(10):
predict_list.append(pd.read_csv(base + "submission_%d.csv"%j)[LABELS])
 predcitions = np.zeros_like(predict_list[0])
 for predict in predict_list:
predcitions = np.add(predcitions, logit(predict))
predcitions /= len(predict_list)
 predcitions = expit(predcitions)
 submission = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 submission[LABELS] = predcitions
submission.to_csv('submission.csv', index=False )
<import_modules><EOS> 
os.remove('best.h5' )