Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<define_variables>
stop = set(stopwords.words('english'))
tf.logging.set_verbosity(tf.logging.ERROR)
 pd.options.display.max_rows = 10
pd.options.display.float_format = '{:.1f}'.format
n_total_train_samples = featured_data_for_training[1].shape[0]
n_steps_per_epoch = int(np.ceil(n_total_train_samples / float(batch_size_for_cls)) )<choose_model_class>
train = pd.read_csv(".. /input/train.csv", sep=",", parse_dates=['project_submitted_datetime'])
 resources = pd.read_csv(".. /input/resources.csv", sep=",")
 test = pd.read_csv(".. /input/test.csv", sep=",", parse_dates=['project_submitted_datetime'] )
with strategy.scope() :
final_bayesian_classifier = build_classifier(
 feature_vector_size=featured_data_for_training[0].shape[1],
n_train_samples=max_iters_of_cls,
hidden_layer_size=hidden_layer_of_cls,
n_hidden_layers=bnn_params['neural_depth'],
C=bnn_params['C'], language='multilang', verbose=True
)<define_variables>
resources['cost'] = resources['quantity'] * resources['price']
resources_aggregated = resources.groupby('id' ).agg({'description': ['nunique'], 'quantity': ['sum'], 'cost': ['mean', 'sum']})
 resources_aggregated.columns = ['unique_items', 'total_quantity', 'mean_cost', 'total_cost']
resources_aggregated.reset_index(inplace=True)
 train = pd.merge(train, resources_aggregated, how='left', on='id')
 test = pd.merge(test, resources_aggregated, how='left', on='id' )
train_classifier(
 trainset=tf.data.Dataset.from_tensor_slices(
 featured_data_for_training
).repeat().shuffle(n_total_train_samples ).batch(batch_size_for_cls),
n_steps=n_steps_per_epoch,
bayesian_classifier=final_bayesian_classifier, verbose=True,
max_epochs=int(np.ceil(max_iters_of_cls / float(n_total_train_samples))),\
tmp_file_name=tmp_cls_name
)<predict_on_test>
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
train.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
test.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 train['project_essay_1'] = train['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_essay_2'] = train['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_1'] = test['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_2'] = test['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_resource_summary'] = train['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_title'] = train['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_resource_summary'] = test['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_title'] = test['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
result_of_submission = predict_with_model(
 classifier=final_bayesian_classifier,
input_data=featured_data_for_submission,
batch_size=batch_size_for_cls,
n_monte_carlo=final_num_monte_carlo
)<save_to_csv>
def get_quantile_based_boundaries(feature_values, num_buckets):
boundaries = np.arange(1.0, num_buckets)/ num_buckets
quantiles = feature_values.quantile(boundaries)
 return [quantiles[q] for q in quantiles.keys() ]
with codecs.open('submission.csv', mode='w', encoding='utf-8', errors='ignore')as fp:
fp.write('id,toxic
')
 for sample_idx in range(identifies_for_submission.shape[0]):
id_val = identifies_for_submission[sample_idx]
proba_val = result_of_submission[sample_idx]
fp.write('{0},{1:.9f}
'.format(id_val, proba_val))<train_model>
bins = [0., 2., 10.,]
test["experience"] = np.digitize(test['teacher_number_of_previously_posted_projects'], bins=bins)
 quantiles = get_quantile_based_boundaries(test['unique_items'], 6)
 test["unique_items_binned"] = np.digitize(test['unique_items'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_quantity'], 6)
 test["total_quantity_binned"] = np.digitize(test['total_quantity'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['mean_cost'], 6)
 test["mean_cost_binned"] = np.digitize(test['mean_cost'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_cost'], 6)
 test["total_cost_binned"] = np.digitize(test['total_cost'], bins=quantiles )
print('Experiment duration is {0:.3f}.'.format(time.time() - experiment_start_time))<import_modules>
def preprocess_features(df):
selected_features = df[
["project_subject_categories",
"project_subject_subcategories",
"project_title",
"project_resource_summary",
"project_essay_1",
"project_essay_2",
"experience",
"unique_items_binned",
"total_quantity_binned",
"mean_cost_binned",
"total_cost_binned",
"encoded_category",
"encoded_subcategory"]]
return selected_features
def preprocess_targets(df):
output_targets = df["project_is_approved"]
return output_targets
import os, re
import tensorflow as tf
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow.keras.layers as L
import tensorflow.keras.optimizers as Optim
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):
assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return ft_trn_series, ft_tst_series
def fast_encode(texts, tokenizer, chunk_size = 256, maxlen = 512):
tokenizer.enable_truncation(max_lenght = maxlen)
 tokenizer.enable_padding(max_length = maxlen)
 all_ids = []
for i in tqdm(range(0, len(texts), chunk_size)) :
text_chunk = texts[i:i+chunk_size].tolist()
encs = tocanizer.encode_batch(text_chunk)
 all_ids.extend([enc.ids for enc in encs])
 return np.array(all_ids )<categorify>
train['encoded_category'], test['encoded_category'] = target_encode(train['project_subject_categories'], test['project_subject_categories'], train['project_is_approved'])
 train['encoded_subcategory'], test['encoded_subcategory'] = target_encode(train['project_subject_subcategories'], test['project_subject_subcategories'], train['project_is_approved'] )
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'])
<choose_model_class>
train = train.reindex(
 np.random.permutation(train.index))
def build_model(transformer, max_len = 512):
input_word_ids = L.Input(shape =(max_len,), dtype = tf.int32, name = 'input_word_ids')
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = L.Dense(1, activation = 'sigmoid' )(cls_token)
 model = Model(inputs = input_word_ids, outputs = out)
 model.compile(Optim.Adam(lr = 1e-5), loss = 'binary_crossentropy', metrics = ['accuracy'])
 return model<set_options>
training_examples = preprocess_features(train.head(122080))
training_targets = preprocess_targets(train.head(122080))
validation_examples = preprocess_features(train.tail(60000))
validation_targets = preprocess_targets(train.tail(60000))
test_examples = preprocess_features(test.copy() )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
vectorizer=TfidfVectorizer(stop_words=stop)
 vectorizer.fit(train['project_subject_categories'])
 train_project_subject_categories = vectorizer.transform(training_examples['project_subject_categories'])
 validation_project_subject_categories = vectorizer.transform(validation_examples['project_subject_categories'])
 test_project_subject_categories = vectorizer.transform(test_examples['project_subject_categories'])
 vectorizer.fit(train['project_subject_subcategories'])
 train_project_subject_subcategories = vectorizer.transform(training_examples['project_subject_subcategories'])
 validation_project_subject_subcategories = vectorizer.transform(validation_examples['project_subject_subcategories'])
 test_project_subject_subcategories = vectorizer.transform(test_examples['project_subject_subcategories'] )
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
TRAIN_EPOCHS = 5
VALID_EPOCHS = 5
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'<load_pretrained>
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 2), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_title'])
 train_project_title = vectorizer.transform(training_examples['project_title'])
 validation_project_title = vectorizer.transform(validation_examples['project_title'])
 test_project_title = vectorizer.transform(test_examples['project_title'])
 vectorizer.fit(train['project_resource_summary'])
 train_project_resource_summary = vectorizer.transform(training_examples['project_resource_summary'])
 validation_project_resource_summary = vectorizer.transform(validation_examples['project_resource_summary'])
 test_project_resource_summary = vectorizer.transform(test_examples['project_resource_summary'] )
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 3), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_essay_1'])
 train_project_essay_1 = vectorizer.transform(training_examples['project_essay_1'])
 validation_project_essay_1 = vectorizer.transform(validation_examples['project_essay_1'])
 test_project_essay_1 = vectorizer.transform(test_examples['project_essay_1'])
 vectorizer.fit(train['project_essay_2'])
 train_project_essay_2 = vectorizer.transform(training_examples['project_essay_2'])
 validation_project_essay_2 = vectorizer.transform(validation_examples['project_essay_2'])
 test_project_essay_2 = vectorizer.transform(test_examples['project_essay_2'] )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
def get_polarity(text):
textblob = TextBlob(text)
 pol = textblob.sentiment.polarity
return round(pol,3)
 def get_subjectivity(text):
textblob = TextBlob(text)
 subj = textblob.sentiment.subjectivity
return round(subj,3 )
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ])
<prepare_x_and_y>
training_examples['polarity1'] = training_examples['project_essay_1'].apply(get_polarity)
 training_examples['subjectivity1'] = training_examples['project_essay_1'].apply(get_subjectivity)
 training_examples['polarity2'] = training_examples['project_essay_2'].apply(get_polarity)
 training_examples['subjectivity2'] = training_examples['project_essay_2'].apply(get_subjectivity )
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen = MAX_LEN)
 x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen = MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values<train_model>
validation_examples['polarity1'] = validation_examples['project_essay_1'].apply(get_polarity)
 validation_examples['subjectivity1'] = validation_examples['project_essay_1'].apply(get_subjectivity)
 validation_examples['polarity2'] = validation_examples['project_essay_2'].apply(get_polarity)
 validation_examples['subjectivity2'] = validation_examples['project_essay_2'].apply(get_subjectivity )
train_dataset =(tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(2048)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.cache() 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
<load_pretrained>
test_examples['polarity1'] = test_examples['project_essay_1'].apply(get_polarity)
 test_examples['subjectivity1'] = test_examples['project_essay_1'].apply(get_subjectivity)
 test_examples['polarity2'] = test_examples['project_essay_2'].apply(get_polarity)
 test_examples['subjectivity2'] = test_examples['project_essay_2'].apply(get_subjectivity )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
to_drop = ['project_subject_categories', 'project_subject_subcategories', 'project_title', 'project_essay_1', 'project_essay_2', 'project_resource_summary']
for col in to_drop:
training_examples.drop([col], axis=1, inplace=True)
 validation_examples.drop([col], axis=1, inplace=True)
 test_examples.drop([col], axis=1, inplace=True )
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=5
)<train_model>
training_features = csr_matrix(hstack([training_examples.values, train_project_subject_categories, train_project_subject_subcategories, train_project_title, train_project_resource_summary, train_project_essay_1, train_project_essay_2]))
validation_features = csr_matrix(hstack([validation_examples.values, validation_project_subject_categories, validation_project_subject_subcategories, validation_project_title, validation_project_resource_summary, validation_project_essay_1, validation_project_essay_2]))
test_features = csr_matrix(hstack([test_examples.values, test_project_subject_categories, test_project_subject_subcategories, test_project_title, test_project_resource_summary, test_project_essay_1, test_project_essay_2]))
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=5
)<load_pretrained>
model.save_weights('UnToxik_V0-1.h5' )<load_pretrained>
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 16,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': 0,
'num_threads': 1,
'lambda_l2': 1,
'min_gain_to_split': 0,
}
model2 = lgb.train(
 params,
lgb.Dataset(training_features, training_targets),
num_boost_round=10000,
valid_sets=[lgb.Dataset(validation_features, validation_targets)],
verbose_eval=100,
early_stopping_rounds=100
)
<import_modules><EOS> 
submission = pd.DataFrame()
submission['id'] = test['id']
submission['project_is_approved'] = model2.predict(test_features, num_iteration=model2.best_iteration)
 submission.to_csv('submission.csv', index=False )