Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<prepare_x_and_y>
plt.style.use('Solarize_Light2')
 %matplotlib inline
pd.set_option("display.max_columns", 2**10 )
x_train = regular_encode(df_train['comment_text'].values, tokenizer, maxlen=MAX_LEN)
 x_valid = regular_encode(valid['comment_text'].values, tokenizer, maxlen=MAX_LEN)
 x_test = regular_encode(test['content'].values, tokenizer, maxlen=MAX_LEN)
 y_train = df_train.toxic.values
y_valid = valid.toxic.values 
<split>
date_cols = ['project_submitted_datetime']
train = pd.read_csv(os.path.join(r'.. /input', 'train.csv'), low_memory=False, parse_dates=date_cols)
 test = pd.read_csv(os.path.join(r'.. /input', 'test.csv'), low_memory=False, parse_dates=date_cols )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.shuffle(len(y_train)) 
.batch(BATCH_SIZE)
.repeat() 
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
 )<load_pretrained>
train['source'] = 'train'
test['source'] = 'test'
test_train = pd.concat(( test, train))
print(f'The shape of the data is {test_train.shape}.' )
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
print("="*60)
 print("Detecting NaN values in data:")
 print("="*60)
 print(test_train.isnull().sum(axis=0)[test_train.isnull().sum(axis=0)> 0] )
n_steps = df_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
shuffle=False,
)<train_model>
most_common = test_train.teacher_prefix.value_counts().idxmax()
test_train.teacher_prefix = test_train.teacher_prefix.fillna(most_common )
n_steps = valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=4,
shuffle=False,
)<save_to_csv>
numerical_cols = []
dummy_categorical_cols = []
text_cols = []
sub['toxic'] = model.predict(x_test, verbose=1)
 sub.to_csv('submission.csv', index=False)
 model.save_weights('checkpoint.h5', overwrite=True)
 sub.head(55 )<set_options>
mask_four_essays = ~(test_train.project_essay_3.isnull() & test_train.project_essay_4.isnull())
 test_train[mask_four_essays] =(test_train[mask_four_essays] 
.assign(project_essay_1 = lambda df: df.project_essay_1 + df.project_essay_2)
.assign(project_essay_2 = lambda df: df.project_essay_3 + df.project_essay_4))
test_train = test_train.drop(columns=['project_essay_3', 'project_essay_4'] )
warnings.filterwarnings("ignore")
<categorify>
resources = pd.read_csv(os.path.join(r'.. /input', 'resources.csv'), low_memory=False)
 print(f'The shape of the data is {resources.shape}.' )
def clean_text(text):
text = str(text)
 text = re.sub(r'[0-9"]', '', text)
 text = re.sub(r'
text = re.sub(r'@[\S]+\b', '', text)
 text = re.sub(r'https?\S+', '', text)
 text = re.sub(r'\s+', ' ', text)
 return text.strip()<string_transform>
print("="*30)
 print("Detecting NaN values in data:")
 print("="*30)
 print(resources.isnull().sum(axis=0)[resources.isnull().sum(axis=0)> 0] )
def text_process(text):
ws = text.split(' ')
 if(len(ws)>160):
text = ' '.join(ws[:160])+ ' ' + ' '.join(ws[-32:])
 return text<categorify>
resources = resources.fillna('X' )
def dict_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_mask=True,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
truncation=True 
)
 return {
"input_ids": np.array(enc_di['input_ids']),
"attention_mask": np.array(enc_di['attention_mask'])
 }<load_pretrained>
print("="*30)
 print("Detecting NaN values in data:")
 print("="*30)
 print(resource_stats.isnull().sum(axis=0)[resource_stats.isnull().sum(axis=0)> 0] )
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )<set_options>
resource_stats.columns = ['_'.join([col, func])for col, func in resource_stats.columns.values]
numerical_cols += list(resource_stats.columns.values )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
test_train = test_train.merge(resource_stats, how='left', left_on='id', right_index=True)
 test_train.sample(1 ).T.tail(6 )
MODEL = 'jplu/tf-xlm-roberta-large'
AUTO = tf.data.experimental.AUTOTUNE
SEED = 2020
EPOCHS_1 = 20
EPOCHS_2 = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
SHUFFLE = 2048
VERBOSE = 1<load_from_csv>
test_train['month'] = test_train.project_submitted_datetime.dt.month.apply(str)
 test_train['daytime'] = pd.Series(np.where(
(( 7 <= test_train.project_submitted_datetime.dt.hour)& 
(test_train.project_submitted_datetime.dt.hour <= 10)) , 1, 0)).apply(str )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2 = pd.concat([train2[['comment_text', 'toxic']].query('toxic<=0.4'),
train2[['comment_text', 'toxic']].query('toxic>=0.6')])
 train2.toxic = train2.toxic.round().astype(int)
 train2 = train2.sample(frac = 1, random_state=SEED)
 train3 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv")
 train4 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv")
 train5 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv")
 train6 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv")
 train7 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv")
 train8 = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 valid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 valid2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')
 valid2.rename(columns={'comment_text': 'old_comment_text', 'translated': 'comment_text'}, inplace=True)
 test = pd.read_csv('.. /input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 test2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<prepare_x_and_y>
dummy_colnames = ['teacher_prefix', 'month', 'school_state', 'daytime']
dummies = pd.get_dummies(test_train.loc[:, dummy_colnames])
 dummy_categorical_cols += dummies.columns.tolist()
test_train = pd.concat(( test_train, dummies), axis=1 )
%%time
x_train = dict_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_valid = dict_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 x_test1 = dict_encode(test1.content.values, tokenizer, maxlen=MAX_LEN)
 x_test2 = dict_encode(test2.translated.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
y_valid = valid.toxic.values
del train, valid
gc.collect()<categorify>
project_cat_colnames = ['project_subject_categories', 'project_subject_subcategories']
df_cats = test_train.loc[:, project_cat_colnames]
for category in unique_categories:
df_cats[category] = np.where(df_cats.project_subject_categories.str.contains(category), 1, 0)
 for category in unique_subcategories:
df_cats[category] = np.where(df_cats.project_subject_subcategories.str.contains(category), 1, 0)
 df_cats = df_cats.drop(columns=project_cat_colnames)
 df_cats.head(1 ).T.head(5 )
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(SHUFFLE)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test1_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test1)
.batch(BATCH_SIZE)
)
 test2_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test2)
.batch(BATCH_SIZE)
)
 del x_train, x_valid, y_train, y_valid, x_test1, x_test2
gc.collect()<choose_model_class>
test_train = pd.concat(( test_train, df_cats), axis=1)
 print(f'The dataset now has ~{len(test_train.columns)} features.')
 test_train.head(1 )
lrs = ReduceLROnPlateau(monitor='val_accuracy', mode ='max', factor = 0.7, min_lr= 1e-7, verbose = 1, patience = 2)
 es1 = EarlyStopping(monitor='val_accuracy', mode='max', verbose = 1, patience = 5, restore_best_weights=True)
 es2 = EarlyStopping(monitor='accuracy', mode='max', verbose = 1, patience = 1, restore_best_weights=True)
 callbacks_list1 = [lrs,es1]
callbacks_list2 = [lrs,es2]<choose_model_class>
text_cols += ['project_essay_1', 'project_essay_2', 'project_resource_summary', 'description_concatenate']
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 attention_mask = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 sequence_output = transformer({"input_ids": input_word_ids, "attention_mask": attention_mask})[0]
avg_pool = GlobalAveragePooling1D()(sequence_output)
 max_pool = GlobalMaxPooling1D()(sequence_output)
 cls_token = Concatenate()([avg_pool, max_pool])
 samples = []
for n in range(2):
sample = Dropout(0.4 )(cls_token)
 sample = Dense(1, activation='sigmoid', name=f'sample_{n}' )(sample)
 samples.append(sample)
 out = Average(name='output' )(samples)
 model = Model(inputs={
"input_ids": input_word_ids,
"attention_mask": attention_mask
},
outputs=out,
name='XLM-R')
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(name='auc'), 'accuracy'])
 return model<load_pretrained>
%%time
subj = lambda x: TextBlob(x ).sentiment.subjectivity
polar = lambda x: TextBlob(x ).sentiment.polarity
test_train['description_subjectivity'] = test_train['description_concatenate'].apply(subj)
 test_train['description_polarity'] = test_train['description_concatenate'].apply(polar)
 test_train['project_resource_summary_subjectivity'] = test_train['project_resource_summary'].apply(subj)
 test_train['project_resource_summary_polarity'] = test_train['project_resource_summary'].apply(polar)
 test_train['project_essay_2_polarity'] = test_train['project_essay_2'].apply(polar)
 test_train['project_essay_1_polarity'] = test_train['project_essay_1'].apply(polar )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
text_cols += ['project_title']
print(text_cols )
model_history_1 = model.fit(
 train_dataset,
steps_per_epoch=n_train_steps,
validation_data=valid_dataset,
epochs=EPOCHS_1,
callbacks=callbacks_list1,
verbose=VERBOSE
)<train_model>
%%time
vectorizer = CountVectorizer()
for text_col in text_cols:
X = vectorizer.fit_transform(test_train[text_col].values)
 col_new_name = text_col.replace('project_', '')
 unique_words =(X > 0 ).sum(axis=1)
 num_words =(X ).sum(axis=1)
 test_train[col_new_name + '_unique_words'] = unique_words
test_train[col_new_name + '_num_words'] = num_words
test_train[col_new_name + '_vocab'] = np.exp(unique_words /(num_words + 10e-10))
model_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_valid_steps,
epochs=EPOCHS_2,
callbacks=callbacks_list2,
verbose=VERBOSE
)<save_to_csv>
for text_col in text_cols:
test_train[text_col] = test_train[text_col].str.replace('"', ' ')
sub.to_csv('submission.csv', index=False )<save_to_csv>
sub.to_csv('submission.csv', index=False )<load_from_csv>
numeric_cols = [
'teacher_number_of_previously_posted_projects',
'description_nunique',
'quantity_sum',
'price_sum',
'price_mean',
'p_desc_len_mean',
'p_desc_len_amin',
'p_desc_len_amax',
'p_total_price_mean',
'p_total_price_amin',
'p_total_price_amax',
'project_resource_summary_subjectivity',
'project_resource_summary_polarity',
'project_essay_2_polarity',
'project_essay_1_polarity',
'essay_1_unique_words',
'essay_1_num_words',
'essay_1_vocab',
'essay_2_unique_words',
'essay_2_num_words',
'essay_2_vocab',
'resource_summary_unique_words',
'resource_summary_num_words',
'resource_summary_vocab',
'description_concatenate_unique_words',
'description_concatenate_num_words',
'description_concatenate_vocab',
'title_unique_words',
'title_num_words',
'title_vocab']
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<categorify>
from tempfile import mkdtemp
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.decomposition import TruncatedSVD
from sklearn.metrics import roc_auc_score, roc_curve, auc
from scipy.sparse import coo_matrix, hstack
from sklearn.preprocessing import RobustScaler
def clean(text):
text = text.fillna("fillna" ).str.lower()
text = text.map(lambda x: re.sub('\
',' ',str(x)))
 text = text.map(lambda x: re.sub("\[\[User.*",'',str(x)))
 text = text.map(lambda x: re.sub("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}",'',str(x)))
 text = text.map(lambda x: re.sub("\(http://.*?\s\(http://.*\)",'',str(x)))
 return text<categorify>
class ColSplitter(BaseEstimator, TransformerMixin):

 def __init__(self, cols, ravel=False):
self.cols = cols
self.ravel = ravel
def fit(self, x, y=None):
return self
def transform(self, x):
return(x.loc[:, self.cols].values.ravel() if self.ravel
else x.loc[:, self.cols].values)
 class LogTransform(BaseEstimator, TransformerMixin):

 def __init__(self, alpha):

 self.alpha = alpha
def fit(self, x, y=None):
return self
def transform(self, x):
return self.alpha * np.log(np.log(x + 2.001)+ 1.001)+(1 - self.alpha)* x
class ParallelPipe(BaseEstimator, TransformerMixin):

 def __init__(self, pipes, *args, **kwargs):
self.pipes = pipes
def fit(self, x, y=None):
[p.fit(x)for p in self.pipes]
return self
def transform(self, x):
try:
return hstack([p.transform(x)for p in self.pipes])
 except:
return np.concatenate(tuple([p.transform(x)for p in self.pipes]), axis=1)
 def _get_param_names(self, *args, **kwargs):
return ['pipes']
def set_params(self, *args, **kwargs):
[p.set_params(*args, **kwargs)for p in self.pipes]
return None
train1["comment_text"] = clean(train1["comment_text"])
 train2["comment_text"] = clean(train2["comment_text"])
 valid["comment_text"] = clean(valid["comment_text"])
 test["content"] = clean(test["content"] )<choose_model_class>
assert np.all(np.isfinite(test_train[numeric_cols].values))
spell = SpellChecker()<feature_engineering>
param_grid = {"estimator__C": np.linspace(0.24285-0.1, 0.24285+0.1, num=6)
 }
grid_search = GridSearchCV(pipeline_logreg, param_grid=param_grid,
scoring='roc_auc',
n_jobs=1,
verbose=1,
cv=3 )
%%time
train1['mispell_count'] = train1['comment_text'].progress_apply(lambda x: len(spell.unknown(x.split())))
 train2['mispell_count'] = train2['comment_text'].progress_apply(lambda x: len(spell.unknown(x.split())) )<filter>
n = 25000
subset_A = test_train.loc[lambda df:(df.project_is_approved == 1)].sample(n)
 subset_B = test_train.loc[lambda df:(df.project_is_approved == 0)].sample(n)
 test_train_subset = pd.concat(( subset_A, subset_B))
test_X = test_train_subset
test_y = test_X.project_is_approved
train1 = train1.query('toxic==0 & mispell_count < 20 | toxic==1')
 train2 = train2.query('toxic==0 & mispell_count < 20 | toxic==1' )<categorify>
perform_grid_search = True
if perform_grid_search:
with warnings.catch_warnings() :
warnings.simplefilter("ignore", category=UserWarning)
 grid_search.fit(test_X, test_y.values.ravel())
 best_estimator = grid_search.best_estimator_
print('Best score:', grid_search.best_score_)
 print('Best params:
', grid_search.best_params_)
 else:
best_estimator = pipeline_logreg
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'] )<load_pretrained>
full_train = test_train.loc[lambda df: df.source == 'train']
y_pred = best_estimator.fit(full_train, full_train.project_is_approved.ravel() )
MODEL = 'jplu/tf-xlm-roberta-large'
MAX_LEN = 192
tokenizer = AutoTokenizer.from_pretrained(MODEL )<categorify>
test = test_train[test_train.source == 'test']
test.loc[:,('project_is_approved',)] = best_estimator.predict_proba(test)[:, 1]
<prepare_x_and_y><EOS> 
test[['id','project_is_approved']].to_csv('submission.csv', index=False )