Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<set_options>
training_dataset = pd.read_csv('.. /input/donorschoose-application-screening/train.csv', sep=',')
 resources_dataset = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv', sep=',')
 test_dataset = pd.read_csv('.. /input/donorschoose-application-screening/test.csv', sep=',' )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
dictionary_dataset = pd.read_csv('.. /input/english-word-frequency/unigram_freq.csv', sep=',')
 with open('.. /input/unix-words/words/en')as f:
content = f.readlines()
` at the end of each line
content = [x.strip() for x in content]
unix_set = set(content)
 USING_BROWN = False
if USING_BROWN:
if '/usr/share/nltk_data' in nltk.data.path:
nltk.data.path.remove('/usr/share/nltk_data')
 nltk.data.path.append('.. /input/brown-corpus/brown')
 nltk.data.path
brown = LazyCorpusLoader('brown', CategorizedTaggedCorpusReader, r'c[a-z]\d\d',
cat_file='cats.txt', tagset='brown', encoding="ascii",
nltk_data_subdir='brown-corpus/brown')
 list_of_lists_of_tuples = brown.tagged_sents()
list_sents = []
for list_of_tuples in list_of_lists_of_tuples:
sentence = " ".join([tupl[0] for tupl in list_of_tuples])
 list_sents.append(sentence)
 list_sents
pd_brown = pd.DataFrame({'sentence': list_sents})
 pd_brown['sentence'] = pd_brown['sentence'].replace('(\-)|(/)|(\.\.\.\.) |(\.\.\.) ', ' ', regex=True)
 pd_brown['sentence'] = pd_brown['sentence'].replace('["
pd_brown['sentence'] = pd_brown['sentence'].replace('() |() ', ' ', regex=True)
 pd_brown['sentence'] = pd_brown['sentence'].str.lower()
pd_brown[0:10]
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path()
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync<categorify>
resources_dataset.isnull().any()
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(len(train)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
 )<choose_model_class>
resources_dataset.loc[resources_dataset["description"].isnull() ]
def build_model(transformer, max_len=512):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy',
metrics=[
keras.metrics.TruePositives(name='tp'),
keras.metrics.FalsePositives(name='fp'),
keras.metrics.TrueNegatives(name='tn'),
keras.metrics.FalseNegatives(name='fn'),
keras.metrics.BinaryAccuracy(name='accuracy'),
keras.metrics.Precision(name='precision'),
keras.metrics.Recall(name='recall'),
keras.metrics.AUC(name='auc')
 ])
 return model<load_pretrained>
resources_dataset = resources_dataset.fillna({'description' : 'no_detail'})
 resources_dataset.isnull().any()
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<set_options>
resources_dataset['cost'] = resources_dataset['quantity'] * resources_dataset['price']
resources_dataset[0:9]
matplotlib.rcParams['figure.figsize'] =(12, 10 )<train_model>
grouped_ids = resources_dataset.groupby(['id'], as_index=False)
 resources_condensed = grouped_ids.agg({'cost' : 'sum'} ).rename(columns={'cost' : 'total_cost'})
 resources_condensed[69060:69065]
n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
group_the_ids = resources_dataset.groupby(['id'], as_index=False)
 all_condensed = grouped_ids.agg({'description' : lambda x: ' '.join(x),
'quantity' : 'sum',
'cost' : 'sum'} ).rename(columns={'description' : 'full_description',
'quantity' : 'total_quantity',
'cost' : 'total_cost'})
 all_condensed[69060:69065]
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_steps,
epochs=EPOCHS
)<save_to_csv>
entry = all_condensed.loc[all_condensed['id'] == 'p069063'].reset_index()
entry.loc[0, 'full_description']
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<install_modules>
combined_training_dataset = pd.merge(training_dataset, all_condensed, on='id')
 combined_training_dataset[0:5]
!pip install -U transformers<import_modules>
combined_test_dataset = pd.merge(test_dataset, all_condensed, on='id')
 combined_test_dataset[0:5]
import os
import gc
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors<categorify>
combined_training_dataset['teacher_prefix'] = combined_training_dataset['teacher_prefix'].fillna('none')
 combined_test_dataset['teacher_prefix'] = combined_test_dataset['teacher_prefix'].fillna('none')
 combined_training_dataset = combined_training_dataset.fillna('')
 combined_test_dataset = combined_test_dataset.fillna('' )
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_mask=False,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
truncation=True 
)
 return np.array(enc_di['input_ids'] )<categorify>
combined_training_dataset.loc[combined_training_dataset['project_essay_4'] != '', 'project_essay_1'] = combined_training_dataset['project_essay_1'] + ' ' + combined_training_dataset['project_essay_2']
combined_training_dataset.loc[combined_training_dataset['project_essay_4'] != '', 'project_essay_2'] = combined_training_dataset['project_essay_3'] + ' ' + combined_training_dataset['project_essay_4']
combined_test_dataset.loc[combined_test_dataset['project_essay_4'] != '', 'project_essay_1'] = combined_test_dataset['project_essay_1'] + ' ' + combined_test_dataset['project_essay_2']
combined_test_dataset.loc[combined_test_dataset['project_essay_4'] != '', 'project_essay_2'] = combined_test_dataset['project_essay_3'] + ' ' + combined_test_dataset['project_essay_4']
combined_training_dataset[16:20]
def dict_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_mask=True,
return_token_type_ids=False,
pad_to_max_length=True,
max_length=maxlen,
truncation=True 
)
 return {
"input_ids": np.array(enc_di['input_ids']),
"attention_mask": np.array(enc_di['attention_mask'])
 }<choose_model_class>
combined_training_dataset.loc[18, 'project_essay_2']
def build_model(transformer, max_len=512):

 input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_ids")
 attention_mask = Input(shape=(max_len,), dtype=tf.int32, name="attention_mask")
 sequence_output = transformer({"input_ids": input_word_ids, "attention_mask": attention_mask})[0]
cls_token = sequence_output[:, 0, :]
out = Dense(1, activation='sigmoid' )(cls_token)
 model = Model(inputs={
"input_ids": input_word_ids,
"attention_mask": attention_mask
}, outputs=out)
 model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
 return model<set_options>
USING_DICTIONARY_CODE = True
if USING_DICTIONARY_CODE == False:
the_essays = [
'project_essay_1',
'project_essay_2',
'full_description',
'project_resource_summary'
]
def tidy_essays(dataset):
for col_name in the_essays:
dataset[col_name] = dataset[col_name].replace(r'(\\r)|(\ 
)', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(r' ', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(r' ', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(r' ', ' ', regex=True)
 tidy_essays(combined_training_dataset)
 tidy_essays(combined_test_dataset)
 text_columns = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary',
'project_subject_categories',
'project_subject_subcategories',
'full_description'
]
punc_pattern = '["
def text_edits(dataset):
for col_name in text_columns:
dataset[col_name] = dataset[col_name].str.lower()
dataset[col_name] = dataset[col_name].replace(r'!', ' !', regex=True)
 dataset[col_name] = dataset[col_name].replace(r'(\-)|(/)', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace(punc_pattern, '', regex=True)
 text_edits(combined_training_dataset)
 text_edits(combined_test_dataset)
 combined_training_dataset[16:22]
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<load_pretrained>
if USING_DICTIONARY_CODE:
text_columns = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary',
'project_subject_categories',
'project_subject_subcategories',
'full_description'
]
def tidy_text(dataset):
for col_name in text_columns:
dataset[col_name] = dataset[col_name].replace('\?', '', regex=True)
 dataset[col_name] = dataset[col_name].str.encode('ascii', 'replace')
 dataset[col_name] = dataset[col_name].str.decode('utf8')
 dataset[col_name] = dataset[col_name].replace('([A-Z][.])+', '', regex=True)
 dataset[col_name] = dataset[col_name].str.lower()
in the text:
dataset[col_name] = dataset[col_name].replace('(\\\\r)|(\\\ 
)', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.com[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.org[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.gov[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.edu[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('[^ ]*\.net[^ ]*', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('
dataset[col_name] = dataset[col_name].replace('\\\', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('(\-)|(/)|(\.\.\.\.) |(\.\.\.) ', ' ', regex=True)
 dataset[col_name] = dataset[col_name].replace('=', '', regex=True)
 dataset[col_name] = dataset[col_name].replace('(?<=\w)[,.;!"](?=\w)', '=', regex=True)
 dataset[col_name] = dataset[col_name].replace('!', ' ! ', regex=True)
 dataset[col_name] = dataset[col_name].replace('["
dataset[col_name] = dataset[col_name].replace(' * *', ' ', regex=True)
 dataset[col_name] = dataset[col_name].str.strip()
return
tidy_text(combined_training_dataset)
 tidy_text(combined_test_dataset )
MAX_LEN = 192
MODEL = 'jplu/tf-xlm-roberta-large'
AUTO = tf.data.experimental.AUTOTUNE
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
if USING_DICTIONARY_CODE:
COLUMNS_TO_CHECK = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary'
]
def total_typing_errors(df):
new_col_names = []
for col in COLUMNS_TO_CHECK:
new_col_name = col + '_typing_errors'
new_col_name = new_col_name[8:]
df[new_col_name] = df[col].str.count('=')
 new_col_names.append(new_col_name)
 df['total_typing_errors'] = df[new_col_names].sum(axis=1)
 return
total_typing_errors(combined_training_dataset)
 total_typing_errors(combined_test_dataset)
 for col_name in text_columns:
combined_training_dataset[col_name] = combined_training_dataset[col_name].replace('=', ' ', regex=True)
 combined_test_dataset[col_name] = combined_test_dataset[col_name].replace('=', ' ', regex=True )
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv' ).sample(frac=1.0)
 n_valid_steps = valid.shape[0] // BATCH_SIZE
test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv' )<concatenate>
if USING_DICTIONARY_CODE:
join_cols = [
'project_title',
'project_essay_1',
'project_essay_2',
'project_resource_summary',
'full_description'
]
all_dataset = pd.concat([combined_training_dataset[join_cols], combined_test_dataset[join_cols]])
 def get_unique_word_sets(df):
new_set = set()
df.str.split().apply(new_set.update)
 return new_set
title_set = get_unique_word_sets(all_dataset['project_title'])
 essay_1_set = get_unique_word_sets(all_dataset['project_essay_1'])
 essay_2_set = get_unique_word_sets(all_dataset['project_essay_2'])
 summary_set = get_unique_word_sets(all_dataset['project_resource_summary'])
 description_set = get_unique_word_sets(all_dataset['full_description'])
 print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))
print(len(description_set))
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=100000, random_state=0)
 ] ).sample(frac=1.0)
 n_train_steps = train.shape[0] // BATCH_SIZE
del train1, train2
gc.collect()<set_options>
if USING_DICTIONARY_CODE:
dictionary_set = set(dictionary_dataset['word'])
 dictionary_set.add('!')
 print(len(dictionary_set))
!free -h<prepare_x_and_y>
if USING_DICTIONARY_CODE:
title_set = title_set - dictionary_set - unix_set
essay_1_set = essay_1_set - dictionary_set - unix_set
essay_2_set = essay_2_set - dictionary_set - unix_set
summary_set = summary_set - dictionary_set - unix_set
description_set = description_set - dictionary_set - unix_set
print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))
print(len(description_set))
%%time
x_train = dict_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
 y_train = train.toxic.values
del train
gc.collect()
x_valid = dict_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
 y_valid = valid.toxic.values
del valid
gc.collect()
x_test = dict_encode(test.content.values, tokenizer, maxlen=MAX_LEN )<set_options>
if USING_DICTIONARY_CODE:
title_set = title_set - description_set
essay_1_set = essay_1_set - description_set
essay_2_set = essay_2_set - description_set
summary_set = summary_set - description_set
print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))
!free -h<split>
if USING_DICTIONARY_CODE:
def make_set_alpha(word_set):
words_to_drop = set()
for word in word_set:
if word.isalpha() == False:
words_to_drop.add(word)
 return word_set - words_to_drop
title_set = make_set_alpha(title_set)
 essay_1_set = make_set_alpha(essay_1_set)
 essay_2_set = make_set_alpha(essay_2_set)
 summary_set = make_set_alpha(summary_set)
 print(len(title_set))
print(len(essay_1_set))
print(len(essay_2_set))
print(len(summary_set))
train_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_train, y_train)) 
.repeat() 
.shuffle(4096)
.batch(BATCH_SIZE)
.prefetch(AUTO)
)
 valid_dataset =(
 tf.data.Dataset 
.from_tensor_slices(( x_valid, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(x_test)
.batch(BATCH_SIZE)
)
 del x_train, x_valid, y_train, y_valid
gc.collect()<load_pretrained>
if USING_DICTIONARY_CODE:
COLS_AND_SETS = {
'project_title' : title_set,
'project_essay_1' : essay_1_set,
'project_essay_2' : essay_2_set,
'project_resource_summary' : summary_set
}
def count_spelling_errors(string, word_set):
list_words = string.split()
error_count = 0
for word in list_words:
if word in word_set:
error_count += 1
return error_count
def total_spelling_errors(df):
new_col_names = []
for col, word_set in COLS_AND_SETS.items() :
new_col_name = col + '_spelling_errors'
new_col_name = new_col_name[8:]
df[new_col_name] = df[col].apply(count_spelling_errors, args=(word_set,))
new_col_names.append(new_col_name)
 df['total_spelling_errors'] = df[new_col_names].sum(axis=1)
 return
total_spelling_errors(combined_training_dataset)
 total_spelling_errors(combined_test_dataset )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL)
 model = build_model(transformer_layer, max_len=MAX_LEN)
 model.summary()<train_model>
if USING_DICTIONARY_CODE:
combined_training_dataset.loc[combined_training_dataset['total_spelling_errors'] > 12]
EPOCHS = 2
train_history = model.fit(
 train_dataset,
steps_per_epoch=n_train_steps,
validation_data=valid_dataset,
epochs=EPOCHS
)<train_model>
combined_training_dataset['project_essay_2'].loc[110575]
train_history_2 = model.fit(
 valid_dataset.repeat() ,
steps_per_epoch=n_valid_steps,
epochs=EPOCHS
)<load_pretrained>
combined_training_dataset['project_is_approved'].loc[110575]
model.save_weights("final_weights.h5" )<save_to_csv>
combined_training_dataset['project_essay_2'].loc[172786]
sub['toxic'] = model.predict(test_dataset, verbose=1)
 sub.to_csv('submission.csv', index=False )<import_modules>
combined_training_dataset['project_submitted_datetime'] = pd.to_datetime(combined_training_dataset['project_submitted_datetime'])
 combined_training_dataset['year_week'] = combined_training_dataset['project_submitted_datetime'].map(lambda x: 100*(x.year - 2000)+ x.week ).apply(str)
 combined_test_dataset['project_submitted_datetime'] = pd.to_datetime(combined_test_dataset['project_submitted_datetime'])
 combined_test_dataset['year_week'] = combined_test_dataset['project_submitted_datetime'].map(lambda x: 100*(x.year - 2000)+ x.week ).apply(str)
 combined_training_dataset[0:5]
import os
import numpy as np
import pandas as pd
from scipy.special import softmax<define_variables>
import matplotlib.pyplot as plt
path = ".. /input/jigsaw-multilingual-toxic-comment-classification/"
record = ".. /input/buffer/"
base = record + "submission-9462.csv"
monos = ["submission-it-9467.csv",
"submission-pt-9470.csv",
"submission-es-9467.csv",
"submission-tr-9470.csv",
"submission-fr-9473.csv",]
get_lang = lambda x: x.split('-')[1]<load_from_csv>
grouped_prefixes = combined_training_dataset.groupby(["teacher_prefix", "project_is_approved"])
 grouped_prefixes = grouped_prefixes.agg({'teacher_prefix' : 'count'} ).rename(columns={'teacher_prefix' : 'count'})
 grouped_prefixes
test = pd.read_csv(path + "test.csv")
 dic_ids = {k:v.id for k,v in test.groupby(["lang"])}<load_from_csv>
arr_counts = grouped_prefixes["count"].tolist()
y_no = []
y_yes = []
for i in range(0, len(arr_counts)- 1):
if i % 2 == 0:
y_no.append(arr_counts[i])
 else:
y_yes.append(arr_counts[i])
 y_yes.append(arr_counts[i+1])
 y_no.append(0)
 x_labels = ['Dr', 'Mr.', 'Mrs', 'Ms', 'Teacher', 'None']
N = len(x_labels)
 ind = np.arange(N)
 width = 0.4
fig = plt.figure()
ax = fig.add_subplot(111)
 rects1 = ax.bar(ind, y_yes, width, color='
rects2 = ax.bar(ind+width, y_no, width, color='
ax.set_ylabel('Submissions Count')
 ax.set_xticks(ind+width)
 ax.set_xticklabels(x_labels)
 ax.legend(( rects1[0], rects2[0]),('Approved', 'NOT Approved'))
def autolabel(rects):
for rect in rects:
h = rect.get_height()
ax.text(rect.get_x() +rect.get_width() /2., 1.05*h, '%d'%int(h),
ha='center', va='bottom')
 autolabel(rects1)
 autolabel(rects2)
 plt.title('Submissions Approved or NOT Approved Based on Name Prefix')
 plt.show()
sub = pd.read_csv(base)
 for m in monos:
res = pd.read_csv(record + m)
 ids = dic_ids[get_lang(m)]
sub.loc[ids,"toxic"] = res.toxic[ids]
sub.head()<feature_engineering>
print('Chance to get approved by prefix')
 arr_total_applications = []
for i in range(0, len(y_yes)) :
arr_total_applications.append(y_yes[i] + y_no[i])
 print(x_labels[i] + ': ' + "%f" %(y_yes[i] / arr_total_applications[i]))
adj = {
"fr":1.04,
"es":1.06,
"pt":.96,
"it":.97,
"tr":.98,
}
for l,v in adj.items() :
ids = dic_ids[l]
sub.loc[ids,"toxic"] *= v
sub.head()<categorify>
bins = [0, 150, 200, 250, 300, 350, 400, 500, 600, 750, 1000, 1000000]
combined_training_dataset['binned_costs'] = pd.cut(combined_training_dataset['total_cost'], bins)
 grouped_costs = combined_training_dataset.groupby(["binned_costs", "project_is_approved"])
 grouped_costs = grouped_costs.agg({'binned_costs' : 'count'} ).rename(columns={'binned_costs' : 'count'})
 arr_percent_yes = []
for i in range(0,(len(bins)- 1)* 2):
if i % 2 == 0:
k = i // 2
label = 'no'
no_count = grouped_costs["count"].values[i]
print("%d+ - %s: %d" %(bins[k], label, no_count))
else:
label = 'yes'
yes_count = grouped_costs["count"].values[i]
percent_yes = yes_count /(no_count + yes_count)
 arr_percent_yes.append(percent_yes)
 print("%d+ - %s: %d, percent: %f" %(bins[k], label, yes_count, percent_yes))
weight = lambda x: softmax(1/(1-x))
def mix_result(subs,pbs):
toxics = np.array([df.toxic.values for df in subs])
 w = weight(np.array(pbs))
print(["{:.3f}".format(i)for i in w])
 return toxics.T@w<load_from_csv>
grouped_previous_submissions = combined_training_dataset.groupby(["teacher_number_of_previously_posted_projects", "project_is_approved"])
 grouped_previous_submissions = grouped_previous_submissions.agg({'teacher_number_of_previously_posted_projects' : 'count'} ).rename(columns={'teacher_number_of_previously_posted_projects' : 'count'})
 grouped_previous_submissions[0:202]
sub1 = pd.read_csv(record+"submission-public-mix-9482.csv")
 sub["toxic"] = mix_result([sub,sub1],[.9508,.9482])
 sub.head()<load_from_csv>
arr_percents_yes = []
for i in range(0,202):
if i % 2 == 0:
k = i / 2
label = 'no'
no_count = grouped_previous_submissions["count"].values[i]
print("%d - %s: %d" %(k, label, no_count))
else:
label = 'yes'
yes_count = grouped_previous_submissions["count"].values[i]
percent_yes = yes_count /(no_count + yes_count)
 arr_percents_yes.append(percent_yes)
 print("%d - %s: %d, percent: %f" %(k, label, yes_count, percent_yes))
sub1 = pd.read_csv(record+"submission-1st-place-9550.csv")
 sub2 = pd.read_csv(record+"submission-2nd-place-9522.csv")
 sub["toxic"] = mix_result([sub,sub2,sub1],[.9514,.9522,.9550])
 sub.head()<save_to_csv>
grouped_states = combined_training_dataset.groupby(["school_state", "project_is_approved"])
 grouped_states = grouped_states.size().rename('count' ).reset_index()
arr_percents = []
for i in range(0,102):
if i % 2 == 0:
no_count = grouped_states["count"].values[i]
else:
yes_count = grouped_states["count"].values[i]
total_count = no_count + yes_count
percent_no = no_count / total_count
percent_yes = yes_count / total_count
arr_percents.append(percent_no)
 arr_percents.append(percent_yes)
 grouped_states['chances'] = pd.Series(arr_percents, index=grouped_states.index)
 grouped_states
sub.to_csv('submission.csv', index=False )<save_to_csv>
grouped_states.loc[(grouped_states['project_is_approved'] == 1)&(grouped_states['chances'] <.83)]
sub.to_csv('submission.csv', index=False )<import_modules>
grouped_states.loc[(grouped_states['project_is_approved'] == 1)&(grouped_states['chances'] >.868)]
import os,gc,pickle,random,sys
import numpy as np
import pandas as pd
import tensorflow as tf
import transformers
from transformers import TFAutoModel
from tqdm.notebook import tqdm
from tensorflow.data import Dataset
from tensorflow.data.experimental import sample_from_datasets<define_variables>
grouped_grades = combined_training_dataset.groupby(["project_grade_category", "project_is_approved"])
 grouped_grades = grouped_grades.size().rename('count' ).reset_index()
grouped_grades
MAX_LEN = 192
HEAD = "max"
PATH = ".. /input/jigsaw-multilingual-toxic-comment-classification/"
INPATH = ".. /input/jmtc-monolinguish-encoding-pickle/"
TRANSFER = ".. /input/buffer/submission-9462.csv"
LANG = "pt"
lang_valid = ["it","es","tr"]
MODEL = "neuralmind/bert-large-portuguese-cased"<define_variables>
grouped_weeks = combined_training_dataset.groupby([pd.Grouper(key='project_submitted_datetime', freq='1W'), "project_is_approved"])
 grouped_weeks = grouped_weeks.size().rename('count' ).reset_index()
arr_percents_w = []
for i in range(0,106):
if i % 2 == 0:
no_count = grouped_weeks["count"].values[i]
else:
yes_count = grouped_weeks["count"].values[i]
total_count = no_count + yes_count
percent_no = no_count / total_count
percent_yes = yes_count / total_count
arr_percents_w.append(percent_no)
 arr_percents_w.append(percent_yes)
 grouped_weeks['chances'] = pd.Series(arr_percents_w, index=grouped_weeks.index)
 grouped_weeks
np.random.seed(1234)
 random.seed(1234)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
with strategy.scope() :
tf.random.set_seed(1234 )<define_variables>
combined_training_dataset['weekday'] = combined_training_dataset['project_submitted_datetime'].dt.weekday
grouped_weekday = combined_training_dataset.groupby(["weekday", "project_is_approved"])
 grouped_weekday = grouped_weekday.size().rename('count' ).reset_index()
arr_percents_weekday = []
for i in range(0,14):
if i % 2 == 0:
no_count = grouped_weekday["count"].values[i]
else:
yes_count = grouped_weekday["count"].values[i]
total_count = no_count + yes_count
percent_no = no_count / total_count
percent_yes = yes_count / total_count
arr_percents_weekday.append(percent_no)
 arr_percents_weekday.append(percent_yes)
 grouped_weekday['chances'] = pd.Series(arr_percents_weekday, index=grouped_weekday.index)
 grouped_weekday
models = {"pt":["neuralmind/bert-large-portuguese-cased"],
"it":["dbmdz/bert-base-italian-xxl-uncased",
"m-polignano-uniba/bert_uncased_L-12_H-768_A-12_italian_alb3rt0"],
"es":["dccuchile/bert-base-spanish-wwm-cased"],
"tr":["dbmdz/bert-base-turkish-128k-cased",
"dbmdz/electra-base-turkish-cased-discriminator"],
"ru":["DeepPavlov/rubert-base-cased"],
"fr":["flaubert/flaubert_large_cased",
"camembert/camembert-large"]}<data_type_conversions>
if USING_DICTIONARY_CODE:
pd_plot = combined_training_dataset[['total_spelling_errors', 'total_typing_errors', 'project_is_approved']].copy()
pd_plot['total_spelling_errors'] = pd.to_numeric(pd_plot['total_spelling_errors'])
 pd_plot['total_typing_errors'] = pd.to_numeric(pd_plot['total_typing_errors'])
 fig, axes = plt.subplots()
sns.violinplot('project_is_approved','total_spelling_errors', data=pd_plot, ax = axes)
 axes.set_title('Spelling Errors vs Approval')
 axes.yaxis.grid(True)
 axes.set_xlabel('project_is_approved')
 axes.set_ylabel('total-spelling_errors')
 plt.show()
trans = pd.read_csv(TRANSFER ).toxic.values.astype('float32' )<randomize_order>
combined_training_dataset = combined_training_dataset.reindex(
 np.random.permutation(combined_training_dataset.index))
fetch_title = lambda x: x.split("/")[-1]
def pick_load_format(path):
with open(INPATH+path,"rb")as f:
return pickle.load(f)
 def get_cong(n,verb=True):
tot = round(1+(n*2)/10_000)*10_000
if verb: print("Pos: {}, Sample neg: {}, Total: {}".format(n,tot-n,tot))
return tot-n
def balance_shuffle(list_arr,pos_thred=0.6,neg_thred=0.4,balance=True):
target = list_arr[-1]
idx = np.arange(len(target))
pos_idx = idx[target>=pos_thred]
neg_idx = idx[target<=neg_thred]
psize,nsize = len(pos_idx),len(neg_idx)
 cong = get_cong(psize,balance)
 if balance and nsize>cong:
neg_idx = np.random.choice(neg_idx,cong,replace=False)
 idx = np.concatenate(( pos_idx, neg_idx))
np.random.shuffle(idx)
 return tuple(arr[idx] for arr in list_arr)
 def load_data(model,lang,frac=.05):
title = fetch_title(model)
 x_train,y_train = balance_shuffle(pick_load_format("train/train_%s"%title))
train = x_train,y_train.astype("float32")
 if lang in lang_valid:
valid = balance_shuffle(pick_load_format("valid/valid_%s"%title),balance=False)
 else:
thres = round(len(train[-1])*frac)
 valid = tuple(arr[:thres] for arr in train)
 train = tuple(arr[thres:] for arr in train)
 x_test,id_test = pick_load_format("test/test_%s"%title)
 quesdo = balance_shuffle(( x_test,trans[id_test]),balance=False)
 return train,valid,quesdo,x_test,id_test<train_model>
USING_OLD_MODELS = False
HEAD = 145664
TAIL = 36416
TARGET_STR = "project_is_approved"
training_examples = combined_training_dataset.head(HEAD)
 validation_examples = combined_training_dataset.tail(TAIL)
 if USING_OLD_MODELS:
training_targets = combined_training_dataset[[TARGET_STR]].head(HEAD)
 validation_targets = combined_training_dataset[[TARGET_STR]].tail(TAIL)
train,valid,quesdo,test,ids = load_data(MODEL,LANG)
 train_size,valid_size,quesdo_size = len(train[1]),len(valid[1]),len(quesdo[1])
 print(train_size,valid_size,quesdo_size )<categorify>
WORDS_STRING = 'words_string'
WHOLE_STRING = 'whole_string'
BUCKET_INT = 'bucket_int'
BUCKET_FLOAT = 'bucket_float'
NUM_INT = 'num_int'
NUM_FLOAT = 'num_float'
ERROR_BUCKETS = [0, 1, 2, 4, 7, 10, 20, 50]
if USING_OLD_MODELS:
DESIRED_FEATURES = [ 
('teacher_prefix', WHOLE_STRING, None), 
('school_state', WHOLE_STRING, None), 
('teacher_number_of_previously_posted_projects', BUCKET_INT, 6), 
('project_is_approved', NUM_INT, None)
 ]
else:
DESIRED_FEATURES = [ 
('project_essay_1', WORDS_STRING, None), 
('project_essay_2', WORDS_STRING, None), 
('teacher_number_of_previously_posted_projects', BUCKET_INT, 6), 
('full_description', WORDS_STRING, None), 
('total_cost', BUCKET_FLOAT, 12), 
('title_spelling_errors', BUCKET_INT, ERROR_BUCKETS), 
('essay_1_spelling_errors', BUCKET_INT, ERROR_BUCKETS), 
('essay_2_spelling_errors', BUCKET_INT, ERROR_BUCKETS), 
('total_typing_errors', BUCKET_INT, ERROR_BUCKETS), 
('total_spelling_errors', BUCKET_INT, ERROR_BUCKETS), 
('project_is_approved', NUM_INT, None)
 ]
DESIRED_COLUMNS = []
for str_name, str_type, dontcare in DESIRED_FEATURES:
DESIRED_COLUMNS.append(str_name)
 STR_TARGET = 'project_is_approved'
IS_DNN_CLASSIFIER = False
def make_dataset_pipeline(dataset, cache=False,repeat_and_shuffle=False,shuffle_size=128_000,seed=386491):
if cache: dataset = dataset.cache()
if repeat_and_shuffle:
dataset = dataset.repeat().shuffle(shuffle_size,seed)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def build_datasets(train,valid,quesdo,test):
dtrain = Dataset.from_tensor_slices(train)
 dvalid = Dataset.from_tensor_slices(valid)
 dquesdo = Dataset.from_tensor_slices(quesdo)
 dtest = Dataset.from_tensor_slices(test)
 train_dataset = make_dataset_pipeline(dtrain,True, repeat_and_shuffle=True)
 valid_dataset = make_dataset_pipeline(dvalid, True,repeat_and_shuffle=True)
 quesdo_dataset = make_dataset_pipeline(dquesdo, True,repeat_and_shuffle=True)
 validset = make_dataset_pipeline(dvalid)
 testset = make_dataset_pipeline(dtest)
 return train_dataset,valid_dataset,quesdo_dataset,validset,testset
def mix_dataset(dss,szs,weight=None,seed=1214):
if weight is None: weight = np.ones(len(szs))
prop = np.array(szs)*weight
return sample_from_datasets(dss,prop/np.sum(prop),seed),sum(szs )<create_dataframe>
def get_quantile_based_buckets(feature_values, num_buckets):

 quantiles = feature_values.quantile(
 [(i+1.) /(num_buckets + 1.) for i in range(num_buckets)])
 return [quantiles[q] for q in quantiles.keys() ]
train_dataset,valid_dataset,quesdo_dataset,validset,testset = build_datasets(train,valid,quesdo,test )<choose_model_class>
for str_name, str_type, num_bucket in DESIRED_FEATURES:
print(str_name)
 if(str_type == BUCKET_INT)or(str_type == BUCKET_FLOAT):
if(type(num_bucket)== list)== False:
print(get_quantile_based_buckets(training_examples[str_name], num_bucket))
else:
print(num_bucket )
def get_cls(x):
return x[:, 0, :]
dic = {"mean":GlobalAveragePooling1D() ,
"max":GlobalMaxPool1D() ,
"cls":get_cls}
def build_model(transformer,head="cls" , loss='binary_crossentropy',
max_len=512, drop_rate=None, lr=1e-5,seed=940208):
input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 sequence_output = transformer(input_word_ids)[0]
x = dic[head](sequence_output)
 if drop_rate is not None:
x = Dropout(drop_rate )(x)
 out = Dense(1, activation='sigmoid',kernel_initializer=GlorotUniform(seed))(x)
 model = Model(inputs=input_word_ids, outputs=out)
 model.compile(Adam(lr=lr), loss=loss, metrics=[AUC() ])
 return model<load_pretrained>
def my_input_fn(features, targets, batch_size=1, shuffle=True, num_epochs=None):

 selected_features_data = features[DESIRED_COLUMNS]
features = {key:np.array(value)for key,value in dict(selected_features_data ).items() }
ds = Dataset.from_tensor_slices(( features,targets))
ds = ds.batch(batch_size ).repeat(num_epochs)
 if shuffle:
ds = ds.shuffle(10000)
 features, labels = ds.make_one_shot_iterator().get_next()
return features, labels
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=1e-5)
 model.summary()<merge>
def vocab_lists_for_each_feature() :
vocab_for_feature = {}
for str_name, str_type, dont_care in DESIRED_FEATURES:
if(str_type == WHOLE_STRING):
vocab_for_feature[str_name] = training_examples[str_name].unique().tolist()
elif(str_type == WORDS_STRING):
new_set = set()
training_examples[str_name].str.split().apply(new_set.update)
 vocab_for_feature[str_name] = new_set
return vocab_for_feature
vocab_lists_for_features = vocab_lists_for_each_feature()
train_dataset,train_size = mix_dataset([train_dataset,quesdo_dataset],[train_size,quesdo_size],weight=None )<train_model>
def construct_feature_columns() :

 arr_num_columns = []
arr_vocabulary_columns = []
arr_dimensions = []
for str_name, str_type, num_bucket in DESIRED_FEATURES:
if(str_name == STR_TARGET):
continue
elif(str_type == NUM_INT)or(str_type == NUM_FLOAT):
arr_num_columns.append(tf.feature_column.numeric_column(str_name))
elif(str_type == BUCKET_INT)or(str_type == BUCKET_FLOAT):
if(type(num_bucket)== list)== False:
bucket_column = tf.feature_column.bucketized_column(
 tf.feature_column.numeric_column(str_name),
boundaries=get_quantile_based_buckets(training_examples[str_name], num_bucket))
else:
bucket_column = tf.feature_column.bucketized_column(
 tf.feature_column.numeric_column(str_name), boundaries=num_bucket)
 arr_num_columns.append(bucket_column)
 elif(str_type == WHOLE_STRING)or(str_type == WORDS_STRING):
arr_vocabulary_columns.append(tf.feature_column.categorical_column_with_vocabulary_list(key=str_name, vocabulary_list=vocab_lists_for_features[str_name]))
if IS_DNN_CLASSIFIER:
fourth_root = int(math.sqrt(math.sqrt(len(vocab_lists_for_features[str_name]))))
if(fourth_root == 0):
fourth_root += 1
arr_dimensions.append(fourth_root)
 end = len(arr_dimensions)
 arr_embedding_columns = []
if IS_DNN_CLASSIFIER:
for i in range(0, end):
arr_embedding_columns.append(tf.feature_column.embedding_column(arr_vocabulary_columns[i], dimension=arr_dimensions[i]))
arr_vocabulary_columns = arr_embedding_columns
feature_columns = set(arr_num_columns + arr_vocabulary_columns)
 return feature_columns
%%time
N_STEPS = train_size //(BATCH_SIZE*2)
 EPOCHS = 8
train_history = model.fit(
 train_dataset,
steps_per_epoch=N_STEPS,
validation_data=validset,
callbacks=callback_list,
epochs=EPOCHS
)<set_options>
def train_linear_classifier_model(
 learning_rate,
regularization_strength,
steps,
batch_size,
feature_columns,
training_examples,
training_targets,
validation_examples,
validation_targets,
periods=1):

 my_optimizer = tf.train.FtrlOptimizer(learning_rate=learning_rate, l1_regularization_strength=regularization_strength)
 my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
 linear_classifier = tf.estimator.LinearClassifier(
 feature_columns=feature_columns,
optimizer=my_optimizer 
)
 training_input_fn = lambda: my_input_fn(training_examples,
training_targets[TARGET_STR],
batch_size=batch_size)
 print("Training model...")
 print("LogLoss(on validation data):")
 for period in range(0, periods):
linear_classifier.train(
 input_fn=training_input_fn,
steps=steps 
)
 training_probabilities = linear_classifier.predict(input_fn=predict_training_input_fn)
 training_probabilities = np.array([item['probabilities'] for item in training_probabilities])
 validation_probabilities = linear_classifier.predict(input_fn=predict_validation_input_fn)
 validation_probabilities = np.array([item['probabilities'] for item in validation_probabilities])
 training_log_loss = metrics.log_loss(training_targets, training_probabilities)
 validation_log_loss = metrics.log_loss(validation_targets, validation_probabilities)
 print(" Training loss for period %02d: %0.2f" %(period, training_log_loss))
print(" Validation loss for period %02d : %0.2f" %(period, validation_log_loss))
print("Model training finished.")
 return linear_classifier
del model
gc.collect()
tf.tpu.experimental.initialize_tpu_system(tpu )<load_pretrained>
predict_training_input_fn = lambda: my_input_fn(training_examples,
training_targets[TARGET_STR],
num_epochs=1,
shuffle=False)
 predict_validation_input_fn = lambda: my_input_fn(validation_examples,
validation_targets[TARGET_STR],
num_epochs=1,
shuffle=False )
%%time
with strategy.scope() :
transformer_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True)
 model = build_model(transformer_layer,head=HEAD,loss='binary_crossentropy', max_len=MAX_LEN,lr=5e-6)
 model.load_weights(model_path )<train_model>
if USING_OLD_MODELS &(not IS_DNN_CLASSIFIER):
linear_classifier = train_linear_classifier_model(
 learning_rate=.1,
regularization_strength=0.1,
steps=1000,
batch_size=140,
periods=1,
feature_columns=construct_feature_columns() ,
training_examples=training_examples,
training_targets=training_targets,
validation_examples=validation_examples,
validation_targets=validation_targets)
%%time
n_steps = valid_size //(BATCH_SIZE)
 EPOCHS = 1
train_history_2 =model.fit(
 valid_dataset,
steps_per_epoch=n_steps,
epochs= EPOCHS
)<set_options>
if USING_OLD_MODELS &(not IS_DNN_CLASSIFIER):
validation_metrics = linear_classifier.evaluate(input_fn=predict_validation_input_fn)
 print("AUC on the validation set: %0.2f" % validation_metrics['auc'])
 print("Accuracy on the validation set: %0.2f" % validation_metrics['accuracy'] )
!rm model1.h5<save_to_csv>
def make_bytes_list_append(col_index, value, example):
example.features.feature[DESIRED_COLUMNS[col_index]].bytes_list.value.append(value.encode())
 def make_bytes_list_extend(col_index, value, example):
arr_strings = value.split(' ')
 arr_bstrings = list(map(lambda x: x.encode() , arr_strings))
example.features.feature[DESIRED_COLUMNS[col_index]].bytes_list.value.extend(arr_bstrings)
 def make_int64_list(col_index, value, example):
example.features.feature[DESIRED_COLUMNS[col_index]].int64_list.value.append(value)
 def make_float_list(col_index, value, example):
example.features.feature[DESIRED_COLUMNS[col_index]].float_list.value.append(value)
 def match_feature_with_tfrecord_function() :
arr_functions = []
for str_name, str_type, dontcare in DESIRED_FEATURES:
if(str_type == WHOLE_STRING):
arr_functions.append(make_bytes_list_append)
 elif(str_type == WORDS_STRING):
arr_functions.append(make_bytes_list_extend)
 elif(str_type == NUM_FLOAT)or(str_type == BUCKET_FLOAT):
arr_functions.append(make_float_list)
 elif(str_type == NUM_INT)or(str_type == BUCKET_INT):
arr_functions.append(make_int64_list)
 return arr_functions
def save_rows_as_TFRecord(array_of_rows, tfrecord_file_name):
with tf.python_io.TFRecordWriter(tfrecord_file_name)as writer:
last_column = len(DESIRED_COLUMNS)
 for row in array_of_rows:
example = tf.train.Example()
for col_index in range(0, last_column):
arr_tfrecord_funcs[col_index](col_index, row[col_index], example)
 writer.write(example.SerializeToString())
 return
def TEST_save_rows_as_TFRecord(array_of_rows, tfrecord_file_name):
last_column = len(DESIRED_COLUMNS)
 for row in array_of_rows:
example = tf.train.Example()
for col_index in range(0, last_column):
arr_tfrecord_funcs[col_index](col_index, row[col_index], example)
 print(example )
sub = pd.read_csv(TRANSFER)
 sub.loc[ids,'toxic'] = model.predict(testset, verbose=1 ).reshape(-1)
 sub.to_csv('submission.csv', index=False )<define_variables>
arr_tfrecord_funcs = match_feature_with_tfrecord_function()
testdata = training_examples[DESIRED_COLUMNS][0:2]
TEST_save_rows_as_TFRecord(testdata.values, 'test' )
model_name = 'jplu/tf-xlm-r-ner-40-lang'
tokenizer_name = "novinsh/xlm-roberta-large-toxicomments-12k"<import_modules>
combined_test_dataset['project_is_approved'] = 0
combined_test_dataset[0:5]
import os
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.callbacks import ModelCheckpoint
from kaggle_datasets import KaggleDatasets
import transformers
from tqdm.notebook import tqdm
from tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer<set_options>
TRAINING_TFRECORD = "training.tfrecords"
VALIDATION_TFRECORD = "validation.tfrecords"
TEST_TFRECORD = "test.tfrecords"
arr_tfrecord_funcs = match_feature_with_tfrecord_function()
save_rows_as_TFRecord(training_examples[DESIRED_COLUMNS].values, TRAINING_TFRECORD)
 save_rows_as_TFRecord(validation_examples[DESIRED_COLUMNS].values, VALIDATION_TFRECORD)
 save_rows_as_TFRecord(combined_test_dataset[DESIRED_COLUMNS].values, TEST_TFRECORD )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
def parse_function(record):

 features_to_parse = {}
for str_name, str_type, dontcare in DESIRED_FEATURES:
if(str_type == WHOLE_STRING):
features_to_parse[str_name] = tf.FixedLenFeature(shape=[1], dtype=tf.string)
 elif(str_type == WORDS_STRING):
features_to_parse[str_name] = tf.VarLenFeature(dtype=tf.string)
 elif(str_type == NUM_FLOAT)or(str_type == BUCKET_FLOAT):
features_to_parse[str_name] = tf.FixedLenFeature(shape=[1], dtype=tf.float32)
 elif(str_type == NUM_INT)or(str_type == BUCKET_INT):
features_to_parse[str_name] = tf.FixedLenFeature(shape=[1], dtype=tf.int64)
 parsed_features = tf.parse_single_example(record, features_to_parse)
 final_features = {}
for str_name, str_type, dontcare in DESIRED_FEATURES:
if(str_name == STR_TARGET):
continue
elif(str_type == WORDS_STRING):
final_features[str_name] = parsed_features[str_name].values
else:
final_features[str_name] = parsed_features[str_name]
labels = parsed_features[STR_TARGET]
return final_features, labels
AUTO = tf.data.experimental.AUTOTUNE
GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')
 if model_name == 't5-large':
EPOCHs = 1
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
else:
EPOCHS = 3
BATCH_SIZE = 8 * strategy.num_replicas_in_sync
MAX_LEN = 128<load_pretrained>
ds = tf.data.TFRecordDataset(TRAINING_TFRECORD)
 ds = ds.map(parse_function)
 n = ds.make_one_shot_iterator().get_next()
sess = tf.Session()
sess.run(n )
tokenizer = transformers.AutoTokenizer.from_pretrained(tokenizer_name)
 if model_name == 't5-large':
tokenizer.pad_token = tokenizer.eos_token
tokenizer.save_pretrained('.')
<load_from_csv>
def tfrecord_input_fn(input_filenames, batch_size=100, num_epochs=None, shuffle=True):
ds = tf.data.TFRecordDataset(input_filenames)
 ds = ds.map(parse_function)
 if shuffle:
ds = ds.shuffle(10000)
 ds = ds.padded_batch(batch_size, ds.output_shapes)
 ds = ds.repeat(num_epochs)
 features, labels = ds.make_one_shot_iterator().get_next()
return features, labels
train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
 train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")
 train2.toxic = train2.toxic.round().astype(int)
 valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<count_values>
def train_linear_classifier_text(learning_rate=.05, steps=2000, batch_size=100):
my_optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate)
 my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
 feature_columns = construct_feature_columns()
classifier = tf.estimator.LinearClassifier(
 feature_columns=feature_columns,
optimizer=my_optimizer, 
)
 classifier.train(
 input_fn=lambda: tfrecord_input_fn(TRAINING_TFRECORD, batch_size=batch_size),
steps=steps)
 predict_training_input_fn = lambda: tfrecord_input_fn(TRAINING_TFRECORD,
batch_size=batch_size,
num_epochs=1,
shuffle=False)
 predict_validation_input_fn = lambda: tfrecord_input_fn(VALIDATION_TFRECORD,
batch_size=batch_size,
num_epochs=1,
shuffle=False)
 return classifier, [predict_training_input_fn, predict_validation_input_fn]
train = pd.concat([
train1[['comment_text', 'toxic']],
train2[['comment_text', 'toxic']].query('toxic==1'),
train2[['comment_text', 'toxic']].query('toxic==0' ).sample(n=150000, random_state=0)
 ])
 print(train.toxic.value_counts())
 print(valid.toxic.value_counts() )<load_from_csv>
if(USING_OLD_MODELS == False)&(IS_DNN_CLASSIFIER == False):
classifier, arr_funcs = train_linear_classifier_text(learning_rate=.01,
steps=1500,
batch_size=100 )
olid = pd.read_csv('/kaggle/input/olid2019/olid-training-v1.0.tsv', sep='\t')
 olid = olid.rename(columns={"id": "id", "tweet": "comment_text", 'subtask_a': 'toxic'})
 olid.toxic =(olid.toxic == 'OFF' ).astype(int)
 olid.head()<load_from_csv>
def train_dnn_classifier_text(learning_rate=.05, steps=2000, batch_size=100, hidden_units=[20,20], dropout=.1):
my_optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
 my_optimizer = tf.contrib.estimator.clip_gradients_by_norm(my_optimizer, 5.0)
 feature_columns = construct_feature_columns()
classifier = tf.estimator.DNNClassifier(
 feature_columns=feature_columns,
hidden_units=hidden_units,
optimizer=my_optimizer,
dropout=dropout 
)
 classifier.train(
 input_fn=lambda: tfrecord_input_fn(TRAINING_TFRECORD, batch_size=batch_size),
steps=steps)
 predict_training_input_fn = lambda: tfrecord_input_fn(TRAINING_TFRECORD,
batch_size=batch_size,
num_epochs=1,
shuffle=False)
 predict_validation_input_fn = lambda: tfrecord_input_fn(VALIDATION_TFRECORD,
batch_size=batch_size,
num_epochs=1,
shuffle=False)
 return classifier, [predict_training_input_fn, predict_validation_input_fn]
olid_test = pd.read_csv('/kaggle/input/olid2019/testset-levela.tsv', sep='\t')
 olid_test_label = pd.read_csv('/kaggle/input/olid2019/labels-levela.csv', names = ['id', 'label'] )<count_values>
if(USING_OLD_MODELS == False)&(IS_DNN_CLASSIFIER):
dnn_classifier, dnn_arr_funcs = train_dnn_classifier_text(learning_rate=.005,
steps=1500,
batch_size=100,
hidden_units=[6],
dropout=.1 )
olid_test_label.label.value_counts()<merge>
predict_test_input_fn = lambda: tfrecord_input_fn(TEST_TFRECORD,
num_epochs=1,
shuffle=False)
 if(USING_OLD_MODELS == False):
if(IS_DNN_CLASSIFIER == False):
predictions_generator = classifier.predict(input_fn=predict_test_input_fn)
 else:
predictions_generator = dnn_classifier.predict(input_fn=predict_test_input_fn)
 predictions_list = list(predictions_generator)
 probabilities = [p["probabilities"][1] for p in predictions_list]
print('Now have the probabilities.' )
olid_test = olid_test.set_index('id' ).join(olid_test_label.set_index('id'))<data_type_conversions>
submission_values = pd.DataFrame({'id': combined_test_dataset['id'], 'project_is_approved': probabilities})
 submission_values[0:5]
<split><EOS> 
submission_values.to_csv('linear-spelling.csv', index=False )