Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<save_to_csv>
import gc
import numpy as np
import pandas as pd
import os
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import KFold, RepeatedKFold
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
from sklearn.preprocessing import LabelEncoder
from sklearn import metrics
from keras import initializers, regularizers, constraints
from keras.layers import Dense, Input, Flatten, RepeatVector, Permute
from keras.layers import Input, Dense, LSTM,merge, Merge, Bidirectional, concatenate, SpatialDropout1D, GRU, BatchNormalization, Dropout, Activation,TimeDistributed
from keras.preprocessing.text import Tokenizer
from keras.layers.embeddings import Embedding
from keras.preprocessing.sequence import pad_sequences
from keras.models import Model, Sequential
from keras.regularizers import l2
from keras.layers import Convolution1D, GlobalMaxPooling1D, Flatten
from keras.layers import Conv1D, MaxPooling1D, Embedding,Lambda
from keras.preprocessing.text import text_to_word_sequence
from keras.callbacks import *
from keras import backend as K
import tensorflow as tf
logR.fit(X,y)
 preds_log = logR.predict(test_df)
 sub_log = pd.DataFrame({'Id':test_df.index, 'Target':preds_log})
 sub_log.to_csv('sub_log1.csv', index=False)
<set_options>
train = pd.read_csv('.. /input/donorschoose-application-screening/train.csv')
 test = pd.read_csv('.. /input/donorschoose-application-screening/test.csv')
 resources = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv')
 train = train.sort_values(by="project_submitted_datetime" )
def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
SEED = 36
seed_everything(SEED)
<categorify>
char_cols = ['project_title', 'project_essay_1', 'project_essay_2',
'project_essay_3', 'project_essay_4', 'project_resource_summary']
train = train.fillna('NA')
 test = test.fillna('NA')
 teachers_train = list(set(train.teacher_id.values))
teachers_test = list(set(test.teacher_id.values))
inter = set(teachers_train ).intersection(teachers_test)
 train['text'] = ''
for col in char_cols:
train['text'] += train[col]
train['text'] += ' '
test['text'] = ''
for col in char_cols:
test['text'] += test[col]
test['text'] += ' '
def preprocess(string):

 string = re.sub(r'(\\r)', ' ', string)
 string = re.sub(r'(\ 
)', ' ', string)
 string = re.sub(r'(\\r\ 
)', ' ', string)
 string = re.sub(r'(\\)', ' ', string)
 string = re.sub(r'\t', ' ', string)
 string = re.sub(r'\:', ' ', string)
 string = re.sub(r'
def regular_encode(texts, tokenizer, maxlen=512):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=True,
return_token_type_ids=True,
pad_to_max_length=True,
max_length=maxlen 
)
 return np.array(enc_di['input_ids'],dtype=np.int32),np.array(enc_di['attention_mask'],dtype=np.int32),np.array(enc_di['token_type_ids'],dtype=np.int32 )<compute_train_metric>
word_index = tokenizer.word_index
EMBEDDING_DIM = 300
MAX_SEQUENCE_LENGTH = 600
sequences = tokenizer.texts_to_sequences(train_text)
 data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
 sequence_test = tokenizer.texts_to_sequences(test_text)
 test_data = pad_sequences(sequence_test, maxlen=MAX_SEQUENCE_LENGTH )
def focal_loss(gamma=2., alpha=.2):
def focal_loss_fixed(y_true, y_pred):
pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
return -K.mean(alpha * K.pow(1.- pt_1, gamma)* K.log(pt_1)) - K.mean(( 1 - alpha)* K.pow(pt_0, gamma)* K.log(1.- pt_0))
return focal_loss_fixed<define_search_model>
X_train, X_val, y_train, y_val = train_test_split(data, y, test_size=0.1, random_state=666 )
def build_model(transformer,max_len=512):
input_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
 input_mask = Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
 segment_ids = Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")
 sequence_output = transformer(( input_ids,input_mask,segment_ids)) [0]
gp = GlobalMaxPooling1D()(sequence_output)
 ap = GlobalAveragePooling1D()(sequence_output)
 stack = concatenate([gp,ap],axis=1)
 out = Dense(1, activation='sigmoid' )(stack)
 model = Model(inputs=[input_ids,input_mask,segment_ids], outputs=out)
 model.compile(Adam(lr=0.25e-5), loss='binary_crossentropy', metrics=['accuracy',tf.keras.metrics.AUC() ])
 return model<set_options>
embedding_matrix = np.load('.. /input/crawlembeddingmatric/embedding_matrix.npy')
 embedding_layer = Embedding(len(word_index)+ 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=False )
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync)
<define_variables>
def auc(y_true, y_pred):
ptas = tf.stack([binary_PTA(y_true,y_pred,k)for k in np.linspace(0, 1, 1000)],axis=0)
 pfas = tf.stack([binary_PFA(y_true,y_pred,k)for k in np.linspace(0, 1, 1000)],axis=0)
 pfas = tf.concat([tf.ones(( 1,)) ,pfas],axis=0)
 binSizes = -(pfas[1:]-pfas[:-1])
 s = ptas*binSizes
return K.sum(s, axis=0)
 def binary_PFA(y_true, y_pred, threshold=K.variable(value=0.5)) :
y_pred = K.cast(y_pred >= threshold, 'float32')
 N = K.sum(1 - y_true)
 FP = K.sum(y_pred - y_pred * y_true)
 return FP/N
def binary_PTA(y_true, y_pred, threshold=K.variable(value=0.5)) :
y_pred = K.cast(y_pred >= threshold, 'float32')
 P = K.sum(y_true)
 TP = K.sum(y_pred * y_true)
 return TP/P
AUTO = tf.data.experimental.AUTOTUNE
EPOCHS = 2
BATCH_SIZE = 16* strategy.num_replicas_in_sync
MAX_LEN = 256
MODEL = 'jplu/tf-xlm-roberta-large' 
<load_pretrained>
def get_model() :
class AttLayer(Layer):
def __init__(self, init='glorot_uniform', kernel_regularizer=None,
bias_regularizer=None, kernel_constraint=None,
bias_constraint=None, **kwargs):
self.supports_masking = True
self.init = initializers.get(init)
 self.kernel_initializer = initializers.get(init)
 self.kernel_regularizer = regularizers.get(kernel_regularizer)
 self.bias_regularizer = regularizers.get(kernel_regularizer)
 self.kernel_constraint = constraints.get(kernel_constraint)
 self.bias_constraint = constraints.get(bias_constraint)
 super(AttLayer, self ).__init__(** kwargs)
 def build(self, input_shape):
assert len(input_shape)==3
self.W = self.add_weight(( input_shape[-1], 1),
initializer=self.kernel_initializer,
name='{}_W'.format(self.name),
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.bias_regularizer,
constraint=self.bias_constraint)
 self.u = self.add_weight(( input_shape[1],),
initializer=self.kernel_initializer,
name='{}_u'.format(self.name),
regularizer=self.kernel_regularizer,
constraint=self.kernel_constraint)
 self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
uit = K.dot(x, self.W)
 uit = K.squeeze(uit, -1)
 uit = uit + self.b
uit = K.tanh(uit)
 ait = uit * self.u
ait = K.exp(ait)
 if mask is not None:
mask = K.cast(mask, K.floatx())
 ait = mask*ait
ait /= K.cast(K.sum(ait, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 ait = K.expand_dims(ait)
 weighted_input = x * ait
output = K.sum(weighted_input, axis=1)
 return output
def compute_output_shape(self, input_shape):
return(input_shape[0], input_shape[-1])
 inputs = Input(shape=(MAX_SEQUENCE_LENGTH,))
embed = embedding_layer(inputs)
 gru = Bidirectional(GRU(100, dropout=0.2, recurrent_dropout=0.1, return_sequences=True))(embed)
 attention = AttLayer()(gru)
 output = Dense(1, activation='sigmoid' )(attention)
 model = Model(inputs, output)
 model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[auc])
 return model
tokenizer = AutoTokenizer.from_pretrained(MODEL )<load_from_csv>
model = get_model()
model.load_weights(".. /input/donorweight/atten0.7822.h5" )
valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')
 test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')
 sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')
<load_from_csv>
checkpoint = ModelCheckpoint('atten.h5', monitor="val_auc", verbose=1, save_best_only=True, mode='max')
 reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.000001 )
df_train = pd.read_csv("/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv")
 df_train['lang'] = 'lang'
df_train = df_train[['comment_text', 'toxic', 'lang']].head(0)
<load_from_csv>
<feature_engineering><EOS> 
p_sub= model.predict([test_data] ).T[0]
sub = pd.read_csv('.. /input/donorschoose-application-screening/sample_submission.csv')
 sub.project_is_approved = p_sub
sub.to_csv('attention.csv', index=False )