Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<choose_model_class>
start = time.time()
train = pd.read_csv(".. /input/train.csv",index_col="id",low_memory=False,
parse_dates=["project_submitted_datetime"])
 traindex = train.index
test = pd.read_csv(".. /input/test.csv",index_col="id",low_memory=False,
parse_dates=["project_submitted_datetime"])
 tesdex = test.index
y = train["project_is_approved"].copy()
df = pd.concat([train.drop("project_is_approved",axis=1),test],axis=0)
 alldex = df.index
rc = pd.read_csv(".. /input/resources.csv",index_col="id" ).fillna("missingpotato" )
class LossFunctionWrapper(tf.keras.losses.Loss):
def __init__(self,
fn,
reduction=losses_utils.ReductionV2.AUTO,
name=None,
**kwargs):
super(LossFunctionWrapper, self ).__init__(reduction=reduction, name=name)
 self.fn = fn
self._fn_kwargs = kwargs
def call(self, y_true, y_pred):
if tensor_util.is_tensor(y_pred)and tensor_util.is_tensor(y_true):
y_pred, y_true = tf_losses_util.squeeze_or_expand_dimensions(y_pred, y_true)
 return self.fn(y_true, y_pred, **self._fn_kwargs)
 def get_config(self):
config = {}
for k, v in six.iteritems(self._fn_kwargs):
config[k] = tf.keras.backend.eval(v)if tf_utils.is_tensor_or_variable(v)\
else v
base_config = super(LossFunctionWrapper, self ).get_config()
return dict(list(base_config.items())+ list(config.items()))<compute_test_metric>
agg_rc = rc.reset_index().groupby('id' ).agg(
 dict(quantity = ['sum',"mean"],
price = ["sum","mean","max","min","std"],
id = 'count',
description = lambda x: ' nicapotato '.join(x)) ).rename(columns={"id" : "count"})
 agg_rc.columns = pd.Index([e[0] +"_"+ e[1] for e in agg_rc.columns.tolist() ])
 agg_rc.rename({'count_count':"count",'description_<lambda>': "description"}, axis=1,inplace=True)
 agg_rc.price_std.fillna(0,inplace=True)
 df = pd.merge(df,agg_rc, left_index=True, right_index=True, how= "left")
 del test, train, rc,agg_rc
gc.collect()
def distance_based_log_loss(y_true, y_pred):
y_pred = ops.convert_to_tensor(y_pred)
 y_true = math_ops.cast(y_true, y_pred.dtype)
 margin = 1.0
p =(1.0 + tf.math.exp(-margin)) /(1.0 + tf.math.exp(y_pred - margin))
return tf.keras.backend.binary_crossentropy(target=y_true, output=p )<choose_model_class>
df['text'] = df.apply(lambda row: ' '.join([
str(row['project_essay_1']),
str(row['project_essay_2']),
str(row['project_essay_3']),
str(row['project_essay_4'])]), axis=1 )
class DBLLogLoss(LossFunctionWrapper):
def __init__(self, reduction=losses_utils.ReductionV2.AUTO,
name='distance_based_log_loss'):
super(DBLLogLoss, self ).__init__(distance_based_log_loss, name=name,
reduction=reduction )<categorify>
%%time
text_cols = ["text","project_resource_summary", "project_title", "description"]
print("Hand Made Text Features.. ")
 SIA = SentimentIntensityAnalyzer()
for cols in text_cols:
df[cols] = df[cols].astype(str)
 df[cols] = df[cols].str.lower()
df[cols+'_num_chars'] = df[cols].apply(len)
 df[cols+'_num_words'] = df[cols].apply(lambda comment: len(comment.split()))
df[cols+'_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 df[cols+'_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words']*100
if cols == "text":
df[cols+"_vader_Compound"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])
 df[cols+'_vader_Negative']= df[cols].apply(lambda x:SIA.polarity_scores(x)['neg'])
 df[cols+'_vader_Positive']= df[cols].apply(lambda x:SIA.polarity_scores(x)['pos'])
 print("{} Done".format(cols))
class MaskCalculator(tf.keras.layers.Layer):
def __init__(self, output_dim, **kwargs):
self.output_dim = output_dim
super(MaskCalculator, self ).__init__(**kwargs)
 def build(self, input_shape):
super(MaskCalculator, self ).build(input_shape)
 def call(self, inputs, **kwargs):
return tf.keras.backend.permute_dimensions(
 x=tf.keras.backend.repeat(
 x=tf.keras.backend.cast(
 x=tf.keras.backend.greater(
 x=inputs,
y=0 
),
dtype='float32' 
),
n=self.output_dim 
),
pattern=(0, 2, 1)
)
 def compute_output_shape(self, input_shape):
assert len(input_shape)== 1
shape = list(input_shape)
 shape.append(self.output_dim)
 return tuple(shape )<init_hyperparams>
df["Year"] = df["project_submitted_datetime"].dt.year
df["Date of Year"] = df['project_submitted_datetime'].dt.dayofyear
df["Weekday"] = df['project_submitted_datetime'].dt.weekday
df["Weekd of Year"] = df['project_submitted_datetime'].dt.week
df["Day of Month"] = df['project_submitted_datetime'].dt.day
df["Quarter"] = df['project_submitted_datetime'].dt.quarter
df = pd.merge(df, df["project_subject_categories"].str.get_dummies(sep=', '),
left_index=True, right_index=True, how="left")
 df = pd.merge(df, df["project_subject_subcategories"].str.get_dummies(sep=', '),
left_index=True, right_index=True, how="left")
 teachr_multi_subs = df['teacher_id'].value_counts().reset_index()
df["multi_apps"]= df['teacher_id'].isin(teachr_multi_subs.loc[teachr_multi_subs["teacher_id"]>1,'index'].tolist())
 print("Teacher App Distribution:
Two Apps: {}%
One App: {}%
".format(*df["multi_apps"].value_counts(normalize=True)*100))
df["Gender"] = None
df.loc[df['teacher_prefix'] == "Mr.","Gender"] = "Male"
df.loc[df['teacher_prefix'] == "Teacher","Gender"] = "Not Specified"
df.loc[(df['teacher_prefix'] == "Mrs.")|(df['teacher_prefix'] == "Ms."),"Gender"] = "Female"
print("Gender Distribution:
Female: {}%
Male: {}%
Not Specified: {}%".format(*df["Gender"].value_counts(normalize=True)*100))
class WeightPosteriorCallback(tf.keras.callbacks.Callback):
def __init__(self, log_every: int):
self.bayesian_layers = []
self.layer_names = []
self.logged_epochs = dict()
self.log_every = log_every
super(WeightPosteriorCallback, self ).__init__()
def on_train_begin(self, logs=None):
self.bayesian_layers = list(filter(
 lambda layer:(layer.name.lower().startswith('bayesianhiddenlayer')or \
layer.name.lower().startswith('bayesianoutputlayer')) ,
self.model.layers 
))
self.layer_names = []
max_layer_idx = 0
for layer in self.bayesian_layers:
if layer.name.lower().startswith('bayesianhiddenlayer'):
new_layer_name = 'Layer'
start_pos = len('bayesianhiddenlayer')
 find_idx = layer.name[start_pos:].find('_')
 assert find_idx > 0
layer_idx = int(layer.name[start_pos:(start_pos + find_idx)])
 new_layer_name += '{0}'.format(layer_idx)
 if layer_idx > max_layer_idx:
max_layer_idx = layer_idx
else:
new_layer_name = 'Layer{0}'.format(max_layer_idx + 1)
 self.layer_names.append(new_layer_name)
 self.logged_epochs = dict()
def on_epoch_end(self, epoch, logs=None):
if(epoch == 0)or(((epoch + 1)% self.log_every)== 0):
qm_vals = [layer.kernel_posterior.mean() for layer in self.bayesian_layers]
qs_vals = [layer.kernel_posterior.stddev() for layer in self.bayesian_layers]
self.logged_epochs[epoch + 1] =(qm_vals, qs_vals )<categorify>
dumyvars= ["Gender",'school_state','project_grade_category']
timevars = ['Weekday','Weekd of Year','Day of Month','Year','Date of Year',"Quarter"]
encode = ['multi_apps', "Date_Cutoff", 'teacher_prefix',"teacher_id"]
categorical_features = dumyvars + timevars + encode
lbl = preprocessing.LabelEncoder()
for col in categorical_features:
df[col] = lbl.fit_transform(df[col].astype(str))
def regular_encode(texts: List[str], tokenizer: XLMRobertaTokenizer,
maxlen: int)-> Tuple[np.ndarray, np.ndarray]:
err_msg = '"{0}" is wrong type for the text list!'.format(type(texts))
assert isinstance(texts, list)or isinstance(texts, tuple), err_msg
enc_di = tokenizer.batch_encode_plus(
 texts,
return_token_type_ids=False,
padding='max_length',
max_length=maxlen 
)
 err_msg = '{0} != {1}'.format(len(texts), len(enc_di['input_ids']))
assert len(texts)== len(enc_di['input_ids']), err_msg
err_msg = '{0} != {1}'.format(len(texts), len(enc_di['attention_mask']))
assert len(texts)== len(enc_di['attention_mask']), err_msg
encoded_tokens = np.zeros(( len(texts), maxlen), dtype=np.int32)
 encoded_masks = np.zeros(( len(texts), maxlen), dtype=np.int32)
 for sample_idx,(encoded_cur_text, encoded_cur_mask)in enumerate(
 zip(enc_di['input_ids'], enc_di['attention_mask'])
):
n_text = len(encoded_cur_text)
 n_mask = len(encoded_cur_mask)
 err_msg = 'Tokens and masks of texts "{0}" are different! '\
'{1} != {2}'.format(texts[sample_idx], n_text, n_mask)
 assert n_text == n_mask, err_msg
if n_text >= maxlen:
encoded_tokens[sample_idx] = np.array(encoded_cur_text[0:maxlen],
dtype=np.int32)
 encoded_masks[sample_idx] = np.array(encoded_cur_mask[0:maxlen],
dtype=np.int32)
 else:
padding = [0 for _ in range(maxlen - n_text)]
encoded_tokens[sample_idx] = np.array(encoded_cur_text + padding,
dtype=np.int32)
 encoded_masks[sample_idx] = np.array(encoded_cur_mask + padding,
dtype=np.int32)
 return encoded_tokens, encoded_masks<load_from_csv>
text_cols = ["project_resource_summary", "project_title","description","text"]
df.drop(['project_subject_categories',"project_subject_subcategories","project_submitted_datetime",
"project_essay_1","project_essay_2","project_essay_3","project_essay_4"
],axis=1,inplace=True)
 normalize = ["teacher_number_of_previously_posted_projects","quantity","price"]
gc.collect()
def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
assert len(sentiment_fields)> 0, 'List of sentiment fields is empty!'
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
for cur_field in sentiment_fields:
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(
 cur_field)
 assert cur_field in header, err_msg2
text_field_index = header.index(text_field)
 try:
lang_field_index = header.index(lang_field)
 except:
lang_field_index = -1
indices_of_sentiment_fields = []
for cur_field in sentiment_fields:
indices_of_sentiment_fields.append(header.index(cur_field))
else:
if len(row)== len(header):
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
max_proba = 0.0
for cur_field_idx in indices_of_sentiment_fields:
try:
cur_proba = float(row[cur_field_idx])
 except:
cur_proba = -1.0
err_msg2 = err_msg + ' Value {0} is wrong!'.format(
 row[cur_field_idx] 
)
 assert(cur_proba >= 0.0)and(cur_proba <= 1.0), err_msg2
if cur_proba > max_proba:
max_proba = cur_proba
new_label = 1 if max_proba >= 0.5 else 0
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, new_label))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang<load_from_csv>
print("
Dtypes of DF features:
",df.dtypes.value_counts())
 print("
DF Shape: {} Rows, {} Columns".format(*df.shape))
def load_test_set(file_name: str, id_field: str, text_field: str,
lang_field: str)-> Dict[str, List[Tuple[str, int]]]:
header = []
line_idx = 1
data_by_lang = dict()
with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore')as fp:
data_reader = csv.reader(fp, quotechar='"', delimiter=',')
 for row in data_reader:
if len(row)> 0:
err_msg = 'File "{0}": line {1} is wrong!'.format(file_name, line_idx)
 if len(header)== 0:
header = copy.copy(row)
 err_msg2 = err_msg + ' Field "{0}" is not found!'.format(text_field)
 assert text_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(id_field)
 assert id_field in header, err_msg2
err_msg2 = err_msg + ' Field "{0}" is not found!'.format(lang_field)
 assert lang_field in header, err_msg2
id_field_index = header.index(id_field)
 text_field_index = header.index(text_field)
 lang_field_index = header.index(lang_field)
 else:
if len(row)== len(header):
try:
id_value = int(row[id_field_index])
 except:
id_value = -1
err_msg2 = err_msg + ' {0} is wrong ID!'.format(
 row[id_field_index])
 assert id_value >= 0, err_msg2
text = row[text_field_index].strip()
assert len(text)> 0, err_msg + ' Text is empty!'
if lang_field_index >= 0:
cur_lang = row[lang_field_index].strip()
assert len(cur_lang)> 0, err_msg + ' Language is empty!'
else:
cur_lang = 'en'
if cur_lang not in data_by_lang:
data_by_lang[cur_lang] = []
data_by_lang[cur_lang].append(( text, id_value))
if line_idx % 10000 == 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx,
file_name))
line_idx += 1
if line_idx > 0:
if(line_idx - 1)% 10000 != 0:
print('{0} lines of the "{1}" have been processed...'.format(line_idx - 1,
file_name))
return data_by_lang<create_dataframe>
tfidf_para = {
"sublinear_tf":True,
"strip_accents":'unicode',
"stop_words":"english",
"analyzer":'word',
"token_pattern":r'\w{1,}',
"dtype":np.float32,
"norm":'l2',
"min_df":5,
"max_df":.9,
"smooth_idf":False
}
def build_siamese_dataset(texts: Dict[str, List[Tuple[str, int]]],
dataset_size: int, tokenizer: XLMRobertaTokenizer,
maxlen: int, batch_size: int,
shuffle: bool)-> Tuple[tf.data.Dataset, int]:
language_pairs = set()
for language in texts.keys() :
for other_language in texts:
if other_language == language:
language_pairs.add(( language, other_language))
else:
new_pair =(language, other_language)
 new_pair_2 =(other_language, language)
 if(new_pair not in language_pairs)and(new_pair_2 not in language_pairs):
language_pairs.add(new_pair)
 language_pairs = sorted(list(language_pairs))
print('Possible language pairs are: {0}.'.format(language_pairs))
err_msg = '{0} is too small size of the data set!'.format(dataset_size)
 assert dataset_size >=(len(language_pairs)* 10), err_msg
n_samples_for_lang_pair = int(np.ceil(dataset_size / float(len(language_pairs))))
text_pairs_and_labels = []
for left_lang, right_lang in language_pairs:
print('{0}-{1}:'.format(left_lang, right_lang))
left_positive_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] > 0, range(len(texts[left_lang])) 
))
left_negative_indices = list(filter(
 lambda idx: texts[left_lang][idx][1] == 0, range(len(texts[left_lang])) 
))
right_positive_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] > 0, range(len(texts[right_lang])) 
))
right_negative_indices = list(filter(
 lambda idx: texts[right_lang][idx][1] == 0, range(len(texts[right_lang])) 
))
used_pairs = set()
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4):
left_idx = random.choice(left_positive_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1 
)
)
 number_of_samples += 1
print(' number of "1-1" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(n_samples_for_lang_pair // 4,(2 * n_samples_for_lang_pair)// 4):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_negative_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_negative_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
1 
)
)
 number_of_samples += 1
print(' number of "0-0" pairs is {0};'.format(number_of_samples))
number_of_samples = 0
for _ in range(( 2 * n_samples_for_lang_pair)// 4, n_samples_for_lang_pair):
left_idx = random.choice(left_negative_indices)
 right_idx = random.choice(right_positive_indices)
 counter = 0
while(( right_idx == left_idx)or(( left_idx, right_idx)in used_pairs)or 
(( right_idx, left_idx)in used_pairs)) and(counter < 100):
right_idx = random.choice(right_positive_indices)
 counter += 1
if counter < 100:
used_pairs.add(( left_idx, right_idx))
used_pairs.add(( right_idx, left_idx))
if random.random() >= 0.5:
text_pairs_and_labels.append(
(
 texts[left_lang][left_idx][0],
texts[right_lang][right_idx][0],
0 
)
)
 else:
text_pairs_and_labels.append(
(
 texts[right_lang][right_idx][0],
texts[left_lang][left_idx][0],
0 
)
)
 number_of_samples += 1
print(' number of "0-1" or "1-0" pairs is {0}.'.format(number_of_samples))
random.shuffle(text_pairs_and_labels)
 n_steps = len(text_pairs_and_labels)// batch_size
print('Samples number of the data set is {0}.'.format(len(text_pairs_and_labels)))
 print('Samples number per each language pair is {0}.'.format(n_samples_for_lang_pair))
tokens_of_left_texts, mask_of_left_texts = regular_encode(
 texts=[cur[0] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 tokens_of_right_texts, mask_of_right_texts = regular_encode(
 texts=[cur[1] for cur in text_pairs_and_labels],
tokenizer=tokenizer, maxlen=maxlen 
)
 siamese_labels = np.array([cur[2] for cur in text_pairs_and_labels], dtype=np.int32)
 print('Number of positive siamese samples is {0} from {1}.'.format(
 int(sum(siamese_labels)) , siamese_labels.shape[0]))
if shuffle:
err_msg = '{0} is too small number of samples for the data set!'.format(
 len(text_pairs_and_labels))
assert n_steps >= 50, err_msg
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_left_texts, mask_of_left_texts,
tokens_of_right_texts, mask_of_right_texts 
),
siamese_labels 
)
 ).repeat().batch(batch_size)
 else:
dataset = tf.data.Dataset.from_tensor_slices(
(
(
 tokens_of_left_texts, mask_of_left_texts,
tokens_of_right_texts, mask_of_right_texts 
),
siamese_labels 
)
 ).batch(batch_size)
 del text_pairs_and_labels
return dataset, n_steps<load_pretrained>
def get_col(col_name):
return lambda x: x[col_name]
df["project_title_count"] = df["project_title"].copy()
textcols = ["text","project_resource_summary","project_title", "project_title_count","description"]
vectorizer = FeatureUnion([ 
('text',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=20000,
**tfidf_para,
preprocessor=get_col('text'))), 
('project_resource_summary',TfidfVectorizer(
 ngram_range=(1, 2),
**tfidf_para,
max_features=2000,
preprocessor=get_col('project_resource_summary'))), 
('project_title',TfidfVectorizer(
 ngram_range=(1, 2),
**tfidf_para,
max_features=1500,
preprocessor=get_col('project_title'))), 
('project_title_count',CountVectorizer(
 ngram_range=(1, 2),
max_features=1500,
preprocessor=get_col('project_title_count'))), 
('description',TfidfVectorizer(
 ngram_range=(1, 2),
**tfidf_para,
max_features=2400,
preprocessor=get_col('description'))),
] )
def build_feature_extractor(transformer_name: str, hidden_state_size: int,
max_len: int)-> tf.keras.Model:
word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_word_ids")
 attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="base_attention_mask")
 transformer_layer = TFXLMRobertaModel.from_pretrained(
 pretrained_model_name_or_path=transformer_name,
name='Transformer' 
)
 sequence_output = transformer_layer([word_ids, attention_mask])[0]
output_mask = MaskCalculator(
 output_dim=hidden_state_size, trainable=False,
name='OutMaskCalculator'
)(attention_mask)
 masked_sequence_output = tf.keras.layers.Multiply(
 name='OutMaskMultiplicator'
)([output_mask, sequence_output])
 masked_sequence_output = tf.keras.layers.Masking(
 name='OutMasking'
)(masked_sequence_output)
 pooled_output = tf.keras.layers.GlobalAvgPool1D(name='AvePool' )(masked_sequence_output)
 text_embedding = tf.keras.layers.Lambda(
 lambda x: tf.math.l2_normalize(x, axis=1),
name='Emdedding'
)(pooled_output)
 fe_model = tf.keras.Model(
 inputs=[word_ids, attention_mask],
outputs=text_embedding,
name='FeatureExtractor' 
)
 fe_model.build(input_shape=[(None, max_len),(None, max_len)])
 return fe_model<compute_test_metric>
start_vect=time.time()
ready_df = vectorizer.fit_transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
print("Vectorization Runtime: %0.2f Minutes"%(( time.time() - start_vect)/60))
def euclidean_distance(vects):
x, y = vects
sum_square = tf.keras.backend.sum(tf.keras.backend.square(x - y),
axis=1, keepdims=True)
 return tf.keras.backend.sqrt(
 tf.keras.backend.maximum(sum_square, tf.keras.backend.epsilon())
 )<concatenate>
df.drop(textcols,axis=1, inplace=True)
 print("
Dtypes of DF features:
",df.dtypes.value_counts())
 print("
DF Shape: {} Rows, {} Columns".format(*df.shape))
def eucl_dist_output_shape(shapes):
shape1, shape2 = shapes
return(shape1[0], 1 )<choose_model_class>
def column_index(df, query_cols):
cols = df.columns.values
sidx = np.argsort(cols)
 return sidx[np.searchsorted(cols,query_cols,sorter=sidx)]
categorical_features_pos = column_index(df,categorical_features )
def build_siamese_nn(transformer_name: str, hidden_state_size: int, max_len: int,
lr: float)-> Tuple[tf.keras.Model, tf.keras.Model]:
left_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_word_ids")
 left_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="left_attention_mask")
 right_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_word_ids")
 right_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,
name="right_attention_mask")
 fe_ = build_feature_extractor(transformer_name, hidden_state_size, max_len)
 left_text_embedding = fe_([left_word_ids, left_attention_mask])
 right_text_embedding = fe_([right_word_ids, right_attention_mask])
 distance_layer = tf.keras.layers.Lambda(
 function=euclidean_distance,
output_shape=eucl_dist_output_shape,
name='L2DistLayer'
)([left_text_embedding, right_text_embedding])
 nn = tf.keras.Model(
 inputs=[left_word_ids, left_attention_mask, right_word_ids, right_attention_mask],
outputs=distance_layer,
name='SiameseXLMR' 
)
 nn.compile(
 optimizer=tfa.optimizers.AdamW(learning_rate=lr, weight_decay=1e-5),
loss=DBLLogLoss() 
)
 fe_.summary()
nn.summary()
return nn, fe_<choose_model_class>
X = hstack([csr_matrix(df.loc[traindex,:].values),ready_df[0:traindex.shape[0]]])
 testing = hstack([csr_matrix(df.loc[tesdex,:].values),ready_df[traindex.shape[0]:]])
 tfvocab = df.columns.tolist() + tfvocab
for shape in [X,testing]:
print("{} Rows and {} Cols".format(*shape.shape))
print("Feature Names Length: ",len(tfvocab))
del df
gc.collect() ;
def build_classifier(language: str, feature_vector_size: int, n_train_samples: int,
hidden_layer_size: int, n_hidden_layers: int,
C: float, verbose: bool)-> tf.keras.Model:
random_seed = 42
sentence_features = tf.keras.layers.Input(
 shape=(feature_vector_size,), dtype=tf.float32,
name="SentenceFeatures_{0}".format(language)
)
 n_samples = tf.cast(float(n_train_samples), dtype=tf.float32)
 kl_weight = tf.cast(1.0 / C, dtype=tf.float32)
 kl_divergence_function =(
 lambda q, p, _:(tfp.distributions.kl_divergence(q, p)* kl_weight / n_samples)
)
 if n_hidden_layers > 0:
hidden_layer = tfp.layers.DenseFlipout(
 units=hidden_layer_size,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation=None, seed=random_seed,
name='BayesianHiddenLayer1_{0}'.format(language)
 )(sentence_features)
 hidden_layer = tf.keras.layers.LayerNormalization(
 name='LayerNorm1_{0}'.format(language)
 )(hidden_layer)
 hidden_layer = tf.keras.layers.Activation(
 activation=tf.keras.activations.relu,
name='Activation1_{0}'.format(language)
 )(hidden_layer)
 for layer_idx in range(1, n_hidden_layers):
hidden_layer = tfp.layers.DenseFlipout(
 units=hidden_layer_size,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation=None, seed=random_seed,
name='BayesianHiddenLayer{0}_{1}'.format(layer_idx + 1, language)
 )(hidden_layer)
 hidden_layer = tf.keras.layers.LayerNormalization(
 name='LayerNorm{0}_{1}'.format(layer_idx + 1, language)
 )(hidden_layer)
 hidden_layer = tf.keras.layers.Activation(
 activation=tf.keras.activations.relu,
name='Activation{0}_{1}'.format(layer_idx + 1, language)
 )(hidden_layer)
 cls_layer = tfp.layers.DenseFlipout(
 units=1,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation='sigmoid',
name='BayesianOutputLayer_{0}'.format(language),
seed=random_seed
)(hidden_layer)
 else:
cls_layer = tfp.layers.DenseFlipout(
 units=1,
kernel_divergence_fn=kl_divergence_function,
bias_divergence_fn=kl_divergence_function,
activation='sigmoid',
name='BayesianOutputLayer_{0}'.format(language),
seed=random_seed
)(sentence_features)
 cls_model = tf.keras.Model(
 inputs=sentence_features,
outputs=cls_layer,
name='BayesianNN_{0}'.format(language)
)
 radam = tfa.optimizers.RectifiedAdam(learning_rate=1e-2)
 ranger = tfa.optimizers.Lookahead(radam, sync_period=6, slow_step_size=0.5)
 cls_model.compile(
 optimizer=ranger, loss='binary_crossentropy',
experimental_run_tf_function=False 
)
 if verbose:
cls_model.summary()
return cls_model<prepare_x_and_y>
imbalance_weight = y.value_counts(normalize=True)[0]/y.value_counts(normalize=True)[1]
print("Imbalance Weight: ",imbalance_weight )
def build_datasets_for_classifier(
 data_for_training: Tuple[np.ndarray, np.ndarray],
data_split: Dict[str, Tuple[np.ndarray, np.ndarray]],
language_for_testing: str 
)-> Tuple[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]:
indices_for_training, indices_for_testing = data_split[language_for_testing]
X_train = data_for_training[0][indices_for_training]
y_train = data_for_training[1][indices_for_training]
X_test = data_for_training[0][indices_for_testing]
y_test = data_for_training[1][indices_for_testing]
del indices_for_training, indices_for_testing
data_for_training =(X_train, y_train)
 data_for_testing =(X_test, y_test)
 return data_for_training, data_for_testing<choose_model_class>
X_train, X_valid, y_train, y_valid = train_test_split(
 X, y, test_size=0.25, random_state=23)
 d_train = xgb.DMatrix(X_train, y_train,feature_names=tfvocab)
 d_valid = xgb.DMatrix(X_valid, y_valid,feature_names=tfvocab)
 d_test = xgb.DMatrix(testing,feature_names=tfvocab )
def train_classifier(trainset: tf.data.Dataset, n_steps: int, bayesian_classifier: tf.keras.Model,
max_epochs: int, verbose: bool, tmp_file_name: str):
callbacks = [
tf.keras.callbacks.ReduceLROnPlateau(
 monitor='loss', factor=0.1, patience=3, verbose=False,
mode='min', min_delta=0.05, cooldown=5, min_lr=1e-7 
),
tf.keras.callbacks.ModelCheckpoint(
 monitor='loss', mode="min", save_weights_only=True, save_best_only=True,
filepath=tmp_file_name 
)
 ]
if verbose:
weight_posterior_callback = WeightPosteriorCallback(
 log_every = max(2, max_epochs // 10)
)
 callbacks.append(weight_posterior_callback)
 else:
weight_posterior_callback = None
if verbose:
print('n_epochs = {0}, steps_per_epoch = {1}'.format(max_epochs, n_steps))
history = bayesian_classifier.fit(
 trainset,
steps_per_epoch=n_steps,
epochs=max_epochs, callbacks=callbacks,
verbose=verbose 
)
 if verbose:
show_training_process(history, 'loss', figure_id=1)
 plot_weight_posteriors(layer_names=weight_posterior_callback.layer_names,
logged_epochs=weight_posterior_callback.logged_epochs)
 del history, callbacks
if os.path.exists(tmp_file_name):
bayesian_classifier.load_weights(tmp_file_name )<predict_on_test>
xgb_params = {'eta': 0.05,
'max_depth': 12,
'subsample': 0.8,
'colsample_bytree': 0.75,
'scale_pos_weight': imbalance_weight,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'seed': 23,
'lambda': 1.5,
'alpha':.6
}
def evaluate_classifier(X_test: np.ndarray, y_test: np.ndarray, language_for_testing: str,
bayesian_nn: tf.keras.Model,
n_monte_carlo: int, batch_size: int, verbose: bool)-> float:
probabilities = predict_with_model(
 classifier=bayesian_nn,
input_data=X_test,
batch_size=batch_size,
n_monte_carlo=n_monte_carlo 
)
 err_msg = '{0} != {1}'.format(y_test.shape, probabilities.shape)
 assert y_test.shape == probabilities.shape, err_msg
if verbose:
show_roc_auc(y_true=y_val, probabilities=probabilities,
label='the testing data(language "{0}")'.format(language_for_testing),
figure_id=3)
 quality = roc_auc_score(y_true=y_test, y_score=probabilities)
 del probabilities
return quality<train_model>
modelstart = time.time()
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(xgb_params, d_train, 2000, watchlist, verbose_eval=50, early_stopping_rounds=50)
 xgb_pred = model.predict(d_test)
 xgb_sub = pd.DataFrame(xgb_pred,columns=["project_is_approved"],index=tesdex)
 xgb_sub.to_csv("xgb_sub.csv",index=True)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
del d_train, d_valid, d_test
gc.collect() ;
def train_siamese_nn(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,
steps_per_epoch: int, validset: tf.data.Dataset, max_duration: int):
assert steps_per_trainset >= steps_per_epoch
n_epochs = int(round(10.0 * steps_per_trainset / float(steps_per_epoch)))
 print('Maximal duration of the Siamese NN training is {0} seconds.'.format(max_duration))
callbacks = [
tf.keras.callbacks.EarlyStopping(patience=5, monitor='val_loss', mode='min',
restore_best_weights=False, verbose=True),
tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)
 ]
history = nn.fit(
 trainset,
steps_per_epoch=steps_per_epoch,
validation_data=validset,
epochs=n_epochs,
callbacks=callbacks 
)
 show_training_process(history, 'loss' )<define_variables>
lgbm_params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 35,
'learning_rate': 0.01,
'feature_fraction': 0.80,
'bagging_fraction': 0.80,
'bagging_freq': 5,
'max_bin':300,
'lambda_l2': 1.5,
'is_unbalance': True
}
lgtrain = lgb.Dataset(X_train, y_train,
feature_name=tfvocab,
categorical_feature = categorical_features)
 lgvalid = lgb.Dataset(X_valid, y_valid,
feature_name=tfvocab,
categorical_feature = categorical_features)
 modelstart = time.time()
lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=2000,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=150
)
def calculate_features_of_texts(texts: Dict[str, List[Tuple[str, int]]],
tokenizer: XLMRobertaTokenizer, maxlen: int,
fe: tf.keras.Model, batch_size: int,
max_dataset_size: int = 0)-> \
Dict[str, Tuple[np.ndarray, np.ndarray]]:
languages = sorted(list(texts.keys()))
datasets_by_languages = dict()
if max_dataset_size > 0:
max_size_per_lang = max_dataset_size // len(languages)
 err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)
 assert max_size_per_lang > 0, err_msg
else:
max_size_per_lang = 0
for cur_lang in languages:
selected_indices = list(range(len(texts[cur_lang])))
 if max_size_per_lang > 0:
if len(selected_indices)> max_size_per_lang:
selected_indices = random.sample(
 population=selected_indices,
k=max_size_per_lang 
)
 tokens_of_texts, mask_of_texts = regular_encode(
 texts=[texts[cur_lang][idx][0] for idx in selected_indices],
tokenizer=tokenizer, maxlen=maxlen 
)
 X = []
n_batches = int(np.ceil(len(selected_indices)/ float(batch_size)))
 for batch_idx in range(n_batches):
batch_start = batch_idx * batch_size
batch_end = min(len(selected_indices), batch_start + batch_size)
 res = fe.predict_on_batch(
 [
tokens_of_texts[batch_start:batch_end],
mask_of_texts[batch_start:batch_end]
] 
)
 if not isinstance(res, np.ndarray):
res = res.numpy()
X.append(res)
 del res
X = np.vstack(X)
 y = np.array([texts[cur_lang][idx][1] for idx in selected_indices], dtype=np.int32)
 datasets_by_languages[cur_lang] =(X, y)
 del X, y, selected_indices
return datasets_by_languages<predict_on_test>
lgbmpred = lgb_clf.predict(testing)
 lgbm_sub = pd.DataFrame(lgbmpred,columns=["project_is_approved"],index=tesdex)
 lgbm_sub.to_csv("lgbm_sub.csv",index=True)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
del lgvalid, lgtrain
gc.collect() ;
def predict_with_model(classifier: tf.keras.Model, input_data: np.ndarray,
batch_size: int, n_monte_carlo: int)-> np.ndarray:
assert n_monte_carlo > 1
predicted = classifier.predict(input_data, batch_size=batch_size ).flatten()
for _ in range(n_monte_carlo - 1):
predicted += classifier.predict(input_data, batch_size=batch_size ).flatten()
return predicted / float(n_monte_carlo )<load_pretrained>
print("Model Correlation: ", pd.Series(lgbmpred ).corr(pd.Series(xgb_pred)) )
def select_best_model(data_for_training: List[Dict[str, Tuple[np.ndarray, np.ndarray]]],
languages_for_training: List[str],
current_strategy: tf.distribute.Strategy,
tpu_system: Union[None, tf.distribute.cluster_resolver.TPUClusterResolver],
feature_vector_size: int, hidden_layer_size: int,
n_monte_carlo: int, max_iters: int, batch_size: int,
tmp_file_name: str)-> Dict[str, float]:
assert len(data_for_training)== len(languages_for_training)
 space = [Real(1e-1, 1e+4, "log-uniform", name='C'),
Integer(0, 4, name='neural_depth')]
n_calls = 36
n_random_starts = 6
n_restarts_optimizer = 3
restart_counter = 1
datasets_for_training = []
for lang_idx in range(len(languages_for_training)) :
n_train_samples = data_for_training[lang_idx]['train'][1].shape[0]
steps_per_epoch = int(np.ceil(n_train_samples / float(batch_size)))
 datasets_for_training.append(
 {
'train': tf.data.Dataset.from_tensor_slices(
 data_for_training[lang_idx]['train']
).repeat().shuffle(n_train_samples ).batch(batch_size),
'n_epochs': int(np.ceil(max_iters / float(n_train_samples))),
'steps_per_epoch': steps_per_epoch
} 
)
 @use_named_args(space)
 def objective_f(C: float, neural_depth: int)-> float:
nonlocal restart_counter
nonlocal current_strategy
nonlocal tpu_system
rocauc_scores = []
n_splits = len(languages_for_training)
 for lang_idx in range(len(languages_for_training)) :
validation_lang = languages_for_training[lang_idx]
with current_strategy.scope() :
bnn = build_classifier(
 feature_vector_size=feature_vector_size, n_train_samples=max_iters,
hidden_layer_size=hidden_layer_size, n_hidden_layers=neural_depth,
C=float(C), verbose=False,
language='{0}{1}'.format(validation_lang.strip() , restart_counter)
)
 train_classifier(
 trainset=datasets_for_training[lang_idx]['train'],
n_steps=datasets_for_training[lang_idx]['steps_per_epoch'],
bayesian_classifier=bnn, verbose=False,
max_epochs=datasets_for_training[lang_idx]['n_epochs'],
tmp_file_name=tmp_file_name 
)
 instant_quality = evaluate_classifier(
 X_test=data_for_training[lang_idx]['test'][0],
y_test=data_for_training[lang_idx]['test'][1],
language_for_testing=languages_for_training[lang_idx],
bayesian_nn=bnn,
n_monte_carlo=n_monte_carlo, batch_size=batch_size, verbose=False 
)
 rocauc_scores.append(instant_quality)
 del bnn
gc.collect()
tf.keras.backend.clear_session()
if os.path.isfile(tmp_file_name):
os.remove(tmp_file_name)
 rocauc_score = hmean(rocauc_scores)
 del rocauc_scores
print(' C={0:.9f}, neural_depth={1}'.format(C, neural_depth))
print(' ROC-AUC score = {0:.9f}'.format(rocauc_score))
restart_counter += 1
if tpu_system:
tf.tpu.experimental.shutdown_tpu_system(tpu_system)
 del current_strategy
tf.tpu.experimental.initialize_tpu_system(tpu_system)
 current_strategy = tf.distribute.experimental.TPUStrategy(tpu_system)
 return -rocauc_score
start_time = time.time()
res_gp = gp_minimize(
 objective_f, space,
n_calls=n_calls, n_random_starts=n_random_starts,
n_restarts_optimizer=n_restarts_optimizer, random_state=42,
verbose=True, n_jobs=1 
)
 best_parameters = {
'C': float(res_gp.x[0]),
'neural_depth': int(res_gp.x[1]),
}
automl_duration = int(round(time.time() - start_time))
print('')
 print('Total duration of the AutoML is {0} seconds.'.format(automl_duration))
print('')
 print('Best parameters are:')
 print('C={0:.9f}, neural_depth={1}'.format(best_parameters['C'],
best_parameters['neural_depth']))
print('')
 del datasets_for_training
plot_convergence(res_gp)
 plot_evaluations(res_gp, bins=10)
 return best_parameters<define_variables>
blend =(lgbmpred*.55)+(xgb_pred*.45)
 blend_sub = pd.DataFrame(blend,columns=["project_is_approved"],index=tesdex)
 blend_sub.to_csv("boost_blend_sub.csv",index=True )
experiment_start_time = time.time()<define_variables>
concat_subs = pd.concat([lgbm_sub,xgb_sub,blend_sub], axis=1)
 concat_subs.columns = ["LGBM","XGB","BLEND"]
concat_subs.corr()
<load_pretrained><EOS> 
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - start)/60))