Kaggle competition: DonorsChoose.org Application Screening
dataset_for_training = calculate_features_of_texts(
 texts=multilingual_corpus,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
fe=feature_extractor,
batch_size=batch_size_for_siamese 
)
 assert len(dataset_for_training)== 3<feature_engineering>
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 resources = pd.read_csv(".. /input/resources.csv")
 train = train.sort_values(by="project_submitted_datetime")
dataset_for_submission = calculate_features_of_texts(
 texts=texts_for_submission,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
fe=feature_extractor,
batch_size=batch_size_for_siamese
)<split>
print("Number teachers train : %s, Number teachers test : %s, Overlap : %s "%(len(teachers_train), len(teachers_test), len(inter)) )
X_embedded = []
y_embedded = []
split_by_languages = dict()
start_pos = 0
for cur_lang in dataset_for_training:
X_embedded.append(dataset_for_training[cur_lang][0])
 y_embedded.append(dataset_for_training[cur_lang][1])
 split_by_languages[cur_lang] =(
 set() ,
set(range(start_pos, start_pos + dataset_for_training[cur_lang][1].shape[0])) 
)
 start_pos = start_pos + dataset_for_training[cur_lang][1].shape[0]
featured_data_for_training =(
 np.vstack(X_embedded),
np.concatenate(y_embedded)
)
 for cur_lang in dataset_for_training:
indices_for_testing = split_by_languages[cur_lang][1]
indices_for_training = set(range(featured_data_for_training[0].shape[0])) - indices_for_testing
split_by_languages[cur_lang] =(
 np.array(sorted(list(indices_for_training)) , dtype=np.int32),
np.array(sorted(list(indices_for_testing)) , dtype=np.int32)
)
 del indices_for_training, indices_for_testing
featured_data_for_submission = []
identifies_for_submission = []
for cur_lang in dataset_for_submission:
X_embedded.append(dataset_for_submission[cur_lang][0])
 featured_data_for_submission.append(dataset_for_submission[cur_lang][0])
 identifies_for_submission.append(dataset_for_submission[cur_lang][1])
 y_embedded.append(
 np.array(
 [-1 for _ in range(dataset_for_submission[cur_lang][0].shape[0])],
dtype=np.int32 
)
)
 featured_data_for_submission = np.vstack(featured_data_for_submission)
 identifies_for_submission = np.concatenate(identifies_for_submission)
 X_embedded = np.vstack(X_embedded)
 y_embedded = np.concatenate(y_embedded )<drop_column>
char_cols = ['project_subject_categories', 'project_subject_subcategories',
'project_title', 'project_essay_1', 'project_essay_2',
'project_essay_3', 'project_essay_4', 'project_resource_summary']
del dataset_for_training, dataset_for_submission
del feature_extractor, xlmroberta_tokenizer<prepare_x_and_y>
resources['total_price'] = resources.quantity * resources.price
mean_total_price = pd.DataFrame(resources.groupby('id' ).total_price.mean())
 sum_total_price = pd.DataFrame(resources.groupby('id' ).total_price.sum())
 count_total_price = pd.DataFrame(resources.groupby('id' ).total_price.count())
 mean_total_price['id'] = mean_total_price.index
sum_total_price['id'] = mean_total_price.index
count_total_price['id'] = mean_total_price.index
def create_features(df):
df = pd.merge(df, mean_total_price, on='id')
 df = pd.merge(df, sum_total_price, on='id')
 df = pd.merge(df, count_total_price, on='id')
 df['year'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[0])
 df['month'] = df.project_submitted_datetime.apply(lambda x: x.split("-")[1])
 for col in char_cols:
df[col] = df[col].fillna("NA")
 df['text'] = df.apply(lambda x: " ".join(x[col] for col in char_cols), axis=1)
 return df
train = create_features(train)
 test = create_features(test)
indices_of_samples = random.sample(
 list(range(featured_data_for_training[1].shape[0])) ,
k=1000 
)
 indices_of_samples += random.sample(
 list(range(featured_data_for_training[1].shape[0], y_embedded.shape[0])) ,
k=1000 
)
 X_embedded = X_embedded[indices_of_samples]
y_embedded = y_embedded[indices_of_samples]<categorify>
cat_features = ["teacher_prefix", "school_state", "year", "month", "project_grade_category", "project_subject_categories", "project_subject_subcategories"]
num_features = ["teacher_number_of_previously_posted_projects", "total_price_x", "total_price_y", "total_price"]
cat_features_hash = [col+"_hash" for col in cat_features]
X_embedded = TSNE(n_components=2, n_jobs=-1, random_state=42 ).fit_transform(X_embedded )<drop_column>
max_size=15000
def feature_hash(df, max_size=max_size):
for col in cat_features:
df[col+"_hash"] = df[col].apply(lambda x: hash(x)%max_size)
 return df
del indices_of_negative_classes
del indices_of_positive_classes
del indices_of_unknown_classes
del indices_of_samples
del X_embedded, y_embedded<set_options>
train = feature_hash(train)
 test = feature_hash(test )
gc.collect()
tf.keras.backend.clear_session()
if tpu:
tf.tpu.experimental.shutdown_tpu_system(tpu)
 del strategy
tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu )<split>
max_features = 50000
maxlen = 300
scaler = StandardScaler()
X_train_num = scaler.fit_transform(train[num_features])
 X_test_num = scaler.transform(test[num_features])
 X_train_cat = np.array(train[cat_features_hash], dtype=np.int)
 X_test_cat = np.array(test[cat_features_hash], dtype=np.int)
 tokenizer = text.Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(train["text"].tolist() +test["text"].tolist())
 list_tokenized_train = tokenizer.texts_to_sequences(train["text"].tolist())
 list_tokenized_test = tokenizer.texts_to_sequences(test["text"].tolist())
 X_train_words = sequence.pad_sequences(list_tokenized_train, maxlen=maxlen)
 X_test_words = sequence.pad_sequences(list_tokenized_test, maxlen=maxlen)
 X_train_target = train.project_is_approved
splitted_data_for_training = []
for cur_lang in all_languages:
datasets = build_datasets_for_classifier(
 data_for_training=featured_data_for_training,
data_split=split_by_languages,
language_for_testing=cur_lang 
)
 splitted_data_for_training.append(
 {
'train': datasets[0],
'test': datasets[1]
} 
)
 del datasets<train_model>
def get_model() :
input_cat = Input(( len(cat_features_hash),))
input_num = Input(( len(num_features),))
input_words = Input(( maxlen,))
x_cat = Embedding(max_size, 10 )(input_cat)
 x_cat = Flatten()(x_cat)
 x_cat = Dropout(0.5 )(x_cat)
 x_words = Embedding(max_features, 100 )(input_words)
 x_words = Convolution1D(100, 3, activation="relu" )(x_words)
 x_words = GlobalMaxPool1D()(x_words)
 x_words = Dropout(0.5 )(x_words)
 x_cat = Dense(100, activation="relu" )(x_cat)
 x_num = Dense(100, activation="relu" )(input_num)
 x_num = Dropout(0.5 )(x_num)
 x = concatenate([x_cat, x_num, x_words])
 x = Dense(50, activation="relu" )(x)
 x = Dropout(0.5 )(x)
 predictions = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=[input_cat, input_num, input_words], outputs=predictions)
 model.compile(optimizer=optimizers.Adam(0.001, decay=1e-6),
loss='binary_crossentropy',
metrics=['accuracy'])
 return model
experiment_duration = int(round(time.time() - experiment_start_time))
print('Duration of siamese XLM-RoBERTa preparing is {0} seconds.'.format(
 experiment_duration))<define_variables>
model = get_model()
bnn_params = select_best_model(
 data_for_training=splitted_data_for_training,
languages_for_training=all_languages,
current_strategy=strategy, tpu_system=tpu,
feature_vector_size=featured_data_for_training[0].shape[1],
hidden_layer_size=hidden_layer_of_cls,
batch_size=batch_size_for_cls, max_iters=max_iters_of_cls,
n_monte_carlo=automl_num_monte_carlo,
tmp_file_name=tmp_cls_name
)<drop_column>
pred_test = model.predict([X_test_cat, X_test_num, X_test_words] )
<drop_column><EOS> 
test["project_is_approved"] = pred_test
test[['id', 'project_is_approved']].to_csv("baseline_submission.csv", index=False )