Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<load_from_csv>
color = sns.color_palette()
%matplotlib inline
py.init_notebook_mode(connected=True)
 pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
texts_for_submission = load_test_set(
 os.path.join(dataset_dir, "test.csv"),
text_field="content", lang_field="lang", id_field="id" 
)
 print('Data for submission:')
 for language in sorted(list(texts_for_submission.keys())) :
print(' {0}\t\t{1} samples'.format(language, len(texts_for_submission[language])) )<create_dataframe>
df = pd.read_csv('.. /input/donorschoose-application-screening/train.csv')
 df.sort_values('project_submitted_datetime' ).head(3 )
dataset_for_training, n_batches_per_data = build_siamese_dataset(
 texts=corpus_for_training, dataset_size=150000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, shuffle=True
)<create_dataframe>
resources_df = pd.read_csv('.. /input/donorschoose-application-screening/resources.csv')
 resources_df.head(3 )
dataset_for_validation, n_batches_per_epoch = build_siamese_dataset(
 texts=corpus_for_validation, dataset_size=1000,
tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,
batch_size=batch_size_for_siamese, shuffle=False
)<set_options>
resources_df['amount'] = resources_df['quantity']*resources_df['price']
amount_df = resources_df.groupby('id')['amount'].agg('sum' ).sort_values(ascending=False ).reset_index()
resource_amount_map = {}
for i, row in amount_df.iterrows() :
resource_amount_map[row['id']] = row['amount']
df.insert(4,'total_amount',df['id'].map(resource_amount_map))
df.tail(3 )
del corpus_for_training, corpus_for_validation
gc.collect()<train_model>
state_df = df[df['project_is_approved']==1]
state_df = state_df.groupby('school_state')['project_is_approved'].agg('sum' ).reset_index().sort_values('school_state' ).reset_index()
a_df = df.groupby('school_state')['total_amount'].agg('sum' ).reset_index().sort_values('school_state' ).reset_index()
submit_df = df['school_state'].value_counts().reset_index()
submit_df.columns=['school_state','total_submit']
submit_df = submit_df.sort_values('school_state' ).reset_index()
data = [ dict(
 type='choropleth',
colorscale = scl,
autocolorscale = False,
locations = state_df['school_state'],
z = a_df['total_amount'].astype(int)/state_df['project_is_approved'].astype(int),
locationmode = 'USA-states',
marker = dict(
 line = dict(
 color = 'rgb(255,255,255)',
width = 2 
)
),
colorbar = dict(
 title = "USD" 
)
)]
layout = dict(
 title = 'US DonorChoose Average Funding in States',
geo = dict(
 scope='usa',
projection=dict(type='albers usa'),
showlakes = True,
lakecolor = 'rgb(255, 255, 255)', 
), 
)
 fig = dict(data=data, layout=layout)
 url = py.iplot(fig, filename='donor-state-funding' )
preparing_duration = int(round(time.time() - experiment_start_time))
print("Duration of data loading and preparing to the Siamese NN training is "
"{0} seconds.".format(preparing_duration))<choose_model_class>
cat_df = df[['project_subject_categories','project_subject_subcategories','project_is_approved']]
new_cat,new_sub,new_approve = [],[],[]
for i, row in cat_df.iterrows() :
cats = row['project_subject_categories'].split(', ')
 subs = row['project_subject_subcategories'].split(', ')
 for j in range(len(cats)) :
for k in range(len(subs)) :
new_cat.append(cats[j])
 new_sub.append(subs[k])
 new_approve.append(row['project_is_approved'])
 new_cat = pd.DataFrame({'project_subject_categories':new_cat})
 new_sub = pd.DataFrame({'project_subject_subcategories':new_sub})
 new_approve = pd.DataFrame({'project_is_approved':new_approve})
 cat_total_df = pd.concat([new_cat,new_sub,new_approve],axis=1 ).reset_index()
cat_approval_df = cat_total_df.groupby(['project_subject_categories','project_subject_subcategories'])['project_is_approved'].agg('sum')
 cat_all_df = cat_total_df.groupby(['project_subject_categories','project_subject_subcategories'] ).count() ['project_is_approved']
cat_heat =(cat_approval_df/cat_all_df ).unstack()
plt.figure(figsize=(18,5))
ax = sns.heatmap(cat_heat, cmap='viridis_r')
 plt.title("Aprroved rate for categories and sub-categories combination" )
with strategy.scope() :
siamese_network, feature_extractor = build_siamese_nn(
 transformer_name=model_name,
hidden_state_size=sentence_embedding_size,
max_len=max_seq_len,
lr=siamese_learning_rate
)<train_model>
df["essay"] = df["project_essay_1"].map(str)+ df["project_essay_2"].map(str)+ df["project_essay_3"].map(str)+ df["project_essay_4"].map(str)
 def get_polarity(text):
textblob = TextBlob(text)
 pol = textblob.sentiment.polarity
return round(pol,2)
 def get_subjectivity(text):
textblob = TextBlob(text)
 subj = textblob.sentiment.subjectivity
return round(subj,2)
 pol_df = df[['essay','project_is_approved']]
pol_df['sent_polarity'] = pol_df['essay'].apply(get_polarity)
 pol_df['sent_subjectivity'] = pol_df['essay'].apply(get_subjectivity)
 pol_rj_df = pol_df[pol_df['project_is_approved']==0]
pol_total_df = pol_df.groupby(['sent_polarity','sent_subjectivity'] ).count() ['project_is_approved'].unstack().clip(upper=5000)
 pol_rj_df = pol_rj_df.groupby(['sent_polarity','sent_subjectivity'] ).count() ['project_is_approved'].unstack().clip(upper=5000)
 pol_rat_df = pol_rj_df/pol_total_df
plt.figure(figsize=(20,20))
ax = sns.heatmap(pol_rat_df, cmap='viridis_r')
 plt.title("Correlation between Polarity and Subjectivity for Rate of")
train_siamese_nn(nn=siamese_network, trainset=dataset_for_training,
steps_per_trainset=n_batches_per_data,
steps_per_epoch=min(5 * n_batches_per_epoch, n_batches_per_data),
validset=dataset_for_validation,
max_duration=int(round(3600.0 - preparing_duration)) )<drop_column>
del dataset_for_training
del dataset_for_validation
gc.collect()<load_pretrained>
siamese_network.save_weights(tmp_roberta_name )<set_options>
LGBM = pd.read_csv('.. /input/submit/LGBMTFIDF.csv')
 GRU = pd.read_csv('.. /input/submit/GRU.csv')
 combine = pd.read_csv('.. /input/submit/submission_combine7.csv')
 res_df = pd.concat([LGBM['project_is_approved'],GRU['project_is_approved'],combine['project_is_approved']],axis=1)
 res_df.columns=['LGBMTDIDF','GRU','Combined']
data = []
for col in res_df.columns:
data.append(go.Box(y=res_df[col], name=col, showlegend=False))
data.append(go.Scatter(x = res_df.columns, y = res_df.mean() , mode='lines', name='mean'))
url = py.iplot(data, filename='pandas-box-plot' )
del siamese_network
gc.collect()<feature_engineering>
output = pd.read_csv('.. /input/submit/submission_combine7.csv')
 output.to_csv('submission.csv', index=False )