Kaggle competition: DonorsChoose.org Application Screening
<SOS> metric: AUC Kaggle data source: donorschoose.org-application-screening<feature_engineering>
stop = set(stopwords.words('english'))
tf.logging.set_verbosity(tf.logging.ERROR)
 pd.options.display.max_rows = 10
pd.options.display.float_format = '{:.1f}'.format
heads['warning'] = -1 *(heads['sanitario1'] + 
(heads['elec'] == 0)+
heads['pisonotiene'] +
heads['abastaguano'] + 
(heads['cielorazo'] == 0))<feature_engineering>
train = pd.read_csv(".. /input/train.csv", sep=",", parse_dates=['project_submitted_datetime'])
 resources = pd.read_csv(".. /input/resources.csv", sep=",")
 test = pd.read_csv(".. /input/test.csv", sep=",", parse_dates=['project_submitted_datetime'] )
heads['phones-per-capita'] = heads['qmobilephone'] / heads['tamviv']
heads['tablets-per-capita'] = heads['v18q1'] / heads['tamviv']
heads['rooms-per-capita'] = heads['rooms'] / heads['tamviv']
heads['rent-per-capita'] = heads['v2a1'] / heads['tamviv']<drop_column>
resources['cost'] = resources['quantity'] * resources['price']
resources_aggregated = resources.groupby('id' ).agg({'description': ['nunique'], 'quantity': ['sum'], 'cost': ['mean', 'sum']})
 resources_aggregated.columns = ['unique_items', 'total_quantity', 'mean_cost', 'total_cost']
resources_aggregated.reset_index(inplace=True)
 train = pd.merge(train, resources_aggregated, how='left', on='id')
 test = pd.merge(test, resources_aggregated, how='left', on='id' )
ind = ind.drop(columns = 'male')
<merge>
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + train.loc[train.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
train.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_1'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2']
test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_2'] = test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_3'] + ' ' + test.loc[test.project_submitted_datetime.dt.date < datetime.date(2016, 5, 7), 'project_essay_4']
test.drop(['project_essay_3', 'project_essay_4'], axis=1, inplace=True)
 train['project_essay_1'] = train['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_essay_2'] = train['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_1'] = test['project_essay_1'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_essay_2'] = test['project_essay_2'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_resource_summary'] = train['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
train['project_title'] = train['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_resource_summary'] = test['project_resource_summary'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
test['project_title'] = test['project_title'].apply(lambda x: x.replace('\\r', ' ' ).replace('\
', ' ' ).replace(' ', ' '))
ind_agg = ind_agg.drop(columns = to_drop)
 ind_feats = list(ind_agg.columns)
 final = heads.merge(ind_agg, on = 'idhogar', how = 'left')
 print('Final features shape: ', final.shape )<drop_column>
def get_quantile_based_boundaries(feature_values, num_buckets):
boundaries = np.arange(1.0, num_buckets)/ num_buckets
quantiles = feature_values.quantile(boundaries)
 return [quantiles[q] for q in quantiles.keys() ]
final=final.drop(["v2a1","rooms",'v18q1','rez_esc','male',],axis=1 )<merge>
bins = [0., 2., 10.,]
test["experience"] = np.digitize(test['teacher_number_of_previously_posted_projects'], bins=bins)
 quantiles = get_quantile_based_boundaries(test['unique_items'], 6)
 test["unique_items_binned"] = np.digitize(test['unique_items'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_quantity'], 6)
 test["total_quantity_binned"] = np.digitize(test['total_quantity'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['mean_cost'], 6)
 test["mean_cost_binned"] = np.digitize(test['mean_cost'], bins=quantiles)
 quantiles = get_quantile_based_boundaries(test['total_cost'], 6)
 test["total_cost_binned"] = np.digitize(test['total_cost'], bins=quantiles )
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )<merge>
def preprocess_features(df):
selected_features = df[
["project_subject_categories",
"project_subject_subcategories",
"project_title",
"project_resource_summary",
"project_essay_1",
"project_essay_2",
"experience",
"unique_items_binned",
"total_quantity_binned",
"mean_cost_binned",
"total_cost_binned",
"encoded_category",
"encoded_subcategory"]]
return selected_features
def preprocess_targets(df):
output_targets = df["project_is_approved"]
return output_targets
head_gender = ind.loc[ind['parentesco1'] == 1, ['idhogar', 'female']]
final = final.merge(head_gender, on = 'idhogar', how = 'left' ).rename(columns = {'female': 'female-head'} )<compute_test_metric>
def target_encode(trn_series=None,
tst_series=None,
target=None,
min_samples_leaf=1,
smoothing=1,
noise_level=0):
assert len(trn_series)== len(target)
 assert trn_series.name == tst_series.name
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - min_samples_leaf)/ smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(["mean", "count"], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return ft_trn_series, ft_tst_series
scorer=make_scorer(f1_score,greater_is_better=True,average='macro' )<data_type_conversions>
train['encoded_category'], test['encoded_category'] = target_encode(train['project_subject_categories'], test['project_subject_categories'], train['project_is_approved'])
 train['encoded_subcategory'], test['encoded_subcategory'] = target_encode(train['project_subject_subcategories'], test['project_subject_subcategories'], train['project_is_approved'] )
train_labels = np.array(list(final[final['Target'].notnull() ]['Target'].astype(np.uint8)))
 train_set = final[final['Target'].notnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 test_set = final[final['Target'].isnull() ].drop(columns = ['Id', 'idhogar', 'Target'])
 print(test_set.shape)
 submission_base = test[['Id', 'idhogar']].copy() 
<categorify>
train = train.reindex(
 np.random.permutation(train.index))
pipeline=Pipeline([('imputer',Imputer(strategy='median')) , 
('scaler',MinMaxScaler())])
 features=list(train_set.columns)
 train_set=pipeline.fit_transform(train_set)
 test_set=pipeline.transform(test_set )<compute_train_metric>
training_examples = preprocess_features(train.head(122080))
training_targets = preprocess_targets(train.head(122080))
validation_examples = preprocess_features(train.tail(60000))
validation_targets = preprocess_targets(train.tail(60000))
test_examples = preprocess_features(test.copy() )
model=RandomForestClassifier(n_estimators=100,random_state=10,n_jobs=-1)
 cv_score=cross_val_score(model,train_set,train_labels,cv=10,scoring=scorer)
 print(f'10 Fold Cross Validation F1 Score = {round(cv_score.mean() , 4)} with std = {round(cv_score.std() , 4)}' )<train_model>
vectorizer=TfidfVectorizer(stop_words=stop)
 vectorizer.fit(train['project_subject_categories'])
 train_project_subject_categories = vectorizer.transform(training_examples['project_subject_categories'])
 validation_project_subject_categories = vectorizer.transform(validation_examples['project_subject_categories'])
 test_project_subject_categories = vectorizer.transform(test_examples['project_subject_categories'])
 vectorizer.fit(train['project_subject_subcategories'])
 train_project_subject_subcategories = vectorizer.transform(training_examples['project_subject_subcategories'])
 validation_project_subject_subcategories = vectorizer.transform(validation_examples['project_subject_subcategories'])
 test_project_subject_subcategories = vectorizer.transform(test_examples['project_subject_subcategories'] )
model.fit(train_set,train_labels)
 feature_importances=pd.DataFrame({'feature':features,'importance':model.feature_importances_})
 feature_importances.head()<import_modules>
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 2), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_title'])
 train_project_title = vectorizer.transform(training_examples['project_title'])
 validation_project_title = vectorizer.transform(validation_examples['project_title'])
 test_project_title = vectorizer.transform(test_examples['project_title'])
 vectorizer.fit(train['project_resource_summary'])
 train_project_resource_summary = vectorizer.transform(training_examples['project_resource_summary'])
 validation_project_resource_summary = vectorizer.transform(validation_examples['project_resource_summary'])
 test_project_resource_summary = vectorizer.transform(test_examples['project_resource_summary'] )
from sklearn.svm import LinearSVC
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier
from sklearn.linear_model import LogisticRegressionCV, RidgeClassifierCV
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.neighbors import KNeighborsClassifier<compute_train_metric>
vectorizer=TfidfVectorizer(stop_words=stop, ngram_range=(1, 3), max_df=0.9, min_df=5, max_features=2000)
 vectorizer.fit(train['project_essay_1'])
 train_project_essay_1 = vectorizer.transform(training_examples['project_essay_1'])
 validation_project_essay_1 = vectorizer.transform(validation_examples['project_essay_1'])
 test_project_essay_1 = vectorizer.transform(test_examples['project_essay_1'])
 vectorizer.fit(train['project_essay_2'])
 train_project_essay_2 = vectorizer.transform(training_examples['project_essay_2'])
 validation_project_essay_2 = vectorizer.transform(validation_examples['project_essay_2'])
 test_project_essay_2 = vectorizer.transform(test_examples['project_essay_2'] )
model_results = cv_model(train_set, train_labels, LinearSVC() ,
'LSVC', model_results )<compute_test_metric>
def get_polarity(text):
textblob = TextBlob(text)
 pol = textblob.sentiment.polarity
return round(pol,3)
 def get_subjectivity(text):
textblob = TextBlob(text)
 subj = textblob.sentiment.subjectivity
return round(subj,3 )
model_results = cv_model(train_set, train_labels,
GaussianNB() , 'GNB', model_results )<choose_model_class>
training_examples['polarity1'] = training_examples['project_essay_1'].apply(get_polarity)
 training_examples['subjectivity1'] = training_examples['project_essay_1'].apply(get_subjectivity)
 training_examples['polarity2'] = training_examples['project_essay_2'].apply(get_polarity)
 training_examples['subjectivity2'] = training_examples['project_essay_2'].apply(get_subjectivity )
model_results = cv_model(train_set, train_labels,
MLPClassifier(hidden_layer_sizes=(32, 64, 128, 64, 32)) ,
'MLP', model_results )<compute_test_metric>
validation_examples['polarity1'] = validation_examples['project_essay_1'].apply(get_polarity)
 validation_examples['subjectivity1'] = validation_examples['project_essay_1'].apply(get_subjectivity)
 validation_examples['polarity2'] = validation_examples['project_essay_2'].apply(get_polarity)
 validation_examples['subjectivity2'] = validation_examples['project_essay_2'].apply(get_subjectivity )
model_results = cv_model(train_set, train_labels,
LinearDiscriminantAnalysis() ,
'LDA', model_results )<compute_train_metric>
test_examples['polarity1'] = test_examples['project_essay_1'].apply(get_polarity)
 test_examples['subjectivity1'] = test_examples['project_essay_1'].apply(get_subjectivity)
 test_examples['polarity2'] = test_examples['project_essay_2'].apply(get_polarity)
 test_examples['subjectivity2'] = test_examples['project_essay_2'].apply(get_subjectivity )
model_results = cv_model(train_set, train_labels,
RidgeClassifierCV() , 'RIDGE', model_results )<find_best_params>
to_drop = ['project_subject_categories', 'project_subject_subcategories', 'project_title', 'project_essay_1', 'project_essay_2', 'project_resource_summary']
for col in to_drop:
training_examples.drop([col], axis=1, inplace=True)
 validation_examples.drop([col], axis=1, inplace=True)
 test_examples.drop([col], axis=1, inplace=True )
for n in [5, 10, 20]:
print(f'
KNN with {n} neighbors
')
 model_results = cv_model(train_set, train_labels,
KNeighborsClassifier(n_neighbors = n),
f'knn-{n}', model_results )<choose_model_class>
training_features = csr_matrix(hstack([training_examples.values, train_project_subject_categories, train_project_subject_subcategories, train_project_title, train_project_resource_summary, train_project_essay_1, train_project_essay_2]))
validation_features = csr_matrix(hstack([validation_examples.values, validation_project_subject_categories, validation_project_subject_subcategories, validation_project_title, validation_project_resource_summary, validation_project_essay_1, validation_project_essay_2]))
test_features = csr_matrix(hstack([test_examples.values, test_project_subject_categories, test_project_subject_subcategories, test_project_title, test_project_resource_summary, test_project_essay_1, test_project_essay_2]))
model_results = cv_model(train_set, train_labels,
ExtraTreesClassifier(n_estimators = 100, random_state = 10),
'EXT', model_results )<compute_train_metric>
model_results = cv_model(train_set, train_labels,
RandomForestClassifier(100, random_state=10),
'RF', model_results )<filter>
params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 16,
'num_leaves': 31,
'learning_rate': 0.025,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': 0,
'num_threads': 1,
'lambda_l2': 1,
'min_gain_to_split': 0,
}
model2 = lgb.train(
 params,
lgb.Dataset(training_features, training_targets),
num_boost_round=10000,
valid_sets=[lgb.Dataset(validation_features, validation_targets)],
verbose_eval=100,
early_stopping_rounds=100
)
test_ids = list(final.loc[final['Target'].isnull() , 'idhogar'])
<predict_on_test>
submission = pd.DataFrame()
submission['id'] = test['id']
submission['project_is_approved'] = model2.predict(test_features, num_iteration=model2.best_iteration)
 submission.to_csv('submission.csv', index=False )