Kaggle competition: DonorsChoose.org Application Screening
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'female', ]
instlevel_cols = [s for s in train.columns.tolist() if 'instlevel' in s]
needless_cols.extend(instlevel_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )<split>
lem = WordNetLemmatizer()
eng_stopwords = set(stopwords.words("english"))
start_time=time.time()
color = sns.color_palette()
sns.set_style("dark")
 warnings.filterwarnings("ignore")
 %matplotlib inline
def split_data(train, y, sample_weight=None, households=None, test_percentage=0.20, seed=None):
train2 = train.copy()
cv_hhs = np.random.choice(households, size=int(len(households)* test_percentage), replace=False)
 cv_idx = np.isin(households, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
if sample_weight is not None:
y_train_weights = sample_weight[~cv_idx]
return X_train, y_train, X_test, y_test, y_train_weights
return X_train, y_train, X_test, y_test<split>
train=pd.read_csv(".. /input/donorschoose-application-screening/train.csv")
 resources=pd.read_csv(".. /input/donorschoose-application-screening/resources.csv")
 test=pd.read_csv(".. /input/donorschoose-application-screening/test.csv")
 sample_sub=pd.read_csv(".. /input/donorschoose-application-screening/sample_submission.csv" )
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1)
 np.random.seed(seed=None)
 train2 = X.copy()
train_hhs = train2.idhogar
households = train2.idhogar.unique()
cv_hhs = np.random.choice(households, size=int(len(households)* 0.15), replace=False)
 cv_idx = np.isin(train2.idhogar, cv_hhs)
 X_test = train2[cv_idx]
y_test = y[cv_idx]
X_train = train2[~cv_idx]
y_train = y[~cv_idx]
X_train = train2
y_train = y
train_households = X_train.idhogar<split>
nrow_train=train.shape[0]
nrow_test=test.shape[0]
sum=nrow_train+nrow_test
print("Checking proportion of Test-train split")
 print(" : train : test")
 print("rows :",nrow_train,":",nrow_test)
 print("perc :",round(nrow_train*100/sum)," :",round(nrow_test*100/sum))
y_train_weights = class_weight.compute_sample_weight('balanced', y_train, indices=None )<define_variables>
print("Check for Percent of missing values in Train dataset")
 null_check=train.isnull().sum() 
(null_check/len(train)) *100
extra_drop_features = [
'agg18_estadocivil1_MEAN',
'agg18_estadocivil6_COUNT',
'agg18_estadocivil7_COUNT',
'agg18_parentesco10_COUNT',
'agg18_parentesco11_COUNT',
'agg18_parentesco12_COUNT',
'agg18_parentesco1_COUNT',
'agg18_parentesco2_COUNT',
'agg18_parentesco3_COUNT',
'agg18_parentesco4_COUNT',
'agg18_parentesco5_COUNT',
'agg18_parentesco6_COUNT',
'agg18_parentesco7_COUNT',
'agg18_parentesco8_COUNT',
'agg18_parentesco9_COUNT',
'geo_elimbasu_LE_4',
'geo_energcocinar_LE_1',
'geo_energcocinar_LE_2',
'geo_epared_LE_0',
'geo_hogar_mayor',
'geo_manual_elec_LE_2',
'geo_pared_LE_3',
'geo_pared_LE_4',
'geo_pared_LE_5',
'geo_pared_LE_6',
'num_over_18',
'parentesco_LE',
'rez_esc']<define_variables>
print("Check for Percent of missing values in RESOURCES file")
 null_check=resources.isnull().sum() 
(null_check/len(resources)) *100
xgb_drop_cols = extra_drop_features + ["idhogar", 'parentesco1']<init_hyperparams>
print("There are",resources.description.isnull().sum() ,"NULL entries in description column of Resources dataset")
 null_ids=resources[resources.description.isnull() ].id
print("Those Null entries are from",len(null_ids.unique()),"projects")
 null_entries_train=train[train.id.isin(null_ids)]
print("There are",len(null_entries_train),"of these projects are in train")
 null_entries_test=test[test.id.isin(null_ids)]
print("There are",len(null_entries_test),"of these projects are in test")
 x=null_entries_train.project_is_approved.value_counts()
print("Approval rate of projects with NULL as the description under Resources:",x[1]/(x[0]+x[1])*100)
 x=null_entries_train[null_entries_train.teacher_number_of_previously_posted_projects==0].project_is_approved.value_counts()
print("Approval rate of projects with NULL as the description under Resources and 0 previous project submissions:",x[1]/(x[0]+x[1])*100 )
opt_parameters = {'max_depth':35, 'eta':0.1, 'silent':0, 'objective':'multi:softmax', 'min_child_weight': 1, 'num_class': 4, 'gamma': 2.0, 'colsample_bylevel': 0.9, 'subsample': 0.84, 'colsample_bytree': 0.88, 'reg_lambda': 0.40 }
opt_parameters = {'max_depth':35, 'eta':0.15, 'silent':1, 'objective':'multi:softmax', 'min_child_weight': 2, 'num_class': 4, 'gamma': 2.5, 'colsample_bylevel': 1, 'subsample': 0.95, 'colsample_bytree': 0.85, 'reg_lambda': 0.35 }
def evaluate_macroF1_lgb(predictions, truth):
pred_labels = predictions.argmax(axis=1)
 truth = truth.get_label()
f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', 1-f1)
 fit_params={"early_stopping_rounds":500,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'verbose': False,
}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 50<split>
null_entries_train[null_entries_train.project_is_approved==0].project_resource_summary.iloc[0]
np.random.seed(100)
 def _parallel_fit_estimator(estimator1, X, y, sample_weight=None, threshold=True, **fit_params):
estimator = clone(estimator1)
 if sample_weight is not None:
X_train, y_train, X_test, y_test, y_train_weight = split_data(X, y, sample_weight, households=train_households)
 else:
X_train, y_train, X_test, y_test = split_data(X, y, None, households=train_households)
 fit_params["eval_set"] = [(X_test,y_test)]
if sample_weight is not None:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
_ = estimator.fit(X_train, y_train, sample_weight=y_train_weight, **fit_params)
 else:
if isinstance(estimator1, ExtraTreesClassifier)or isinstance(estimator1, RandomForestClassifier):
estimator.fit(X_train, y_train)
 else:
_ = estimator.fit(X_train, y_train, **fit_params)
 if not isinstance(estimator1, ExtraTreesClassifier)and not isinstance(estimator1, RandomForestClassifier)and not isinstance(estimator1, xgb.XGBClassifier):
best_cv_round = np.argmax(estimator.evals_result_['validation_0']['mlogloss'])
 best_cv = np.max(estimator.evals_result_['validation_0']['mlogloss'])
 best_train = estimator.evals_result_['train']['macroF1'][best_cv_round]
else:
best_train = f1_score(y_train, estimator.predict(X_train), average="macro")
 best_cv = f1_score(y_test, estimator.predict(X_test), average="macro")
 print("Train F1:", best_train)
 print("Test F1:", best_cv)
 if threshold:
if(( best_cv > 0.37)and(best_train > 0.75)) or(( best_cv > 0.44)and(best_train > 0.65)) :
return estimator
else:
print("Unacceptable!!! Trying again...")
 return _parallel_fit_estimator(estimator1, X, y, sample_weight=sample_weight, **fit_params)
 else:
return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, threshold=True, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, threshold=threshold, **fit_params)
 for clf in clfs if clf is not None)
 return self<train_model>
null_entries_train[null_entries_train.project_is_approved==0].project_resource_summary.iloc[1]
clfs = []
for i in range(15):
clf = xgb.XGBClassifier(random_state=217+i, n_estimators=300, learning_rate=0.15, n_jobs=4, **opt_parameters)
 clfs.append(( 'xgb{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del(clfs)
 _ = vc.fit(X_train.drop(xgb_drop_cols, axis=1), y_train, sample_weight=y_train_weights, threshold=False, **fit_params)
 clf_final = vc.estimators_[0]<compute_train_metric>
def get_text_stats(text_col):

 title="Text Stats of " + text_col
target_col='project_is_approved'
text_stats=pd.DataFrame()
text_stats[target_col]=train[target_col]
text_stats['word_count']=train[text_col].apply(lambda x: len(str(x ).split()))
text_stats['count_unique_word']=train[text_col].apply(lambda x: len(set(str(x ).split())))
 text_stats['word_unique_percent']=(text_stats['count_unique_word']*100)/text_stats['word_count']
temp_df = pd.melt(text_stats, value_vars=['word_count', 'count_unique_word'], id_vars=target_col)
 print("------ Sample from an Approved project ------
")
 print(approvals[text_col].iloc[0])
 print("
------ Sample from a Rejected project ------
")
 print(rejects[text_col].iloc[0])
 plt.figure(figsize=(16,5))
plt.subplot(121)
 plt.suptitle(title,fontsize=16)
 plt.title("Word Count")
 sns.violinplot(x='variable', y='value', hue=str(target_col), data=temp_df,inner='quartile')
 plt.ylabel('
plt.subplot(122)
 plt.title("Percentage of Unique words - effect on Approval")
 ax=sns.kdeplot(text_stats[text_stats.project_is_approved == 0].word_unique_percent, label="Not Approved",shade=True,color='r')
 ax=sns.kdeplot(text_stats[text_stats.project_is_approved == 1].word_unique_percent, label="Approved")
 plt.legend()
plt.xlabel('Percent unique words', fontsize=12)
 plt.ylabel('
plt.show()
global_score = f1_score(y_test, clf_final.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test.drop(xgb_drop_cols, axis=1)) , average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))<compute_test_metric>
gender_mapping = {"Ms.": "Female", "Mrs.":"Female", "Mr.":"Male", "Teacher":"Unknown", "Dr.":"Unknown", np.nan:"Unknown" }
train["gender"] = train.teacher_prefix.map(gender_mapping)
 approvals["gender"] = approvals.teacher_prefix.map(gender_mapping)
 rejects["gender"] = rejects.teacher_prefix.map(gender_mapping)
 test['gender'] = test.teacher_prefix.map(gender_mapping)
 make_custom_plot('gender','Analyzing Gender' )
ranked_features = feature_importance(clf_final, X_train.drop(xgb_drop_cols, axis=1))<define_variables>
subject_cats=','.join(train['project_subject_categories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_categories'])
 cats.project_subject_categories=cats.project_subject_categories.str.strip()
x=cats.project_subject_categories.value_counts()
print("There are",len(x),"different subject categories after cleaning")
 subject_cats=','.join(approvals['project_subject_categories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_categories'])
 cats.project_subject_categories=cats.project_subject_categories.str.strip()
x1=cats.project_subject_categories.value_counts()
subject_cats=','.join(rejects['project_subject_categories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_categories'])
 cats.project_subject_categories=cats.project_subject_categories.str.strip()
x2=cats.project_subject_categories.value_counts()
et_drop_cols = ['agg18_age_MAX', 'agg18_age_MEAN', 'agg18_age_MIN', 'agg18_dis_MEAN',
'agg18_escolari_MAX', 'agg18_escolari_MEAN', 'agg18_escolari_MIN',
'agg18_estadocivil1_COUNT', 'agg18_estadocivil1_MEAN',
'agg18_estadocivil2_COUNT', 'agg18_estadocivil2_MEAN',
'agg18_estadocivil3_COUNT', 'agg18_estadocivil3_MEAN',
'agg18_estadocivil4_COUNT', 'agg18_estadocivil4_MEAN',
'agg18_estadocivil5_COUNT', 'agg18_estadocivil5_MEAN',
'agg18_estadocivil6_COUNT', 'agg18_estadocivil6_MEAN',
'agg18_estadocivil7_COUNT', 'agg18_estadocivil7_MEAN',
'agg18_parentesco10_COUNT', 'agg18_parentesco10_MEAN',
'agg18_parentesco11_COUNT', 'agg18_parentesco11_MEAN',
'agg18_parentesco12_COUNT', 'agg18_parentesco12_MEAN',
'agg18_parentesco1_COUNT', 'agg18_parentesco1_MEAN',
'agg18_parentesco2_COUNT', 'agg18_parentesco2_MEAN',
'agg18_parentesco3_COUNT', 'agg18_parentesco3_MEAN',
'agg18_parentesco4_COUNT', 'agg18_parentesco4_MEAN',
'agg18_parentesco5_COUNT', 'agg18_parentesco5_MEAN',
'agg18_parentesco6_COUNT', 'agg18_parentesco6_MEAN',
'agg18_parentesco7_COUNT', 'agg18_parentesco7_MEAN',
'agg18_parentesco8_COUNT', 'agg18_parentesco8_MEAN',
'agg18_parentesco9_COUNT', 'agg18_parentesco9_MEAN']
et_drop_cols.extend(["idhogar", "parentesco1", 'fe_rent_per_person', 'fe_rent_per_room',
'fe_tablet_adult_density', 'fe_tablet_density'] )<train_model>
subject_cats=','.join(train['project_subject_subcategories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_subcategories'])
 cats.project_subject_subcategories=cats.project_subject_subcategories.str.strip()
x=cats.project_subject_subcategories.value_counts()
print("There are ",len(x)," different subject sub-categories after Cleaning")
 print("They are:- ",x.index.values)
 subject_cats=','.join(approvals['project_subject_subcategories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_subcategories'])
 cats.project_subject_subcategories=cats.project_subject_subcategories.str.strip()
x1=cats.project_subject_subcategories.value_counts()
subject_cats=','.join(rejects['project_subject_subcategories'])
 cats=pd.DataFrame(subject_cats.split(','),columns=['project_subject_subcategories'])
 cats.project_subject_subcategories=cats.project_subject_subcategories.str.strip()
x2=cats.project_subject_subcategories.value_counts()
ets = []
for i in range(10):
rf = RandomForestClassifier(max_depth=None, random_state=217+i, n_jobs=4, n_estimators=700, min_impurity_decrease=1e-3, min_samples_leaf=2, verbose=0, class_weight="balanced")
 ets.append(( 'rf{}'.format(i), rf))
vc2 = VotingClassifierLGBM(ets, voting='soft')
 _ = vc2.fit(X_train.drop(et_drop_cols, axis=1), y_train, threshold=False )<compute_train_metric>
end_preprocess=time.time()
print("Time till sub-category:",end_preprocess-start_time,"s" )
vc2.voting = 'soft'
global_rf_score_soft = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 vc2.voting = 'hard'
global_rf_score_hard = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_rf_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_rf_score_hard))<compute_train_metric>
state_vals=train.groupby(['school_state'])['project_is_approved','teacher_id'].agg(
 {'project_is_approved':['sum','count'],'teacher_id':['nunique']} ).reset_index()
state_vals.columns=['state','approved','total','teacher_count']
state_vals['approval_perc']=(state_vals.approved*100)/state_vals.total
state_vals['proj_per_teacher']=(state_vals.total)/state_vals.teacher_count
state_vals=state_vals.round(2)
 print("Top States")
 state_vals.sort_values('total',ascending=False ).head()
vc2.voting = 'soft'
global_rf_score_soft = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 vc2.voting = 'hard'
global_rf_score_hard = f1_score(y_test, vc2.predict(X_test.drop(et_drop_cols, axis=1)) , average='macro')
 print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_rf_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_rf_score_hard))<predict_on_test>
py.init_notebook_mode(connected=True)
 state_vals['text'] = 'Approved Projects: '+state_vals['approved'].astype(str)+ '
'+'Total Projects:'+state_vals['total'].astype(str)+'
'+\
'Total Teachers:'+state_vals['teacher_count'].astype(str)+ '
'+'
'+\
'Approval Rate: '+state_vals['approval_perc'].astype(str)
 scl = [[0.0, 'rgb(0,39,143)'],[0.2, 'rgb(0,107,177)'],[0.4, 'rgb(0,154,200)'],\
[0.6, 'rgb(0,189,220)'],[0.8, 'rgb(0,218,235)'],[1.0, 'rgb(0,240,247)']]
data = [ dict(
 type='choropleth',
colorscale = scl,
autocolorscale = False,
locations = state_vals['state'],
z = state_vals['approval_perc'],
locationmode = 'USA-states',
text = state_vals['text'],
marker = dict(
 line = dict(
 color = 'rgb(0,200,255)',
width = 2 
)) ,
colorbar = dict(
 title = "Approval Percentage")
)]
layout = dict(
 title = 'State wide Approval Analysis 
(Hover for breakdown)',
geo = dict(
 scope='usa',
projection=dict(type='albers usa'),
showlakes = True,
lakecolor = 'rgb(255, 255, 255)'), 
)
 fig = dict(data=data, layout=layout)
 py.iplot(fig, validate=False,filename='d3-cloropleth-map' )
def combine_voters(data, weights=[0.5, 0.5]):
vc.voting="soft"
vc1_probs = vc.predict_proba(data.drop(xgb_drop_cols, axis=1))
vc2.voting="soft"
vc2_probs = vc2.predict_proba(data.drop(et_drop_cols, axis=1))
final_vote =(vc1_probs * weights[0])+(vc2_probs * weights[1])
 predictions = np.argmax(final_vote, axis=1)
 return predictions<compute_train_metric>
approvals['project_submitted_datetime'] = pd.to_datetime(approvals['project_submitted_datetime'])
 approvals['datetime_dow'] = approvals['project_submitted_datetime'].dt.dayofweek
approvals['datetime_year'] = approvals['project_submitted_datetime'].dt.year
approvals['datetime_month'] = approvals['project_submitted_datetime'].dt.month
approvals['datetime_hour'] = approvals['project_submitted_datetime'].dt.hour
approvals['datetime_day'] = approvals['project_submitted_datetime'].dt.day
approvals['datetime_date'] = approvals['project_submitted_datetime'].dt.date
rejects['project_submitted_datetime'] = pd.to_datetime(rejects['project_submitted_datetime'])
 rejects['datetime_dow'] = rejects['project_submitted_datetime'].dt.dayofweek
rejects['datetime_year'] = rejects['project_submitted_datetime'].dt.year
rejects['datetime_month'] = rejects['project_submitted_datetime'].dt.month
rejects['datetime_hour'] = rejects['project_submitted_datetime'].dt.hour
rejects['datetime_day'] = rejects['project_submitted_datetime'].dt.day
rejects['datetime_date'] = rejects['project_submitted_datetime'].dt.date
app_date=approvals.groupby('datetime_date')['datetime_date'].count()
rej_date=rejects.groupby('datetime_date')['datetime_date'].count()
combo_preds = combine_voters(X_test, weights=[0.5, 0.5])
 global_combo_score_soft = f1_score(y_test, combo_preds, average='macro')
 global_combo_score_soft<compute_train_metric>
text_col='project_title'
target_col='project_is_approved'
text_stats=pd.DataFrame()
text_stats[target_col]=train[target_col]
text_stats['word_count']=train[text_col].apply(lambda x: len(str(x ).split()))
text_stats['count_unique_word']=train[text_col].apply(lambda x: len(set(str(x ).split())))
combo_preds = combine_voters(X_test, weights=[0.4, 0.6])
 global_combo_score_soft= f1_score(y_test, combo_preds, average='macro')
 global_combo_score_soft<compute_train_metric>
temp=train.groupby('project_title')['project_is_approved'].agg(['sum','count'])
 temp['approval_rate']=(temp['sum']*100)/temp['count']
temp.columns=['
temp=temp.sort_values(by='
temp=temp.iloc[0:20]
temp
combo_preds = combine_voters(X_test, weights=[0.6, 0.4])
 global_combo_score_soft = f1_score(y_test, combo_preds, average='macro')
 global_combo_score_soft<create_dataframe>
approved_mask=np.array(Image.open(".. /input/imagesfordonorchoose/1_approve_heart.png"))
rejected_mask=np.array(Image.open(".. /input/imagesfordonorchoose/2_reject_heart.png"))
approved_mask=~approved_mask[:,:,1]
rejected_mask=~rejected_mask[:,:,1]
make_word_clouds('project_title',approved_mask,rejected_mask,"What does a good title contain?" )
y_subm = pd.DataFrame()
y_subm['Id'] = test_ids<predict_on_test>
train['student_description']=train['project_essay_1']
train.loc[train.project_essay_3.notnull() ,'student_description']=train.loc[train.project_essay_3.notnull() ,'project_essay_1']+train.loc[train.project_essay_3.notnull() ,'project_essay_2']
test['student_description']=test['project_essay_1']
test.loc[test.project_essay_3.notnull() ,'student_description']=test.loc[test.project_essay_3.notnull() ,'project_essay_1']+test.loc[test.project_essay_3.notnull() ,'project_essay_2']
vc.voting = 'soft'
y_subm_lgb = y_subm.copy(deep=True)
 y_subm_lgb['Target'] = vc.predict(test.drop(xgb_drop_cols, axis=1)) + 1
vc2.voting = 'soft'
y_subm_rf = y_subm.copy(deep=True)
 y_subm_rf['Target'] = vc2.predict(test.drop(et_drop_cols, axis=1)) + 1
y_subm_ens = y_subm.copy(deep=True)
 y_subm_ens['Target'] = combine_voters(test)+ 1<save_to_csv>
train['project_description']=train['project_essay_2']
train.loc[train.project_essay_3.notnull() ,'project_description']=train.loc[train.project_essay_3.notnull() ,'project_essay_3']+train.loc[train.project_essay_3.notnull() ,'project_essay_4']
test['project_description']=test['project_essay_2']
test.loc[test.project_essay_3.notnull() ,'project_description']=test.loc[test.project_essay_3.notnull() ,'project_essay_3']+test.loc[test.project_essay_3.notnull() ,'project_essay_4']
test[test.project_essay_3.notnull() ].head(1 ).project_description.values
now = datetime.now()
sub_file_lgb = 'submission_soft_XGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_rf = 'submission_soft_RF_{:.4f}_{}.csv'.format(global_rf_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_ens = 'submission_ens_{:.4f}_{}.csv'.format(global_combo_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm_lgb.to_csv(sub_file_lgb, index=False)
 y_subm_rf.to_csv(sub_file_rf, index=False)
 y_subm_ens.to_csv(sub_file_ens, index=False )<import_modules>
del(train['project_essay_1'])
 del(train['project_essay_2'])
 del(train['project_essay_3'])
 del(train['project_essay_4'])
 del(test['project_essay_1'])
 del(test['project_essay_2'])
 del(test['project_essay_3'])
 del(test['project_essay_4'])
 approvals=train[train.project_is_approved==1]
rejects=train[train.project_is_approved==0]
import pandas as pd
import numpy as np
from scipy import stats<import_modules>
rejected_mask=np.array(Image.open(".. /input/imagesfordonorchoose/1_approve_hand.png"))
approved_mask=np.array(Image.open(".. /input/imagesfordonorchoose/2_reject_hand.png"))
approved_mask=~approved_mask[:,:,1]
rejected_mask=~rejected_mask[:,:,1]
make_word_clouds('student_description',approved_mask,rejected_mask,"How are Students described in Approved VS Rejected projects?" )
import matplotlib.pyplot as plt
import seaborn as sns<set_options>
approved_mask=np.array(Image.open(".. /input/imagesfordonorchoose/1_approve_tick.png"))
rejected_mask=np.array(Image.open(".. /input/imagesfordonorchoose/2_reject_tick.png"))
approved_mask=~approved_mask[:,:,1]
rejected_mask=~rejected_mask[:,:,1]
make_word_clouds('project_description',approved_mask,rejected_mask,"How are Projects described in Approved VS Rejected projects?" )
sns.set_palette(palette='icefire', n_colors=10)
<set_options>
end_time=time.time()
print("Time till plotting section",end_time-start_time,"s" )
density = sns.color_palette(palette= 'RdYlGn', n_colors= 10 )<load_from_csv>
resources['total_cost']=resources['quantity']*resources['price']
resources['description']=resources['description'].astype(str)
 x=resources.groupby('id')['description'].apply(lambda x: "%s" % ', '.join(x))
x.head(2 )
in_train = pd.read_csv('.. /input/train.csv')
 in_test = pd.read_csv('.. /input/test.csv' )<count_unique_values>
resources_agg=resources.groupby('id')['quantity','price','total_cost'].agg({'quantity':['sum','count'],'price':['mean'],'total_cost':['sum']})
 resources_agg.columns=['item_quantity_sum','variety_of_items','avg_price_per_item','total_cost']
resources_agg['collated_description']=x
resources_agg=resources_agg.reset_index()
print('Unique households in training set:', in_train['idhogar'].nunique())
 print('Unique households in training set:', in_test['idhogar'].nunique() )<count_duplicates>
train_merge=pd.merge(left=train,right=resources_agg,on='id',how='left')
 train_merge.sort_values("total_cost",ascending=False ).head()
in_train['Id'].duplicated().sum()
in_test['Id'].duplicated().sum()<concatenate>
all_data=pd.concat([train,test],axis=0)
 all_data_merge=pd.merge(left=all_data,right=resources_agg,on='id',how='left' )
df = pd.concat([in_train, in_test], axis = 0, sort=False, ignore_index= True)
 df.shape<filter>
def preprocess_and_clean(text_col):
word_list = gensim.utils.simple_preprocess(text_col, deacc=True)
 clean_words = [w for w in word_list if not w in eng_stopwords]
clean_words=[lem.lemmatize(word, "v")for word in clean_words]
return(clean_words)
 x=all_data_merge['collated_description'].apply(preprocess_and_clean )
df.isnull().sum() [df.isnull().sum() > 0]<filter>
bigram_transformer = gensim.models.Phrases(x)
 x=x.apply(lambda word : bigram_transformer[word])
 x.head()
df[['tipovivi1', 'tipovivi2', 'tipovivi3', 'tipovivi4', 'tipovivi5']][df['v2a1'].isnull() ].sum()<filter>
model = Word2Vec(x, size=100, window=5, min_count=10, workers=4,seed=10 )
df[(df['tipovivi4'] == 1)&(df['v2a1'] > 0)]<filter>
dictionary = Dictionary(x)
 print("There are",len(dictionary),"number of words in the final dictionary" )
df[(df['tipovivi5'] == 1)&(df['v2a1'] > 0)]<correct_missing_values>
corpus = [dictionary.doc2bow(text)for text in x]
df['v2a1'].fillna(0, inplace = True )<count_missing_values>
ldamodel = LdaModel(corpus=corpus, num_topics=15, id2word=dictionary)
 end_lda=time.time()
print("Time till Lda model creation:",end_lda-start_time,"s" )
df['v18q'][df['v18q1'].isnull() ].value_counts(dropna=False )<correct_missing_values>
pyLDAvis.enable_notebook()
df['v18q1'].fillna(0, inplace = True )<feature_engineering>
pyLDAvis.gensim.prepare(ldamodel, corpus, dictionary )
df['rez_esc'].fillna(0, inplace = True)
 df['Schooling_age'] =(( df['age'] >= 7)&(df['age'] <= 19)) *1<feature_engineering>
topic_probability_mat = ldamodel[corpus]
df.loc[13069,'rez_esc'] = 0<sort_values>
train_matrix=topic_probability_mat[:train.shape[0]]
test_matrix=topic_probability_mat[train.shape[0]:]
df[['age', 'escolari', 'meaneduc', 'idhogar', 'parentesco1']][df['meaneduc'].isnull() ].sort_values(by = 'idhogar' )<filter>
index = similarities.MatrixSimilarity(ldamodel[corpus] )
for household in no_meanedu:
print(household, df['meaneduc'][df['idhogar'] == household].values )<feature_engineering>
sims = index[vec_lda]
mean_educ = {}
for household in no_meanedu:
mean_educ[household] = df['escolari'][(df['idhogar'] == household)&(df['age'] >= 18)].mean()<filter>
sims = sorted(enumerate(sims), key=lambda item: -item[1] )
df[['escolari', 'age']][df['idhogar'] == 'c49af2e64']<feature_engineering>
end_lda=time.time()
print("Time till Lda model viz:",end_lda-start_time,"s" )
for household in no_meanedu:
df.loc[df['idhogar'] == household, 'meaneduc'] = mean_educ[household]<correct_missing_values>
del(ldamodel)
 del(corpus)
 del(dictionary)
 del(model)
 gc.collect()
df['meaneduc'].fillna(0, inplace = True )<feature_engineering>
def preprocess_and_clean(text_col):

 word_list = gensim.utils.simple_preprocess(text_col, deacc=True)
 clean_words = [w for w in word_list if not w in eng_stopwords]
clean_words=[lem.lemmatize(word, "v")for word in clean_words]
return(' '.join(clean_words))
print("Before clean:",train.project_description.iloc[16])
 print("After clean:",preprocess_and_clean(train.project_description.iloc[16]))
df['SQBmeaned'] = df['meaneduc'] **2<define_variables>
train.teacher_prefix=train.teacher_prefix.fillna('Unknown')
 test.teacher_prefix=test.teacher_prefix.fillna('Unknown')
 train.gender=train.gender.fillna('Unknown')
 test.gender=test.gender.fillna('Unknown')
 print(train.shape)
 print(test.shape)
 y=train['project_is_approved']
train_id=train['id']
test_id=test['id']
del(train['project_is_approved'])
 all_data=pd.concat([train,test],axis=0)
 print(all_data.shape)
 all_data_merge=pd.merge(left=all_data,right=resources_agg,on='id',how='left' )
ind_indicators = ['Id', 'escolari', 'rez_esc', 'Schooling_age', 'dis', 'male', 'female',
'estadocivil1', 'estadocivil2', 'estadocivil3', 'estadocivil4',
'estadocivil5', 'estadocivil6', 'estadocivil7', 'parentesco1',
'parentesco2', 'parentesco3', 'parentesco4', 'parentesco5', 'parentesco6',
'parentesco7', 'parentesco8', 'parentesco9', 'parentesco10', 'parentesco11',
'parentesco12', 'instlevel1', 'instlevel2', 'instlevel3', 'instlevel4', 'instlevel5',
'instlevel6', 'instlevel7', 'instlevel8', 'instlevel9', 'age', 'SQBescolari', 'SQBage', 'agesq']<define_variables>
hh_indicators = [col for col in df.columns if col not in ind_indicators]<count_unique_values>
cols = [
'teacher_id',
'teacher_prefix',
'school_state',
'project_grade_category',
'project_subject_categories',
'project_subject_subcategories',
'gender']
for c in tqdm(cols):
le = preprocessing.LabelEncoder()
le.fit(all_data_merge[c].astype(str))
all_data_merge[c] = le.transform(all_data_merge[c].astype(str))
inconsistant_col = {}
for col in ['Target']:
consistant_value = df[df[col].notnull() ].groupby('idhogar')[col].apply(lambda x: x.nunique() == 1)
 if(consistant_value == False ).sum() > 0:
print('{} Inconsistant value for {} within household'.format(( consistant_value == False ).sum() , col))
inconsistant_col[col] = consistant_value[consistant_value == False]<feature_engineering>
all_data_merge['avg_price_per_item']=np.log1p(all_data_merge['avg_price_per_item'])
 all_data_merge['total_cost']=np.log1p(all_data_merge['total_cost'])
 all_data_merge['project_submitted_datetime'] = pd.to_datetime(train['project_submitted_datetime'])
 all_data_merge['datetime_dow'] = all_data_merge['project_submitted_datetime'].dt.dayofweek
all_data_merge['datetime_year'] = all_data_merge['project_submitted_datetime'].dt.year
all_data_merge['datetime_month'] = all_data_merge['project_submitted_datetime'].dt.month
all_data_merge['datetime_hour'] = all_data_merge['project_submitted_datetime'].dt.hour
all_data_merge['datetime_day'] = all_data_merge['project_submitted_datetime'].dt.day
for household in inconsistant_col['Target'].index:
target_val_for_parentesco1 = df['Target'][(df['idhogar'] == household)&(df['parentesco1'] == 1)]
if target_val_for_parentesco1.notnull().any() :
df.loc[df['idhogar'] == household, 'Target'] = target_val_for_parentesco1.values<count_values>
text_cols=['project_title',
'collated_description',
'project_resource_summary',
'student_description',
'project_description']
for c in tqdm(text_cols):
all_data_merge[c+'_len']=all_data_merge[c].apply(len)
 all_data_merge[c+'_word_count']=all_data_merge[c].apply(lambda x: len(str(x ).split()))
all_data_merge[c]=all_data_merge[c].apply(preprocess_and_clean)
 end_time=time.time()
print("Time till end",end_time-start_time,"s" )
df['hacdor'][df['Target'].notnull() ].value_counts()<count_values>
train_shape=train.shape
test_shape=test.shape
del(train_merge)
 del(resources)
 del(resources_agg)
 del(subset)
 del(approvals)
 del(rejects)
 del(all_data)
 gc.collect()
df['Target'][df['hacdor'] == 1].value_counts(normalize = True )<count_values>
df['hacapo'][df['Target'].notnull() ].value_counts()<count_values>
del(all_data_merge['project_submitted_datetime'])
 all_data_merge_1=all_data_merge.to_dict('records')
 del(all_data_merge)
 gc.collect()
df['Target'][(df['Target'].notnull())&(df['v18q'] == 1)].value_counts() /df['Target'].value_counts()<create_dataframe>
def get_col(col_name):
return lambda x: x[col_name]
vectorizer = FeatureUnion([ 
('project_title',CountVectorizer(
 ngram_range=(1, 2),
max_features=10000,
preprocessor=get_col('project_title'))), 
('project_resource_summary',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=30000,
preprocessor=get_col('project_resource_summary'))), 
('student_description',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=100000,
preprocessor=get_col('student_description'))), 
('project_description',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=100000,
preprocessor=get_col('project_description'))), 
('collated_description',CountVectorizer(
 ngram_range=(1, 2),
max_features=30000,
preprocessor=get_col('collated_description'))), 
('Non_text',DictVectorizer())
 ] )
df[df['parentesco1'] == 1].pivot_table(values = 'idhogar', columns = 'v18q1', index = 'Target', aggfunc = 'count' )<feature_engineering>
start_vect=time.time()
all_data_vectorized = vectorizer.fit_transform(all_data_merge_1)
 end_time=time.time()
print("total time in vectorization creation",end_time-start_vect,"s")
 print("total time till vectorization creation",end_time-start_time,"s" )
df['hogar_mid'] = df['hogar_adul'] -df['hogar_mayor']<count_values>
final_dataset=all_data_vectorized.tocsr()
df['Target'][df['computer'] == 1].value_counts() /df['Target'].value_counts()<count_values>
train_x=final_dataset[0:train_shape[0]]
test_x=final_dataset[train_shape[0]:]
df['Target'][df['television'] == 1].value_counts() /df['Target'].value_counts()<count_values>
del final_dataset,all_data_vectorized
gc.collect()
df['Target'][df['mobilephone'] == 1].value_counts() /df['Target'].value_counts()<feature_engineering>
X_train, X_valid, y_train, y_valid = train_test_split(train_x, y, test_size=0.33, random_state=2018)
 params = {
'boosting_type': 'gbdt',
'objective': 'binary',
'metric': 'auc',
'max_depth': 16,
'num_leaves': 31,
'learning_rate': 0.25,
'feature_fraction': 0.85,
'bagging_fraction': 0.85,
'bagging_freq': 5,
'verbose': 1,
'num_threads': 4,
'lambda_l2': 1,
'min_gain_to_split': 0,
'seed':1234
}
df['dependencyR'] =(df['hogar_nin'] + df['hogar_mayor'])/ df['hogar_mid']<count_values>
model = lgb.train(
 params,
lgb.Dataset(X_train, y_train),
num_boost_round=10000,
valid_sets=[lgb.Dataset(X_valid, y_valid)],
early_stopping_rounds=100,
verbose_eval=25 )
df['dependencyR'].value_counts()<data_type_conversions>
valid_preds = model.predict(X_valid, num_iteration=model.best_iteration)
 test_preds = model.predict(test_x, num_iteration=model.best_iteration)
 auc = roc_auc_score(y_valid, valid_preds)
 print('AUC:',auc)
 plt.show()
df['dependencyR'].fillna(0, inplace = True)
 df['dependencyR'].replace(np.inf, 9, inplace = True )<feature_engineering>
end_time=time.time()
print("total time till LGB model",end_time-start_time,"s" )
df['phonespp'] = df['qmobilephone'] / df['tamviv']
df['tabletspp'] = df['v18q1'] / df['tamviv']
df['roomspp'] = df['rooms'] / df['tamviv']
df['rentpp'] = df['v2a1'] / df['tamviv']<categorify>
xgb_params = {'eta': 0.2,
'max_depth': 5,
'subsample': 0.8,
'colsample_bytree': 0.8,
'objective': 'binary:logistic',
'eval_metric': 'auc',
'seed': 1234
}
X_train, X_valid, y_train, y_valid = train_test_split(train_x, y, test_size=0.33, random_state=2018)
 d_train = xgb.DMatrix(X_train, y_train)
 d_valid = xgb.DMatrix(X_valid, y_valid)
 d_test = xgb.DMatrix(test_x )
df['median_schooling'] = df['escolari'].groupby(df['idhogar'] ).transform('median')
 df['max_schooling'] = df['escolari'].groupby(df['idhogar'] ).transform('max' )<feature_engineering>
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model_xgb = xgb.train(xgb_params, d_train, 500, watchlist, verbose_eval=50, early_stopping_rounds=20 )
df['eduForHeadofHH'] = 0
df.loc[(df['parentesco1']== 1), 'eduForHeadofHH'] = df['escolari']<categorify>
xgb_pred_test = model_xgb.predict(d_test)
 xgb_pred_valid = model_xgb.predict(d_valid)
 auc = roc_auc_score(y_valid, xgb_pred_valid)
 print('AUC:',auc )
df['eduForHeadofHH'] = df['eduForHeadofHH'].groupby(df['idhogar'] ).transform('max' )<feature_engineering>
xgb_pred_train = model_xgb.predict(d_train)
df['SecondaryEduLess'] =(( df[['instlevel1','instlevel2', 'instlevel3', 'instlevel4']] == 1 ).any(axis = 1)&(df['age'] > 19)) *1
df['SecondaryEduMore'] =(( df[['instlevel5','instlevel6', 'instlevel7', 'instlevel8', 'instlevel9']] == 1 ).any(axis = 1)&(df['age'] > 19)) *1<count_values>
eli5.show_weights(model_xgb,vec=vectorizer )
df['SecondaryEduMore'].value_counts()<categorify>
final_preds=0.4*xgb_pred_test+0.6*test_preds
df['MembersWithSecEdu'] = df['SecondaryEduMore'].groupby(df['idhogar'] ).transform('sum')
 df['MembersWithPrimEdu'] = df['SecondaryEduLess'].groupby(df['idhogar'] ).transform('sum' )<feature_engineering>
x_preds = pd.DataFrame(final_preds)
 x_preds.columns = ['project_is_approved']
sub_id=sample_sub['id']
submission = pd.concat([sub_id, x_preds], axis=1)
 submission.to_csv('lgbm_xgb_blend.csv', index=False )
<feature_engineering><EOS> 
end_time=time.time()
print("total time spent in FE and model",end_time-start_vect,"s")
 print("total time till end",end_time-start_time,"s" )