Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<define_variables>
%matplotlib inline
warnings.filterwarnings("ignore" )
if __name__ == '__main__':
Thread(target = func_rfr(20)).start()
Thread(target = func_rfr(50)).start()
Thread(target = func_rfr(100)).start()<find_best_model_class>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean', 'count'],
'escolari': ['min', 'max', 'mean', 'std']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df
rfr = RandomForestRegressor(n_estimators=100)
 rfr.fit(X_train, y_train)
 pred = rfr.predict(X_valid)
 score_rfr = cross_val_score(rfr, X_train, y_train, cv=5, scoring='neg_mean_squared_log_error' )<compute_test_metric>
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
print('cross_val_score average: ', abs(np.mean(score_rfr)))
 print('MAE: ',mean_absolute_error(y_valid, pred))
print('MSE: ',mean_squared_error(y_valid, pred))
print('RMSE: ',np.sqrt(mean_squared_error(y_valid, pred)) )<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
pred = rfr.predict(test[['vendor_id','passenger_count',
'pickup_longitude', 'pickup_latitude', 'dropoff_longitude',
'dropoff_latitude', 'store_and_fwd_flag', 'travel_distance_km',
'hour_pickup', 'day_pickup']] )<save_to_csv>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
arr_id = test['id']
submission = pd.DataFrame({'id': arr_id, 'trip_duration': np.exp(pred)})
 print(submission)
 submission.to_csv("submit_file.csv", index=False )<load_from_csv>
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
print(df_train.isnull().sum() )<remove_duplicates>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
print('duplicates IDs: {}'.format(len(df_train)- len(df_train.drop_duplicates(subset='id'))))<compute_test_metric>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e + '_MEAN' for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
 train, test = train_test_apply_func(train, test, convert_geo2aggs )
def haversine(lat1, lon1, lat2, lon2):
R = 6371800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))<feature_engineering>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )
<feature_engineering>
cols_2_drop = ['abastagua_LE', 'agg18_estadocivil1_MEAN', 'agg18_instlevel6_MEAN', 'agg18_parentesco10_MEAN', 'agg18_parentesco11_MEAN', 'agg18_parentesco12_MEAN', 'agg18_parentesco4_MEAN', 'agg18_parentesco5_MEAN', 'agg18_parentesco6_MEAN', 'agg18_parentesco7_MEAN', 'agg18_parentesco8_MEAN', 'agg18_parentesco9_MEAN', 'fe_people_not_living', 'fe_people_weird_stat', 'geo_elimbasu_LE_3_MEAN', 'geo_elimbasu_LE_4_MEAN', 'geo_energcocinar_LE_0_MEAN', 'geo_energcocinar_LE_1_MEAN', 'geo_energcocinar_LE_2_MEAN', 'geo_epared_LE_0_MEAN', 'geo_epared_LE_2_MEAN', 'geo_etecho_LE_2_MEAN', 'geo_eviv_LE_0_MEAN', 'geo_hogar_mayor_MEAN', 'geo_hogar_nin_MEAN', 'geo_manual_elec_LE_1_MEAN', 'geo_manual_elec_LE_2_MEAN', 'geo_manual_elec_LE_3_MEAN', 'geo_pared_LE_0_MEAN', 'geo_pared_LE_1_MEAN', 'geo_pared_LE_3_MEAN', 'geo_pared_LE_4_MEAN', 'geo_pared_LE_5_MEAN', 'geo_pared_LE_6_MEAN', 'geo_pared_LE_7_MEAN', 'hacapo', 'hacdor', 'mobilephone', 'parentesco1', 'parentesco_LE', 'rez_esc', 'techo_LE', 'v14a', 'v18q']
X.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True)
 test.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True )
df_train['distance'] = df_train.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'], row['dropoff_latitude'], row['dropoff_longitude']), axis=1)
 df_test['distance'] = df_test.apply(lambda row: haversine(row['pickup_latitude'], row['pickup_longitude'], row['dropoff_latitude'], row['dropoff_longitude']), axis=1 )<feature_engineering>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=314, stratify=y )
df_train['trip_duration'] = np.log(df_train['trip_duration'].values )<data_type_conversions>
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
'eval_names': ['train', 'valid'],
'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_power_0997)],
'verbose': False,
'categorical_feature': 'auto'}
fit_params['verbose'] = 200
df_train['pickup_datetime'] = pd.to_datetime(df_train['pickup_datetime'], format='%Y-%m-%d %H:%M:%S')
 df_test['pickup_datetime'] = pd.to_datetime(df_test['pickup_datetime'], format='%Y-%m-%d %H:%M:%S' )<feature_engineering>
df_train['hour'] = df_train.loc[:,'pickup_datetime'].dt.hour;
df_train['week'] = df_train.loc[:,'pickup_datetime'].dt.week;
df_train['weekday'] = df_train.loc[:,'pickup_datetime'].dt.weekday;
df_train['hour'] = df_train.loc[:,'pickup_datetime'].dt.hour;
df_train['month'] = df_train.loc[:,'pickup_datetime'].dt.month;
df_test['hour'] = df_test.loc[:,'pickup_datetime'].dt.hour;
df_test['week'] = df_test.loc[:,'pickup_datetime'].dt.week;
df_test['weekday'] = df_test.loc[:,'pickup_datetime'].dt.weekday;
df_test['hour'] = df_test.loc[:,'pickup_datetime'].dt.hour;
df_test['month'] = df_test.loc[:,'pickup_datetime'].dt.month;<data_type_conversions>
opt_parameters = {
'bagging_fraction': 1.0,
'colsample_bytree': 0.75,
'feature_fraction': 0.1,
'lambda_l1': 5.0,
'lambda_l2': 3.0,
'max_depth': 5,
'min_child_samples': 90,
'min_child_weight': 5.0,
'min_split_gain': 0.001,
'num_leaves': 19,
'subsample': 0.7,
'min_sum_hessian_in_leaf': 1,
'importance_type': 'gain'
}
cat_vars = ['store_and_fwd_flag']
for col in cat_vars:
df_train[col] = df_train[col].astype('category' ).cat.codes
df_train.head()
for col in cat_vars:
df_test[col] = df_test[col].astype('category' ).cat.codes
df_test.head()<prepare_x_and_y>
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):

 if sample_weight is not None:
estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X, y, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self
y_train = df_train["trip_duration"]
X_train = df_train[["vendor_id", "store_and_fwd_flag","passenger_count", "pickup_longitude", "pickup_latitude", "distance", "dropoff_longitude","dropoff_latitude", "hour", "week", "weekday", "month" ]]<train_model>
class VotingPrefitClassifier(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
self.estimators_ = [x[1] for x in self.estimators]
self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
return self
%%time
m = RandomForestRegressor(n_estimators=100,min_samples_leaf=3, min_samples_split=15, n_jobs=-1, max_features="auto")
 m.fit(X_train, y_train )<predict_on_test>
def train_lgbm_model(X_, y_, random_state_=None, opt_parameters_={}, fit_params_={}):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=random_state_, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters_)
 return clf.fit(X_, y_, **fit_params_)
 clfs = []
inner_seed = 31416
inner_n = 10
outer_seed = 314
outer_n = 10
perf_eval = {'f1_oof': [],
'f1_ave': [],
'f1_std': []}
outer_cv = StratifiedKFold(outer_n, shuffle=True, random_state=outer_seed)
 for n_outer_fold,(outer_trn_idx, outer_val_idx)in enumerate(outer_cv.split(X,y)) :
print('--- Outer loop iteration: {} ---'.format(n_outer_fold))
X_out, y_out = X.iloc[outer_trn_idx], y.iloc[outer_trn_idx]
X_stp, y_stp = X.iloc[outer_val_idx], y.iloc[outer_val_idx]
inner_cv = StratifiedKFold(inner_n, shuffle=True, random_state=inner_seed+n_outer_fold)
 y_oof = pd.Series(np.zeros(shape=(X_out.shape[0],)) ,
index=X_out.index)
 f1_scores_inner = []
for n_inner_fold,(inner_trn_idx, inner_val_idx)in enumerate(inner_cv.split(X_out,y_out)) :
X_trn, y_trn = X_out.iloc[inner_trn_idx], y_out.iloc[inner_trn_idx]
X_val, y_val = X_out.iloc[inner_val_idx], y_out.iloc[inner_val_idx]
fit_params["eval_set"] = [(X_trn,y_trn),(X_stp,y_stp)]
clf = train_lgbm_model(X_trn, y_trn, 314+n_inner_fold, opt_parameters, fit_params)
 y_oof.iloc[inner_val_idx] = clf.predict(X_val)
 f1_scores_inner.append(f1_score(y_val, y_oof.iloc[inner_val_idx], average='macro'))
del clf, X_trn, y_trn, X_val, y_val
perf_eval['f1_oof'].append(f1_score(y_out, y_oof, average='macro'))
perf_eval['f1_ave'].append(np.array(f1_scores_inner ).mean())
 perf_eval['f1_std'].append(np.array(f1_scores_inner ).std())
 fit_params["eval_set"] = [(X_out,y_out),(X_stp,y_stp)]
print('Fit the final model on the outer loop iteration: ')
 clf = train_lgbm_model(X_out, y_out, 314+n_outer_fold, opt_parameters, fit_params)
 clfs.append(( 'lgbm{}'.format(n_outer_fold), clf))
del inner_cv, X_out, y_out, X_stp, y_stp
X_test = df_test[["vendor_id", "store_and_fwd_flag","passenger_count","pickup_longitude", "pickup_latitude", "distance","dropoff_longitude","dropoff_latitude", "hour", "week", "weekday", "month"]]
prediction = m.predict(X_test)
 prediction<load_from_csv>
vc = VotingPrefitClassifier(clfs)
 vc = vc.fit(X,y)
 clf_final = vc.estimators_[0]
submit = pd.read_csv('.. /input/sample_submission.csv')
 submit.head()<prepare_output>
global_score = np.mean(perf_eval['f1_oof'])
 global_score_std = np.std(perf_eval['f1_oof'])
 print('Mean validation score LGBM Classifier: {:.4f}'.format(global_score))
print('Std validation score LGBM Classifier: {:.4f}'.format(global_score_std))
my_submission = pd.DataFrame({'id': df_test.id, 'trip_duration': np.exp(prediction)})
 my_submission.head()<save_to_csv>
from sklearn.metrics import precision_score, recall_score, classification_report
my_submission.to_csv('submission.csv', index=False )<set_options>
y_subm = pd.read_csv('.. /input/sample_submission.csv' )
warnings.filterwarnings('ignore')
 pd.set_option('display.max_colwidth', -1)
 plt.style.use('fivethirtyeight')
<load_from_csv>
y_subm['Target'] = clf_final.predict(test)+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test)+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test)+ 1
train=pd.read_csv(".. /input/train.csv")
 test=pd.read_csv(".. /input/test.csv")
 test.shape<data_type_conversions>
train['pickup_datetime']=pd.to_datetime(train['pickup_datetime'],format='%Y-%m-%d %H:%M:%S')
 train['dropoff_datetime']=pd.to_datetime(train['dropoff_datetime'],format='%Y-%m-%d %H:%M:%S')
 train.head()<feature_engineering>
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False)