Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<train_model>
%matplotlib inline
warnings.filterwarnings("ignore" )
%%time
modelparams = { 'booster':'gbtree', 'verbosity':1, 'max_depth':15, 'subsample': 1, 'lamda':0, 'max_delta_step':3, 'objective':'reg:linear', 'learning_rate':0.08, 'colsample_bytree':0.9, 'colsample_bylevel':0.9}
data_train = xgboost.DMatrix(X_train,y_train)
 xg = xgboost.train(modelparams, data_train, num_boost_round=200 )<train_model>
def do_features(df):
feats_div = [('children_fraction', 'r4t1', 'r4t3'), 
('working_man_fraction', 'r4h2', 'r4t3'), 
('all_man_fraction', 'r4h3', 'r4t3'), 
('human_density', 'tamviv', 'rooms'), 
('human_bed_density', 'tamviv', 'bedrooms'), 
('rent_per_person', 'v2a1', 'r4t3'), 
('rent_per_room', 'v2a1', 'rooms'), 
('mobile_density', 'qmobilephone', 'r4t3'), 
('tablet_density', 'v18q1', 'r4t3'), 
('mobile_adult_density', 'qmobilephone', 'r4t2'), 
('tablet_adult_density', 'v18q1', 'r4t2'),
]
feats_sub = [('people_not_living', 'tamhog', 'tamviv'), 
('people_weird_stat', 'tamhog', 'r4t3')]
for f_new, f1, f2 in feats_div:
df['fe_' + f_new] =(df[f1] / df[f2] ).astype(np.float32)
 for f_new, f1, f2 in feats_sub:
df['fe_' + f_new] =(df[f1] - df[f2] ).astype(np.float32)
 aggs_num = {'age': ['min', 'max', 'mean'],
'escolari': ['min', 'max', 'mean']
}
aggs_cat = {'dis': ['mean']}
for s_ in ['estadocivil', 'parentesco', 'instlevel']:
for f_ in [f_ for f_ in df.columns if f_.startswith(s_)]:
aggs_cat[f_] = ['mean', 'count']
for name_, df_ in [('18', df.query('age >= 18')) ]:
df_agg = df_.groupby('idhogar' ).agg({**aggs_num, **aggs_cat} ).astype(np.float32)
 df_agg.columns = pd.Index(['agg' + name_ + '_' + e[0] + "_" + e[1].upper() for e in df_agg.columns.tolist() ])
 df = df.join(df_agg, how='left', on='idhogar')
 del df_agg
df.drop([f_ for f_ in df.columns if f_.startswith('SQB')or f_ == 'agesq'], axis=1, inplace=True)
 df.drop(['Id'], axis=1, inplace=True)
 df.drop(['hhsize', 'female', 'area2'], axis=1, inplace=True)
 return df
<feature_engineering>
def convert_OHE2LE(df):
tmp_df = df.copy(deep=True)
 for s_ in ['pared', 'piso', 'techo', 'abastagua', 'sanitario', 'energcocinar', 'elimbasu',
'epared', 'etecho', 'eviv', 'estadocivil', 'parentesco',
'instlevel', 'lugar', 'tipovivi',
'manual_elec']:
if 'manual_' not in s_:
cols_s_ = [f_ for f_ in df.columns if f_.startswith(s_)]
elif 'elec' in s_:
cols_s_ = ['public', 'planpri', 'noelec', 'coopele']
sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print('The OHE in {} is incomplete.A new column will be added before label encoding' 
.format(s_))
col_dummy = s_+'_dummy'
tmp_df[col_dummy] =(tmp_df[cols_s_].sum(axis=1)== 0 ).astype(np.int8)
 cols_s_.append(col_dummy)
 sum_ohe = tmp_df[cols_s_].sum(axis=1 ).unique()
if 0 in sum_ohe:
print("The category completion did not work")
 tmp_cat = tmp_df[cols_s_].idxmax(axis=1)
 tmp_df[s_ + '_LE'] = LabelEncoder().fit_transform(tmp_cat ).astype(np.int16)
 if 'parentesco1' in cols_s_:
cols_s_.remove('parentesco1')
 tmp_df.drop(cols_s_, axis=1, inplace=True)
 return tmp_df
X_test = featuresSelection(df_test)
 X_test.head()<predict_on_test>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
y_test_predict = np.expm1(xg.predict(xgboost.DMatrix(X_test)))
 y_test_predict[:5]<prepare_output>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
submission = pd.DataFrame(df_test.loc[:, 'id'])
 submission['trip_duration']=y_test_predict
print(submission.shape)
 submission.head()<save_to_csv>
def process_df(df_):
encode_data(df_)
 for f_ in ['v2a1', 'v18q1', 'meaneduc', 'SQBmeaned']:
df_[f_] = df_[f_].fillna(0)
 df_['rez_esc'] = df_['rez_esc'].fillna(-1)
 return do_features(df_)
 train = process_df(train)
 test = process_df(test )
submission.to_csv("submit_file.csv", index=False )<set_options>
def train_test_apply_func(train_, test_, func_):
test_['Target'] = 0
xx = pd.concat([train_, test_])
 xx_func = func_(xx)
 train_ = xx_func.iloc[:train_.shape[0], :]
test_ = xx_func.iloc[train_.shape[0]:, :].drop('Target', axis=1)
 del xx, xx_func
return train_, test_
%matplotlib inline<load_from_csv>
train, test = train_test_apply_func(train, test, convert_OHE2LE )
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )<count_missing_values>
cols_2_ohe = ['eviv_LE', 'etecho_LE', 'epared_LE', 'elimbasu_LE',
'energcocinar_LE', 'sanitario_LE', 'manual_elec_LE',
'pared_LE']
cols_nums = ['age', 'meaneduc', 'dependency',
'hogar_nin', 'hogar_adul', 'hogar_mayor', 'hogar_total',
'bedrooms', 'overcrowding']
def convert_geo2aggs(df_):
tmp_df = pd.concat([df_[(['lugar_LE', 'idhogar']+cols_nums)],
pd.get_dummies(df_[cols_2_ohe],
columns=cols_2_ohe)],axis=1)
 geo_agg = tmp_df.groupby(['lugar_LE','idhogar'] ).mean().groupby('lugar_LE' ).mean().astype(np.float32)
 geo_agg.columns = pd.Index(['geo_' + e for e in geo_agg.columns.tolist() ])
 del tmp_df
return df_.join(geo_agg, how='left', on='lugar_LE')
train.isna().sum()<count_duplicates>
X = train.query('parentesco1==1')
 y = X['Target'] - 1
X = X.drop(['Target'], axis=1 )
train.duplicated().sum()<filter>
cols_2_drop = ['agg18_estadocivil1_MEAN', 'agg18_estadocivil3_COUNT', 'agg18_estadocivil4_COUNT', 'agg18_estadocivil5_COUNT', 'agg18_estadocivil6_COUNT', 'agg18_estadocivil7_COUNT', 'agg18_instlevel1_COUNT', 'agg18_instlevel2_COUNT', 'agg18_instlevel3_COUNT', 'agg18_instlevel4_COUNT', 'agg18_instlevel5_COUNT', 'agg18_instlevel6_COUNT', 'agg18_instlevel7_COUNT', 'agg18_instlevel8_COUNT', 'agg18_instlevel9_COUNT', 'agg18_parentesco10_COUNT', 'agg18_parentesco10_MEAN', 'agg18_parentesco11_COUNT', 'agg18_parentesco11_MEAN', 'agg18_parentesco12_COUNT', 'agg18_parentesco12_MEAN', 'agg18_parentesco1_COUNT', 'agg18_parentesco2_COUNT', 'agg18_parentesco3_COUNT', 'agg18_parentesco4_COUNT', 'agg18_parentesco4_MEAN', 'agg18_parentesco5_COUNT', 'agg18_parentesco6_COUNT', 'agg18_parentesco6_MEAN', 'agg18_parentesco7_COUNT', 'agg18_parentesco7_MEAN', 'agg18_parentesco8_COUNT', 'agg18_parentesco8_MEAN', 'agg18_parentesco9_COUNT', 'fe_people_weird_stat', 'hacapo', 'hacdor', 'mobilephone', 'parentesco1', 'parentesco_LE', 'rez_esc', 'v14a', 'v18q']
X.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True)
 test.drop(( cols_2_drop+['idhogar']), axis=1, inplace=True )
train = train[train['passenger_count']>0]
train = train[train['passenger_count']<6]<filter>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=42, stratify=y )
train = train.loc[train['pickup_longitude']> -85]
train = train.loc[train['pickup_latitude']< 46]<filter>
train = train.loc[train['dropoff_longitude']> -80]
train = train.loc[train['dropoff_latitude']> 36]<feature_engineering>
def evaluate_macroF1_lgb(truth, predictions):
pred_labels = predictions.reshape(len(np.unique(truth)) ,-1 ).argmax(axis=0)
 f1 = f1_score(truth, pred_labels, average='macro')
 return('macroF1', f1, True)
 fit_params={"early_stopping_rounds":300,
"eval_metric" : evaluate_macroF1_lgb,
"eval_set" : [(X_train,y_train),(X_test,y_test)],
'eval_names': ['train', 'valid'],
'verbose': False,
'categorical_feature': 'auto'}
def learning_rate_power_0997(current_iter):
base_learning_rate = 0.1
min_learning_rate = 0.02
lr = base_learning_rate * np.power (.995, current_iter)
 return max(lr, min_learning_rate)
 fit_params['verbose'] = 200
fit_params['callbacks'] = [lgb.reset_parameter(learning_rate=learning_rate_power_0997)]
train = train[(train['trip_duration'] > 60)&(train['trip_duration'] < 3600)]
train[train['trip_duration']<=120].shape
train[train['trip_duration']>=3600].shape
train['trip_duration'] = np.log(train['trip_duration'].values)
 train['hour'] = train['pickup_datetime'].apply(lambda x: int(x.split() [1][0:2]))
test['hour'] = test['pickup_datetime'].apply(lambda x: int(x.split() [1][0:2]))<compute_test_metric>
opt_parameters = {
'bagging_fraction': 0.85,
'colsample_bytree': 1.0,
'feature_fraction': 0.1,
'lambda_l1': 5.0,
'lambda_l2': 3.0,
'max_depth': 9,
'min_child_samples': 55,
'min_child_weight': 5.0,
'min_split_gain': 0.1,
'num_leaves': 45,
'subsample': 0.84,
'min_sum_hessian_in_leaf': 1,
'importance_type': 'gain'
}
def haversine(lat1, lon1, lat2, lon2):
R = 6372800
phi1, phi2 = math.radians(lat1), math.radians(lat2)
 dphi = math.radians(lat2 - lat1)
 dlambda = math.radians(lon2 - lon1)
 a = math.sin(dphi/2)**2 + \
math.cos(phi1)*math.cos(phi2)*math.sin(dlambda/2)**2
return 2*R*math.atan2(math.sqrt(a), math.sqrt(1 - a))
train['dist_long'] = train['pickup_longitude'] - train['dropoff_longitude']
test['dist_long'] = test['pickup_longitude'] - test['dropoff_longitude']
train['dist_lat'] = train['pickup_latitude'] - train['dropoff_latitude']
test['dist_lat'] = test['pickup_latitude'] - test['dropoff_latitude']
train['dist'] = np.sqrt(np.square(train['dist_long'])+ np.square(train['dist_lat']))
test['dist'] = np.sqrt(np.square(test['dist_long'])+ np.square(test['dist_lat']))<data_type_conversions>
def _parallel_fit_estimator(estimator, X, y, sample_weight=None, **fit_params):

 if sample_weight is not None:
estimator.fit(X, y, sample_weight=sample_weight, **fit_params)
 else:
estimator.fit(X, y, **fit_params)
 return estimator
class VotingClassifierLGBM(VotingClassifier):

 def fit(self, X, y, sample_weight=None, **fit_params):
if isinstance(y, np.ndarray)and len(y.shape)> 1 and y.shape[1] > 1:
raise NotImplementedError('Multilabel and multi-output'
' classification is not supported.')
 if self.voting not in('soft', 'hard'):
raise ValueError("Voting must be 'soft' or 'hard'; got(voting=%r)"
% self.voting)
 if self.estimators is None or len(self.estimators)== 0:
raise AttributeError('Invalid `estimators` attribute, `estimators`'
' should be a list of(string, estimator)'
' tuples')
 if(self.weights is not None and
len(self.weights)!= len(self.estimators)) :
raise ValueError('Number of classifiers and weights must be equal'
'; got %d weights, %d estimators'
%(len(self.weights), len(self.estimators)))
 if sample_weight is not None:
for name, step in self.estimators:
if not has_fit_parameter(step, 'sample_weight'):
raise ValueError('Underlying estimator '%s' does not'
' support sample weights.' % name)
 names, clfs = zip(*self.estimators)
 self._validate_names(names)
 n_isnone = np.sum([clf is None for _, clf in self.estimators])
 if n_isnone == len(self.estimators):
raise ValueError('All estimators are None.At least one is '
'required to be a classifier!')
 self.le_ = LabelEncoder().fit(y)
 self.classes_ = self.le_.classes_
self.estimators_ = []
transformed_y = self.le_.transform(y)
 self.estimators_ = Parallel(n_jobs=self.n_jobs )(
 delayed(_parallel_fit_estimator )(clone(clf), X, transformed_y,
sample_weight=sample_weight, **fit_params)
 for clf in clfs if clf is not None)
 return self
train['pickup_datetime'] = pd.to_datetime(train['pickup_datetime'])
 train['dropoff_datetime'] = pd.to_datetime(train['dropoff_datetime'])
 test['pickup_datetime'] = pd.to_datetime(test['pickup_datetime'] )<feature_engineering>
clfs = []
for i in range(20):
clf = lgb.LGBMClassifier(max_depth=-1, learning_rate=0.1, objective='multiclass',
random_state=314+i, silent=True, metric='None',
n_jobs=4, n_estimators=5000, class_weight='balanced')
 clf.set_params(**opt_parameters)
 clfs.append(( 'lgbm{}'.format(i), clf))
vc = VotingClassifierLGBM(clfs, voting='soft')
 del clfs
_ = vc.fit(X_train, y_train, **fit_params)
 clf_final = vc.estimators_[0]
train['minute'] = train.pickup_datetime.dt.minute
train['hour'] = train.pickup_datetime.dt.hour
train['day'] = train.pickup_datetime.dt.dayofweek
train['month'] = train.pickup_datetime.dt.month
test['minute'] = train.pickup_datetime.dt.minute
test['hour'] = test.pickup_datetime.dt.hour
test['day'] = test.pickup_datetime.dt.dayofweek
test['month'] = test.pickup_datetime.dt.month<prepare_x_and_y>
global_score = f1_score(y_test, clf_final.predict(X_test), average='macro')
 vc.voting = 'soft'
global_score_soft = f1_score(y_test, vc.predict(X_test), average='macro')
 vc.voting = 'hard'
global_score_hard = f1_score(y_test, vc.predict(X_test), average='macro')
 print('Validation score of a single LGBM Classifier: {:.4f}'.format(global_score))
print('Validation score of a VotingClassifier on 3 LGBMs with soft voting strategy: {:.4f}'.format(global_score_soft))
print('Validation score of a VotingClassifier on 3 LGBMs with hard voting strategy: {:.4f}'.format(global_score_hard))
y = train["trip_duration"]
X_train = train[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","month","hour","day"]]
X_test = test[["vendor_id","passenger_count","pickup_longitude", "pickup_latitude", "dropoff_longitude","dropoff_latitude","dist","month","hour","day"]]<import_modules>
y_subm = pd.read_csv('.. /input/sample_submission.csv' )
from sklearn.model_selection import train_test_split, cross_val_score<split>
y_subm['Target'] = clf_final.predict(test)+ 1
vc.voting = 'soft'
y_subm_soft = y_subm.copy(deep=True)
 y_subm_soft['Target'] = vc.predict(test)+ 1
vc.voting = 'hard'
y_subm_hard = y_subm.copy(deep=True)
 y_subm_hard['Target'] = vc.predict(test)+ 1
xtrain, xvalid, ytrain, yvalid = train_test_split(X_train,y, test_size=0.2, random_state=42)
 xtrain.shape, xvalid.shape, xtrain.shape, yvalid.shape<import_modules>
y_subm_0forNonHeads = y_subm.copy(deep=True )
<create_dataframe><EOS> 
now = datetime.now()
sub_file = 'submission_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_soft = 'submission_soft_LGB_{:.4f}_{}.csv'.format(global_score_soft, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_hard = 'submission_hard_LGB_{:.4f}_{}.csv'.format(global_score_hard, str(now.strftime('%Y-%m-%d-%H-%M')))
 sub_file_0forNonHeads = 'submission_0forNonHead_LGB_{:.4f}_{}.csv'.format(global_score, str(now.strftime('%Y-%m-%d-%H-%M')))
 y_subm.to_csv(sub_file, index=False)
 y_subm_soft.to_csv(sub_file_soft, index=False)
 y_subm_hard.to_csv(sub_file_hard, index=False)
 y_subm_0forNonHeads.to_csv(sub_file_0forNonHeads, index=False )