Kaggle competition: Costa Rican Household Poverty Level Prediction
<SOS> metric: MacroFScore Kaggle data source: costa-rican-household-poverty-level-prediction<drop_column>
input_dir = '.. /input/'
working_dir = '.. /working/'
df.drop(['dataset'], axis=1, inplace=True)
 df.drop(['id', 'spacegroup', 'number_of_total_atoms'], axis=1, inplace=True )<concatenate>
%matplotlib inline
df_new = pd.concat([spectrum_pca_df, df], axis=1)
 df_new.head()<drop_column>
train = pd.read_csv(os.path.join(input_dir, 'train.csv'))
test = pd.read_csv(os.path.join(input_dir, 'test.csv'))
train.index = train['Id'].values
test.index = test['Id'].values
print(train.shape)
 print(test.shape )
df_new.drop([395, 126, 1215, 1886, 2075, 353, 308, 2154, 531, 1379, 2319, 2337, 2370, 2333], axis=0, inplace=True )<prepare_x_and_y>
def data_cleaning(data):
data['dependency']=np.sqrt(data['SQBdependency'])
 data['rez_esc']=data['rez_esc'].fillna(0)
 data['v18q1']=data['v18q1'].fillna(0)
 data['v2a1']=data['v2a1'].fillna(0)
 conditions = [ 
(data['edjefe']=='no')&(data['edjefa']=='no'), 
(data['edjefe']=='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']=='yes'), 
(data['edjefe']!='no')&(data['edjefe']!='yes')&(data['edjefa']=='no'), 
(data['edjefe']=='no')&(data['edjefa']!='no')
 ]
choices = [0, 1, 1, data['edjefe'], data['edjefa']]
data['edjefx']=np.select(conditions, choices)
 data['edjefx']=data['edjefx'].astype(int)
 data.drop(['edjefe', 'edjefa'], axis=1, inplace=True)
 meaneduc_nan=data[data['meaneduc'].isnull() ][['Id','idhogar','escolari']]
me=meaneduc_nan.groupby('idhogar')['escolari'].mean().reset_index()
for row in meaneduc_nan.iterrows() :
idx=row[0]
idhogar=row[1]['idhogar']
m=me[me['idhogar']==idhogar]['escolari'].tolist() [0]
data.at[idx, 'meaneduc']=m
data.at[idx, 'SQBmeaned']=m*m
return data
df_len = len(df_new)
 train_len = df_len - test_len
X_train = df_new.drop(['formation_energy_ev_natom', 'bandgap_energy_ev'], axis=1)[:train_len].values
X_test = df_new.drop(['formation_energy_ev_natom', 'bandgap_energy_ev'], axis=1)[train_len:].values
y_formation = df_new['formation_energy_ev_natom'][:train_len].values
y_bandgap = df_new['bandgap_energy_ev'][:train_len].values<train_on_grid>
train = train.query('parentesco1==1')
 train = train.drop('parentesco1', axis=1)
 test = test.drop('parentesco1', axis=1)
 print(train.shape )
xgb_formation = xgb.XGBRegressor()
parameters = {
'max_depth': [2, 3, 4],
'n_estimators' : [100, 200, 300],
}
cv_formation = GridSearchCV(xgb_formation, param_grid=parameters, cv=4, verbose=1)
 cv_formation.fit(X_train, y_formation )<find_best_params>
def get_numeric(data, status_name):
status_cols = [s for s in data.columns.tolist() if status_name in s]
print('status column names')
 print(status_cols)
 status_df = data[status_cols]
status_df.columns = list(range(status_df.shape[1]))
status_numeric = status_df.idxmax(1)
 status_numeric.name = status_name
data = pd.concat([data, status_numeric], axis=1)
 return data
cv_formation.best_params_<train_on_grid>
status_name_list = ['epared', 'etecho', 'eviv', 'instlevel']
for status_name in status_name_list:
train = get_numeric(train, status_name)
 test = get_numeric(test, status_name )
xgb_bandgap = xgb.XGBRegressor()
parameters = {
'max_depth': [2, 3, 4],
'n_estimators' : [100, 200, 300],
}
cv_bandgap = GridSearchCV(xgb_bandgap, param_grid=parameters, cv=4, verbose=1)
 cv_bandgap.fit(X_train, y_bandgap )<find_best_params>
needless_cols = ['r4t3', 'tamhog', 'tamviv', 'hhsize', 'v18q', 'v14a', 'agesq',
'mobilephone', 'paredother', 'pisoother', 'abastaguano',
'energcocinar1', 'techootro', 'sanitario6', 'elimbasu6',
'estadocivil7', 'parentesco12', 'tipovivi5',
'lugar1', 'area1', 'female', 'epared1', 'epared2',
'epared3', 'etecho1', 'etecho2', 'etecho3',
'eviv1', 'eviv2', 'eviv3', 'instlevel1', 'instlevel2',
'instlevel3', 'instlevel4', 'instlevel5', 'instlevel6',
'instlevel7', 'instlevel8', 'instlevel9']
SQB_cols = [s for s in train.columns.tolist() if 'SQB' in s]
parentesco_cols = [s for s in train.columns.tolist() if 'parentesco' in s]
needless_cols.extend(SQB_cols)
 needless_cols.extend(parentesco_cols)
 train = train.drop(needless_cols, axis=1)
 test = test.drop(needless_cols, axis=1 )
cv_bandgap.best_params_<predict_on_test>
ori_train = pd.read_csv(os.path.join(input_dir, 'train.csv'))
ori_train_X = ori_train.drop(['Id', 'Target', 'idhogar'], axis=1)
 train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)
 print('feature columns
{} -> {}'.format(ori_train_X.shape[1], train_X.shape[1]))
formation_pred = cv_formation.predict(X_test)
 bandgap_pred = cv_bandgap.predict(X_test )<create_dataframe>
train_Id = train['Id']
train_idhogar = train['idhogar']
train_y = train['Target']
train_X = train.drop(['Id', 'Target', 'idhogar'], axis=1)
 test_Id = test['Id']
test_idhogar = test['idhogar']
test_X = test.drop(['Id', 'idhogar'], axis=1)
 all_Id = pd.concat([train_Id, test_Id], axis=0, sort=False)
 all_idhogar = pd.concat([train_idhogar, test_idhogar], axis=0, sort=False)
 all_X = pd.concat([train_X, test_X], axis=0, sort=False )
submission = pd.DataFrame(np.arange(1, test_len + 1), columns=['id'])
 submission['formation_energy_ev_natom'] = formation_pred
submission['bandgap_energy_ev'] = bandgap_pred
submission.shape<save_to_csv>
X_train, X_test, y_train, y_test = train_test_split(train_X, train_y, test_size=0.1, random_state=0)
 F1_scorer = make_scorer(f1_score, greater_is_better=True, average='macro')
 gbm = lgb.LGBMClassifier(boosting_type='dart', objective='multiclassova', class_weight='balanced', random_state=0)
 gbm.fit(X_train, y_train)
submission.to_csv('submission.csv', index=False )<save_to_csv>
with open(os.path.join(working_dir, '20180801_lgbm.pickle'), mode='wb')as f:
pickle.dump(gbm, f )
submission.to_csv('submission.csv', index=False )<import_modules>
y_test_pred = gbm.predict(X_test)
 cm = confusion_matrix(y_test, y_test_pred)
 f1 = f1_score(y_test, y_test_pred, average='macro')
 print("confusion matrix:
", cm)
 print("macro F1 score:
", f1 )
<categorify><EOS> 
pred = gbm.predict(test_X)
 pred = pd.Series(data=pred, index=test_Id.values, name='Target')
 pred = pd.concat([test_Id, pred], axis=1, join_axes=[test_Id.index])
 pred.to_csv('20180801_lgbm.csv', index=False )