Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<train_model>
%matplotlib inline
xgb_model.fit(X_train, y_train )<predict_on_test>
data_df = pd.read_csv('.. /input/train.csv')
 data_df.head()
pred_xgb=np.expm1(xgb_model.predict(test_sc))
pred_xgb<load_from_csv>
test_data_df = pd.read_csv('.. /input/test.csv' )
sub=pd.read_csv('.. /input/sample_submission.csv')
 sub.target=(pred_lgbm+pred_cb+pred_rf+pred_xgb)/4.0
sub.head()<save_to_csv>
train_df = data_df.copy()
test_df = test_data_df.copy()
sub.to_csv('sub1.csv',index=False )<set_options>
data_df['LotFrontage'].isnull().sum()
%matplotlib inline 
<load_from_csv>
data_df[data_df['TotalBsmtSF'] > 6000]
train_df = pd.read_csv('.. /input/train.csv')
 train_df.head()<count_missing_values>
outliers = set(data_df[data_df['TotalBsmtSF'] > 6000].index.values)
 print('Running list of outliers: ', outliers )
train_df.isnull().values.any()<load_from_csv>
data_df[data_df['1stFlrSF'] > 4000]
test_df = pd.read_csv('.. /input/test.csv')
 test_df.head()<count_missing_values>
data_df[data_df['GrLivArea'] > 4000].loc[:, ['GrLivArea', 'SalePrice']]
test_df.isnull().values.any()<count_values>
zscore = stats.zscore(data_df['GrLivArea'])
 thresh = 4
print(np.where(zscore > thresh), zscore[np.where(zscore > thresh)] )
print('Number of constant columns in the train set:', len(constant_train))
print('Number of constant columns in the test set:', len(constant_test))<drop_column>
outliers.update([outlier for outlier in list(np.where(data_df['GrLivArea'] > 4000)[0])if outlier not in outliers])
 print('Running list of outliers: ', outliers )
columns_to_use = test_df.columns.tolist()
del columns_to_use[0]
columns_to_use = [x for x in columns_to_use if x not in constant_train]
len(columns_to_use )<concatenate>
data_df[data_df['PoolArea'] != 0].loc[:, ['PoolArea', 'PoolQC']]
train_df[columns_to_use].values.flatten().shape<prepare_output>
train_df['PoolQC'].fillna('None', inplace=True )
train_zeros = pd.DataFrame({'Percentile':(( train_df[columns_to_use].values)==0 ).mean(axis=0),
'Column' : columns_to_use})
 train_zeros.head()<prepare_output>
test_df['PoolQC'].isnull().sum()
test_zeros = pd.DataFrame({'Percentile':(np.log(1+test_df[columns_to_use].values)==0 ).mean(axis=0),
'Column' : columns_to_use})
 test_zeros.head()<prepare_x_and_y>
test_df[test_df['PoolArea'] != 0].loc[:, ['PoolArea', 'PoolQC']]
y = np.log(1+train_df.target.values)
 y.shape<create_dataframe>
test_df[(test_df['PoolArea'] != 0)&(test_df['PoolQC'].isnull())].loc[:, ['PoolQC']].fillna('Fa', inplace=True)
 test_df['PoolQC'].fillna('None', inplace=True )
train = lgb.Dataset(train_df[columns_to_use],y ,feature_name = "auto" )<train_model>
train_df.drop(columns=['MiscFeature'], inplace=True )
params = {'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.01,
'num_leaves': 100,
'feature_fraction': 0.4,
'bagging_fraction': 0.6,
'max_depth': 5,
'min_child_weight': 10}
clf = lgb.train(params,
train,
num_boost_round = 400,
verbose_eval=True )<predict_on_test>
test_df.drop(columns=['MiscFeature'], inplace=True )
preds = clf.predict(test_df[columns_to_use])
 preds<save_to_csv>
train_df['Alley'].fillna('None', inplace=True)
 test_df['Alley'].fillna('None', inplace=True )
sample_submission = pd.read_csv(".. /input/sample_submission.csv")
 sample_submission.target = np.exp(preds)-1
sample_submission.to_csv('simple_lgbm_1.csv', index=False)
 sample_submission.head()<split>
train_df.drop(columns=['Fence'], inplace=True)
 test_df.drop(columns=['Fence'], inplace=True )
nr_splits = 5
random_state = 1054
y_oof = np.zeros(( y.shape[0]))
total_preds = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof)) )<split>
print('Number of missing values in FireplaceQu = ', data_df['FireplaceQu'].isnull().sum())
 print('Number of missing values in Fireplaces = ', data_df['Fireplaces'].isnull().sum() )
params['max_depth'] = 4
y_oof_2 = np.zeros(( y.shape[0]))
total_preds_2 = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds_2 += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof_2[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_2)) )<split>
len(data_df[(data_df['Fireplaces']!=0)&(data_df['FireplaceQu'].isnull())] )
params['max_depth'] = 6
y_oof_3 = np.zeros(( y.shape[0]))
total_preds_3 = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds_3 += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof_3[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_3)) )<split>
len(test_df[(test_df['Fireplaces']!=0)&(test_df['FireplaceQu'].isnull())] )
params['max_depth'] = 7
y_oof_4 = np.zeros(( y.shape[0]))
total_preds_4 = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds_4 += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof_4[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_4)) )<split>
train_df['FireplaceQu'].fillna('None', inplace=True )
params['max_depth'] = 8
y_oof_5 = np.zeros(( y.shape[0]))
total_preds_5 = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds_5 += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof_5[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_5)) )<split>
test_df['FireplaceQu'].fillna('None', inplace=True )
params['max_depth'] = 10
y_oof_6 = np.zeros(( y.shape[0]))
total_preds_6 = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds_6 += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof_6[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_6)) )<split>
train_df['LotFrontage'].fillna(data_df['LotFrontage'].median() , inplace=True )
params['max_depth'] = 12
y_oof_7 = np.zeros(( y.shape[0]))
total_preds_7 = 0
kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = lgb.Dataset(X_train,y_train ,feature_name = "auto")
 val = lgb.Dataset(X_val ,y_val ,feature_name = "auto")
 clf = lgb.train(params,train,num_boost_round = 400,verbose_eval=True)
 total_preds_7 += clf.predict(test_df[columns_to_use])/nr_splits
pred_oof = clf.predict(X_val)
 y_oof_7[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_7)) )<compute_test_metric>
print('Number of missing values for LotFrontage in test set = ', test_df['LotFrontage'].isnull().sum())
 print('Mean = ', test_df['LotFrontage'].mean())
 print('Median = ', test_df['LotFrontage'].median() )
print('Total error', np.sqrt(mean_squared_error(y, 1.4*(1.6*y_oof_7-0.6*y_oof_6)-0.4*y_oof_5)) )<compute_test_metric>
zscore = np.abs(stats.zscore(data_df['LotFrontage']))
thresh = 4
print(np.where(zscore > thresh), zscore[np.where(zscore > thresh)] )
print('Total error', np.sqrt(mean_squared_error(y, -0.5*y_oof-0.5*y_oof_2-y_oof_3
+3*y_oof_4)) )<compute_test_metric>
data_df[data_df['LotFrontage'] > 300]
print('Total error', np.sqrt(mean_squared_error(y, 0.75*(1.4*(1.6*y_oof_7-0.6*y_oof_6)-0.4*y_oof_5)+
0.25*(-0.5*y_oof-0.5*y_oof_2-y_oof_3
+3*y_oof_4))))<prepare_output>
outliers.update([outlier for outlier in list(np.where(data_df['LotFrontage'] > 300)[0])if outlier not in outliers])
 print('Running list of outliers: ', outliers )
sub_preds =(0.75*(1.4*(1.6*total_preds_7-0.6*total_preds_6)-0.4*total_preds_5)+
0.25*(-0.5*total_preds-0.5*total_preds_2-total_preds_3
+3*total_preds_4))
sample_submission.target = np.exp(sub_preds)-1
sample_submission.to_csv('blended_submission_2.csv', index=False)
 sample_submission.head()<prepare_x_and_y>
len(data_df[(data_df['GarageArea']!=0)&(( data_df['GarageCond'].isnull())| 
(data_df['GarageType'].isnull())| 
(data_df['GarageYrBlt'].isnull())| 
(data_df['GarageFinish'].isnull())| 
(data_df['GarageQual'].isnull())) ] )
params = {'objective': 'reg:linear',
'eval_metric': 'rmse',
'eta': 0.01,
'max_depth': 10,
'subsample': 0.6,
'colsample_bytree': 0.6,
'alpha':0.001,
'random_state': 42,
'silent': True}
y_oof_8 = np.zeros(( y.shape[0]))
total_preds_8 = 0
dtest = xgb.DMatrix(test_df[columns_to_use])
 kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[columns_to_use].iloc[train_index], train_df[columns_to_use].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = xgb.DMatrix(X_train, y_train)
 val = xgb.DMatrix(X_val, y_val)
 watchlist = [(train, 'train'),(val, 'val')]
clf = xgb.train(params, train, 1000, watchlist,
maximize=False, early_stopping_rounds = 60, verbose_eval=100)
 total_preds_8 += clf.predict(dtest, ntree_limit=clf.best_ntree_limit)/nr_splits
pred_oof = clf.predict(val, ntree_limit=clf.best_ntree_limit)
 y_oof_8[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_8)) )<compute_test_metric>
train_df['GarageCond'].fillna('None', inplace=True)
 train_df['GarageType'].fillna('None', inplace=True)
 train_df['GarageYrBlt'].fillna('None', inplace=True)
 train_df['GarageFinish'].fillna('None', inplace=True)
 train_df['GarageQual'].fillna('None', inplace=True )
print('Total error', np.sqrt(mean_squared_error(y, 0.7*(0.75*(1.4*(1.6*y_oof_7-0.6*y_oof_6)-0.4*y_oof_5)+
0.25*(-0.5*y_oof-0.5*y_oof_2-y_oof_3
+3*y_oof_4)) +0.3*y_oof_8)) )<define_variables>
print('Number of missing values in garage area = ', test_df['GarageArea'].isnull().sum() )
sub_preds =(0.7*(0.75*(1.4*(1.6*total_preds_7-0.6*total_preds_6)-0.4*total_preds_5)+
0.25*(-0.5*total_preds-0.5*total_preds_2-total_preds_3
+3*total_preds_4)) +0.3*total_preds_8)
 sample_submission.target = np.exp(sub_preds)-1
sample_submission.to_csv('blended_submission_3.csv', index=False)
 sample_submission.head()<compute_train_metric>
test_df[(test_df['GarageArea']!=0)&(( test_df['GarageType'].isnull())| 
(test_df['GarageYrBlt'].isnull())| 
(test_df['GarageFinish'].isnull())| 
(test_df['GarageQual'].isnull())
)].loc[:, ['GarageArea', 'GarageType', 'GarageYrBlt', 'GarageFinish', 'GarageQual']]
feature_importances = clf.get_fscore()<define_variables>
test_df['GarageArea'].fillna(test_df['GarageArea'].median() , inplace=True )
best_2500 =[ x[0] for x in best_2500]<prepare_x_and_y>
test_df['GarageFinish'].value_counts()
params = {'objective': 'reg:linear',
'eval_metric': 'rmse',
'eta': 0.01,
'max_depth': 10,
'subsample': 0.6,
'colsample_bytree': 0.6,
'alpha':0.001,
'random_state': 42,
'silent': True}
y_oof_9 = np.zeros(( y.shape[0]))
total_preds_9 = 0
dtest = xgb.DMatrix(test_df[best_2500])
 kf = KFold(n_splits=nr_splits, shuffle=True, random_state=random_state)
 for i,(train_index, val_index)in enumerate(kf.split(y)) :
print('Fitting fold', i+1, 'out of', nr_splits)
 X_train, X_val = train_df[best_2500].iloc[train_index], train_df[best_2500].iloc[val_index]
y_train, y_val = y[train_index], y[val_index]
train = xgb.DMatrix(X_train, y_train)
 val = xgb.DMatrix(X_val, y_val)
 watchlist = [(train, 'train'),(val, 'val')]
clf = xgb.train(params, train, 1500, watchlist,
maximize=False, early_stopping_rounds = 60, verbose_eval=100)
 total_preds_9 += clf.predict(dtest, ntree_limit=clf.best_ntree_limit)/nr_splits
pred_oof = clf.predict(val, ntree_limit=clf.best_ntree_limit)
 y_oof_9[val_index] = pred_oof
print('Fold error', np.sqrt(mean_squared_error(y_val, pred_oof)))
 print('Total error', np.sqrt(mean_squared_error(y, y_oof_9)) )<compute_test_metric>
test_df['GarageQual'].value_counts()
print('Total error', np.sqrt(mean_squared_error(y, 0.5*y_oof_9+0.5*(0.7*(0.75*(1.4*(1.6*y_oof_7-0.6*y_oof_6)-0.4*y_oof_5)+
0.25*(-0.5*y_oof-0.5*y_oof_2-y_oof_3
+3*y_oof_4)) +0.3*y_oof_8))))
sub_preds = 0.5*total_preds_9+0.5*(0.7*(0.75*(1.4*(1.6*total_preds_7-0.6*total_preds_6)-0.4*total_preds_5)+
0.25*(-0.5*total_preds-0.5*total_preds_2-total_preds_3
+3*total_preds_4)) +0.3*total_preds_8)
 sample_submission.target = np.exp(sub_preds)-1
sample_submission.to_csv('blended_submission_4.csv', index=False)
 sample_submission.head()<set_options>
test_df.at[666, 'GarageYrBlt'] = 1983
test_df.at[1116, 'GarageYrBlt'] = 1999
test_df.at[666, 'GarageFinish'] = 'Unf'
test_df.at[1116, 'GarageFinish'] = 'Unf'
test_df.at[666, 'GarageQual'] = 'TA'
test_df.at[1116, 'GarageQual'] = 'TA'
plt.style.use('seaborn')
 sns.set(font_scale=2)
 %matplotlib inline
py.init_notebook_mode(connected=True)
<load_from_csv>
test_df['GarageCond'].fillna('None', inplace=True)
 test_df['GarageType'].fillna('None', inplace=True)
 test_df['GarageYrBlt'].fillna('None', inplace=True)
 test_df['GarageFinish'].fillna('None', inplace=True)
 test_df['GarageQual'].fillna('None', inplace=True )
df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv')
 print('Train(rows, columns)', df_train.shape)
 print('test(rows, columns)', df_test.shape )<statistical_test>
train_df.isnull().sum().sum()
columns = df_train.columns
r_value_array = []
for num in range(2, df_train.shape[1]):
x = df_train[columns[num]]
y = df_train['target']
slope, intercept, r_value, p_value, std_err = stats.linregress(x,y)
 r_value_array.append(r_value )<create_dataframe>
( data_df['BsmtExposure'].isnull().sum() ,
data_df['BsmtFinType2'].isnull().sum() ,
data_df['BsmtFinType1'].isnull().sum() ,
data_df['BsmtCond'].isnull().sum() ,
data_df['BsmtQual'].isnull().sum() )
df_r_values = pd.DataFrame({'column': columns[2:], 'r_value': r_value_array} )<filter>
data_df['TotalBsmtSF'].isnull().sum()
high_r_value_dataframe = df_r_values.loc[df_r_values.r_value.abs() > 0.05, :]<prepare_x_and_y>
data_df[(data_df['TotalBsmtSF'] != 0)&(( data_df['BsmtExposure'].isnull())| 
(data_df['BsmtFinType2'].isnull())| 
(data_df['BsmtFinType1'].isnull())| 
(data_df['BsmtCond'].isnull())| 
(data_df['BsmtQual'].isnull())
)].loc[:, ['TotalBsmtSF', 'BsmtExposure', 'BsmtFinType2', 'BsmtFinType1', 'BsmtCond', 'BsmtQual']]
new_df_train = df_train[high_r_value_dataframe['column']]
new_df_test = df_test[high_r_value_dataframe['column']]
Y = np.log(df_train.target+1 ).values<init_hyperparams>
data_df['BsmtExposure'].value_counts()
lgbm_params = {
'task': 'train',
'boosting_type': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'max_depth': 8,
'num_leaves': 32,
'feature_fraction': 0.8,
'bagging_fraction': 0.8,
'learning_rate': 0.01,
'verbose': 1
}<choose_model_class>
data_df['BsmtFinType2'].value_counts()
kf = KFold(n_splits=10, shuffle=True, random_state=1989 )<split>
train_df.at[332, 'BsmtFinType2'] = 'Unf'
train_df.at[948, 'BsmtExposure'] = 'No'
Y_target = []
for fold_id,(train_idx, val_idx)in enumerate(kf.split(new_df_train)) :
print('FOLD:',fold_id)
 X_train = new_df_train.values[train_idx]
y_train = Y[train_idx]
X_valid = new_df_train.values[val_idx]
y_valid = Y[val_idx]
lgtrain = lgb.Dataset(X_train, y_train,
feature_name=new_df_train.columns.tolist() , 
)
 lgvalid = lgb.Dataset(X_valid, y_valid,
feature_name=new_df_train.columns.tolist() , 
)
 modelstart = time.time()
lgb_clf = lgb.train(
 lgbm_params,
lgtrain,
num_boost_round=30000,
valid_sets=[lgtrain, lgvalid],
valid_names=['train','valid'],
early_stopping_rounds=100,
verbose_eval=100 
)
 test_pred = lgb_clf.predict(new_df_test.values)
 Y_target.append(np.exp(test_pred)-1)
 print('fold finish after', time.time() -modelstart )<save_to_csv>
train_df['BsmtExposure'].fillna('None', inplace=True)
 train_df['BsmtFinType2'].fillna('None', inplace=True)
 train_df['BsmtFinType1'].fillna('None', inplace=True)
 train_df['BsmtCond'].fillna('None', inplace=True)
 train_df['BsmtQual'].fillna('None', inplace=True )
Y_target = np.array(Y_target)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub['target'] = Y_target.mean(axis=0)
 sub.to_csv('simple_r_value_filter_result.csv', index=False )<import_modules>
( test_df['BsmtExposure'].isnull().sum() ,
test_df['BsmtFinType2'].isnull().sum() ,
test_df['BsmtFinType1'].isnull().sum() ,
test_df['BsmtCond'].isnull().sum() ,
test_df['BsmtQual'].isnull().sum() ,
test_df['TotalBsmtSF'].isnull().sum() )
from sklearn.model_selection import KFold
import lightgbm as lgb
from collections import Counter
import gc<load_from_csv>
test_df[(test_df['TotalBsmtSF'] != 0)&(( test_df['BsmtExposure'].isnull())| 
(test_df['BsmtFinType2'].isnull())| 
(test_df['BsmtFinType1'].isnull())| 
(test_df['BsmtCond'].isnull())| 
(test_df['BsmtQual'].isnull())
)].loc[:, ['TotalBsmtSF', 'BsmtExposure', 'BsmtFinType2', 'BsmtFinType1', 'BsmtCond', 'BsmtQual']]
def setup_dataset() :
train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 subs = pd.read_csv(".. /input/sample_submission.csv")
 train["target"] = np.log1p(train["target"])
 subs["target"] = 0.0
train = prepare(train.replace(0, np.nan),"train")
 test = prepare(test.replace(0, np.nan),"test")
 return train,test,subs
def prepare(data,case):
if case=="train":
del_col=["ID","target"]
elif case=="test":
del_col=["ID"]
data_c = data.copy()
data['mean'] = data_c.drop(del_col,axis=1 ).mean(axis=1)
 data['std'] = data_c.drop(del_col,axis=1 ).std(axis=1)
 data['min'] = data_c.drop(del_col,axis=1 ).min(axis=1)
 data['max'] = data_c.drop(del_col,axis=1 ).max(axis=1)
 data['number_of_different'] = data_c.drop(del_col,axis=1 ).nunique(axis=1)
 data['non_zero_count'] = data_c.drop(del_col,axis=1 ).fillna(0 ).astype(bool ).sum(axis=1)
 d_matrix = data_c.drop(del_col,axis=1 ).fillna(0 ).apply(pair_check,axis=1)
 tmp_result = pd.DataFrame(list(d_matrix),columns=["not_pair_cnt", "not_pair_sum", "not_pair_max", "not_pair_min", "not_pair_mean","even_pair_cnt","even_pair_sum", "even_pair_max","even_pair_min","even_pair_mean"])
 for col in tmp_result.columns.tolist() :
data[col]=tmp_result[col]
return data<randomize_order>
basement = ['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1',
'BsmtFinType2', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath']
test_df.loc[660, basement]
def pair_check(column_values):
v = column_values.values.tolist()
not_pair_cnt=0
not_pair_sum=0
not_pair_max=0
not_pair_min=9999999999
even_pair_cnt=0
even_pair_sum=0
even_pair_max=0
even_pair_min=0
for key,cnt in Counter(v ).items() :
if key ==0:
continue
if cnt %2==1:
not_pair_cnt+=cnt
not_pair_sum+=key
if not_pair_max<key:
not_pair_max=key
if not_pair_min>key:
not_pair_min=key
else:
even_pair_cnt+=cnt
even_pair_sum+=key
if even_pair_max<key:
even_pair_max=key
if even_pair_min>key:
even_pair_min=key
if not_pair_cnt==0:
not_pair_mean=0
else:
not_pair_mean = not_pair_sum / not_pair_cnt
if even_pair_cnt==0:
even_pair_mean=0
else:
even_pair_mean = even_pair_sum / even_pair_cnt
return(not_pair_cnt, not_pair_sum, not_pair_max, not_pair_min, not_pair_mean, even_pair_cnt,even_pair_sum, even_pair_max,even_pair_min, even_pair_mean )<train_model>
test_df.at[660, 'BsmtQual'] = None
test_df.at[660, 'BsmtCond'] = None
test_df.at[660, 'BsmtExposure'] = None
test_df.at[660, 'BsmtFinType1'] = None
test_df.at[660, 'BsmtFinSF1'] = 0
test_df.at[660, 'BsmtFinType2'] = None
test_df.at[660, 'BsmtFinSF2'] = 0
test_df.at[660, 'BsmtUnfSF'] = 0
test_df.at[660, 'TotalBsmtSF'] = 0
test_df.at[660, 'BsmtFullBath'] = None
test_df.at[660, 'BsmtHalfBath'] = None
def train_lgb(d_train, d_test, d_subs, epoch, kfold_num=3,
delete_cols=[], lgb_params=lgb_params):
best_score=0
print("epoch:{}".format(epoch))
d_train = d_train.drop(delete_cols,axis=1)
 d_test = d_test.drop(delete_cols,axis=1)
 X_test = d_test.drop(["ID"],axis=1)
 fti_list=np.zeros(d_train.shape[1] - 2)
 low_fti=[]
del d_test
gc.collect()
for i,(train_idx, valid_idx)in enumerate(KFold(n_splits=kfold_num ).split(d_train)) :
X_train = d_train.iloc[train_idx,:].drop(["ID","target"],axis=1)
 X_valid = d_train.iloc[valid_idx,:].drop(["ID","target"],axis=1)
 Y_train = d_train.iloc[train_idx,:]["target"]
Y_valid = d_train.iloc[valid_idx,:]["target"]
lgb_train = lgb.Dataset(X_train, label=Y_train)
 lgb_valid = lgb.Dataset(X_valid, label=Y_valid)
 lgb_clf = lgb.train(
 lgb_params,
lgb_train,
num_boost_round=10000,
valid_sets=[lgb_train, lgb_valid],
valid_names=['train','valid'],
early_stopping_rounds=50,
verbose_eval=50 
)
 fti_list += lgb_clf.feature_importance()
d_subs["target"] += np.expm1(lgb_clf.predict(X_test))
best_score += lgb_clf.best_score["valid"]['rmse']
d_subs["target"] /= kfold_num
best_score /= kfold_num
tee=[]
for i, feat in enumerate(X_train.columns.tolist()):
if fti_list[i]==0.0:
low_fti.append(feat)
 else:
tee.append([feat,fti_list[i]])
 print("best_score:{:.4f}".format(best_score))
return d_subs, low_fti, best_score,tee<train_model>
train_df['BsmtCond'].value_counts()
def execute_santander() :
print("start")
 train,d_test,subs = setup_dataset()
delete_cols=[]
fti_results=[]
score=99999.9
for epoch in range(10):
d_train = train.copy()
d_subs = subs.copy()
d_subs, low_fti, best_score,tee = train_lgb(d_train, d_test, d_subs, epoch=epoch,
kfold_num=3, delete_cols=delete_cols,
lgb_params=lgb_params)
 if score < best_score:
print("score is not improved")
 break
else:
score = best_score
delete_cols.extend(low_fti)
 print("delete_cols numbers:",len(delete_cols))
d_subs.to_csv("subs_lgb.csv",index=False)
 fti_results.append(tee)
 print("finish")
 return fti_results<feature_engineering>
train_df['BsmtQual'].value_counts()
fti_results = execute_santander()<create_dataframe>
test_df.at[27, 'BsmtExposure'] = 'No'
test_df.at[580, 'BsmtCond'] = 'TA'
test_df.at[725, 'BsmtCond'] = 'TA'
test_df.at[757, 'BsmtQual'] = 'TA'
test_df.at[758, 'BsmtQual'] = 'TA'
test_df.at[888, 'BsmtExposure'] = 'No'
test_df.at[1064, 'BsmtCond'] = 'TA'
df = pd.DataFrame(fti_results[0],columns=["name","score"])
 df.sort_values(by="score",ascending=False )<set_options>
test_df['BsmtExposure'].fillna('None', inplace=True)
 test_df['BsmtFinType2'].fillna('None', inplace=True)
 test_df['BsmtFinType1'].fillna('None', inplace=True)
 test_df['BsmtCond'].fillna('None', inplace=True)
 test_df['BsmtQual'].fillna('None', inplace=True )
sns.set_style('darkgrid')
 mpl.rcParams['figure.figsize'] = [15,10]<find_best_params>
( data_df['MasVnrType'].isnull().sum() , data_df['MasVnrArea'].isnull().sum() )
env = nflrush.make_env()<load_from_csv>
data_df[(data_df['MasVnrType'].isnull())|(data_df['MasVnrArea'].isnull())].loc[:, ['MasVnrType', 'MasVnrArea']]
train = pd.read_csv('.. /input/nfl-big-data-bowl-2020/train.csv', dtype={'WindSpeed': 'object'} )<feature_engineering>
len(data_df[data_df['MasVnrArea'] == 0])/len(data_df['MasVnrArea'] )
train['DefendersInTheBox_vs_Distance'] = train['DefendersInTheBox'] / train['Distance']<count_values>
train_df['MasVnrArea'].fillna(0, inplace=True )
train['StadiumType'].value_counts()<categorify>
train_df['MasVnrType'].fillna('None', inplace=True )
def clean_StadiumType(txt):
if pd.isna(txt):
return np.nan
txt = txt.lower()
txt = ''.join([c for c in txt if c not in punctuation])
 txt = re.sub(' +', ' ', txt)
 txt = txt.strip()
txt = txt.replace('outside', 'outdoor')
 txt = txt.replace('outdor', 'outdoor')
 txt = txt.replace('outddors', 'outdoor')
 txt = txt.replace('outdoors', 'outdoor')
 txt = txt.replace('oudoor', 'outdoor')
 txt = txt.replace('indoors', 'indoor')
 txt = txt.replace('ourdoor', 'outdoor')
 txt = txt.replace('retractable', 'rtr.')
 return txt<feature_engineering>
print('Number of missing values in MasVnrType ', test_df['MasVnrType'].isnull().sum() , 'and MasVnArea ', test_df['MasVnrArea'].isnull().sum() )
train['StadiumType'] = train['StadiumType'].apply(clean_StadiumType )<categorify>
test_df[(test_df['MasVnrType'].isnull())| 
(test_df['MasVnrArea'].isnull())].loc[:, ['MasVnrType', 'MasVnrArea']]
def transform_StadiumType(txt):
if pd.isna(txt):
return np.nan
if 'outdoor' in txt or 'open' in txt:
return 1
if 'indoor' in txt or 'closed' in txt:
return 0
return np.nan<feature_engineering>
test_df['MasVnrType'].value_counts()
train['StadiumType'] = train['StadiumType'].apply(transform_StadiumType )<define_variables>
test_df.at[1150, 'MasVnrType'] = 'BrkFace'
test_df['MasVnrType'].fillna('None', inplace=True)
 test_df['MasVnrArea'].fillna(test_df['MasVnrArea'].median() , inplace=True )
Turf = {'Field Turf':'Artificial', 'A-Turf Titan':'Artificial', 'Grass':'Natural', 'UBU Sports Speed S5-M':'Artificial',
'Artificial':'Artificial', 'DD GrassMaster':'Artificial', 'Natural Grass':'Natural',
'UBU Speed Series-S5-M':'Artificial', 'FieldTurf':'Artificial', 'FieldTurf 360':'Artificial', 'Natural grass':'Natural', 'grass':'Natural',
'Natural':'Natural', 'Artifical':'Artificial', 'FieldTurf360':'Artificial', 'Naturall Grass':'Natural', 'Field turf':'Artificial',
'SISGrass':'Artificial', 'Twenty-Four/Seven Turf':'Artificial', 'natural grass':'Natural'}
train['Turf'] = train['Turf'].map(Turf)
 train['Turf'] = train['Turf'] == 'Natural'<filter>
print('Number of missing values = ', data_df['Electrical'].isnull().sum() )
train[(train['PossessionTeam']!=train['HomeTeamAbbr'])&(train['PossessionTeam']!=train['VisitorTeamAbbr'])][['PossessionTeam', 'HomeTeamAbbr', 'VisitorTeamAbbr']]<categorify>
data_df['Electrical'].value_counts()
map_abbr = {'ARI': 'ARZ', 'BAL': 'BLT', 'CLE': 'CLV', 'HOU': 'HST'}
for abb in train['PossessionTeam'].unique() :
map_abbr[abb] = abb<categorify>
train_df['Electrical'].fillna('Sbrkr', inplace=True )
train['PossessionTeam'] = train['PossessionTeam'].map(map_abbr)
 train['HomeTeamAbbr'] = train['HomeTeamAbbr'].map(map_abbr)
 train['VisitorTeamAbbr'] = train['VisitorTeamAbbr'].map(map_abbr )<feature_engineering>
print('Number of missing values = ', test_df['Electrical'].isnull().sum() )
train['HomePossesion'] = train['PossessionTeam'] == train['HomeTeamAbbr']<feature_engineering>
train_df.isnull().sum().sum()
train['Field_eq_Possession'] = train['FieldPosition'] == train['PossessionTeam']
train['HomeField'] = train['FieldPosition'] == train['HomeTeamAbbr']<count_values>
test_df.columns[np.where(test_df.isnull().sum() != 0)]
off_form = train['OffenseFormation'].unique()
train['OffenseFormation'].value_counts()<categorify>
test_df['MSZoning'].fillna('RL', inplace=True)
 test_df['Utilities'].fillna('AllPub', inplace=True)
 test_df['Exterior1st'].fillna('VinylSd', inplace=True)
 test_df['Exterior2nd'].fillna('VinylSd', inplace=True)
 test_df['BsmtFullBath'].fillna(0, inplace=True)
 test_df['BsmtHalfBath'].fillna(0, inplace=True)
 test_df['KitchenQual'].fillna('TA', inplace=True)
 test_df['Functional'].fillna('Typ', inplace=True)
 test_df['GarageCars'].fillna(2, inplace=True)
 test_df['SaleType'].fillna('WD', inplace=True )
train = pd.concat([train.drop(['OffenseFormation'], axis=1), pd.get_dummies(train['OffenseFormation'], prefix='Formation')], axis=1)
 dummy_col = train.columns<count_values>
test_df.isnull().sum().sum()
train['GameClock'].value_counts()<feature_engineering>
train_df.drop(columns=['Id', 'SalePrice'], inplace=True)
 test_df.drop(columns=['Id'], inplace=True )
train['GameClock'] = train['GameClock'].apply(strtoseconds )<feature_engineering>
print('Current list of outliers ', list(outliers))
train['PlayerHeight'] = train['PlayerHeight'].apply(lambda x: 12*int(x.split('-')[0])+int(x.split('-')[1]))<feature_engineering>
train_df.drop(index=list(outliers), inplace=True )
train['PlayerBMI'] = 703*(train['PlayerWeight']/(train['PlayerHeight'])**2 )<data_type_conversions>
data_df.drop(index=list(outliers), inplace=True )
train['TimeHandoff'] = train['TimeHandoff'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))
train['TimeSnap'] = train['TimeSnap'].apply(lambda x: datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%S.%fZ"))<feature_engineering>
dummies = pd.get_dummies(pd.concat(( train_df, test_df), axis=0))
train['TimeDelta'] = train.apply(lambda row:(row['TimeHandoff'] - row['TimeSnap'] ).total_seconds() , axis=1 )<feature_engineering>
X = dummies.iloc[:train_df.shape[0]]
X_test = dummies.iloc[train_df.shape[0]:]
train['PlayerBirthDate'] = train['PlayerBirthDate'].apply(lambda x: datetime.datetime.strptime(x, "%m/%d/%Y"))<feature_engineering>
y = np.log(data_df['SalePrice'] + 1 )
seconds_in_year = 60*60*24*365.25
train['PlayerAge'] = train.apply(lambda row:(row['TimeHandoff']-row['PlayerBirthDate'] ).total_seconds() /seconds_in_year, axis=1 )<drop_column>
GBR3 = GradientBoostingRegressor(learning_rate= 0.1, n_estimators=150, max_depth=4)
 cv_score = cross_val_score(GBR3, X, y, cv=5, scoring='neg_mean_squared_error')
 print('Cross validation scores for GBR model:', np.sqrt(-cv_score ).mean() )
train = train.drop(['TimeHandoff', 'TimeSnap', 'PlayerBirthDate'], axis=1 )<count_values>
RF3 = RandomForestRegressor(n_estimators=100, max_features=20, random_state=0)
 cv_score2 = cross_val_score(RF3, X, y, cv=5, scoring='neg_mean_squared_error')
 print('Cross validation scores for GBR model:', np.sqrt(-cv_score2 ).mean() )
<feature_engineering><EOS> 
GBR3.fit(X, y)
 y_pred = np.exp(GBR3.predict(X_test)) - 1
answer = pd.DataFrame(data=y_pred, columns=['SalePrice'])
 answer.insert(loc=0, column='Id', value=test_data_df['Id'])
 answer.to_csv('submission.csv', index=False )