Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<feature_engineering>
folds = 5
debug = False
num_rounds=1000
use_ucf = True
use_sort = True
del_2016 = True
ucf_year = [2017, 2018]
numeric_feats = all_data.dtypes[train.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna()))
skewed_feats = skewed_feats[skewed_feats > 0.5]
skewed_feats = skewed_feats.index
all_data[skewed_feats] = np.log1p(all_data[skewed_feats])
<feature_engineering>
py.init_notebook_mode(connected=True)
numcols=all_data._get_numeric_data().columns
all_data_num=all_data.loc[:,numcols]
k=all_data_num.columns.to_series() [np.isinf(all_data_num ).any() ]
all_data[k] = all_data[k].replace(-np.inf, np.nan )<create_dataframe>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
all_data_mod=all_data[['LotArea','GrLivArea','TotRmsAbvGrd','OverallQual','OverallCond','YearBuilt','YearRemodAdd','TotalBsmtSF','BsmtQual_o','ExterQual_o','GarageCars','1stFlrSF','2ndFlrSF','FullBath','FireplaceQu_o','KitchenQual_o','MSZoning_o','PavedDrive_o','LotShape_o', 'Utilities_o', 'LandContour_o', 'Functional_o','tot_bath','bed_bath_kitch','area_floors','GarageArea','Fireplaces','BedroomAbvGr','tot_quality','sale_built_yr','remod_built_yr','MSSubClass','neighborhood_flag','MasVnrArea','garage_area_per_car','floor_by_lot','remod_flag','new_flag','unf_bsmt','bsmt_by_total','bsmt_bath','flag','BsmtFinSF1','BsmtFinType2','BsmtFinSF2','BsmtUnfSF']]
all_data_mod<drop_column>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
all_data_mod=all_data[['LotArea','GrLivArea','TotRmsAbvGrd','OverallQual','OverallCond','YearBuilt','YearRemodAdd','TotalBsmtSF','BsmtQual_o','ExterQual_o','GarageCars','1stFlrSF','2ndFlrSF','FullBath','FireplaceQu_o','KitchenQual_o','MSZoning_o','PavedDrive_o','LotShape_o', 'Utilities_o', 'LandContour_o', 'Functional_o','tot_bath','bed_bath_kitch','area_floors','GarageArea','Fireplaces','BedroomAbvGr','tot_quality','sale_built_yr','remod_built_yr','MSSubClass','neighborhood_flag','MasVnrArea','floor_by_lot','remod_flag','new_flag','unf_bsmt','bsmt_by_total','bsmt_bath','flag','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF']]
train_mod=all_data_mod.loc[all_data_mod['flag']=='train',:]
test_mod=all_data_mod.loc[all_data_mod['flag']=='test',:]
del train_mod['flag']
del test_mod['flag']
my_imputer = SimpleImputer(missing_values=np.nan,strategy='mean')
 train_imp = my_imputer.fit_transform(train_mod)
 test_imp = my_imputer.fit_transform(test_mod )<find_best_model_class>
ucf_root = Path('.. /input/ashrae-ucf-spider-and-eda-full-test-labels')
 leak_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
print(len(leak_df))
X=train_imp
y=np.log1p(salePrice)
 X_train,X_test,Y_train,Y_Test=train_test_split(X, y,test_size=0.3,random_state=1)
 regressor = RandomForestRegressor(n_estimators=100, random_state=1,max_depth=10,max_features=10,min_samples_leaf=5)
 feature_list=train_mod.columns
model=regressor.fit(X_train,Y_train)
 pred_rf=model.predict(X_test)
 cv_results = cross_validate(regressor, X_train, Y_train, cv=5,scoring='r2')
 sorted(cv_results.keys())
 scoreOfModel = model.score(X_test, Y_Test)
 print("RSquared value for Model",scoreOfModel)
 feat_importances = pd.Series(model.feature_importances_, index=feature_list)
 feat_importances=feat_importances.sort_values()
feat_importances.plot(kind='barh',figsize=(12,9))
print("RMSLE for Model",np.sqrt(mean_squared_log_error(np.exp(Y_Test),np.exp(pred_rf)))) 
<save_to_csv>
if use_ucf:
if del_2016:
print('delete all buildings site0 in 2016')
 bids = leak_df.building_id.unique()
train_df = train_df[train_df.building_id.isin(bids)== False]
leak_df = leak_df[leak_df.timestamp.dt.year.isin(ucf_year)]
train_df = pd.concat([train_df, leak_df])
 train_df.reset_index(inplace=True)
 weather_train_df = pd.concat([weather_train_df, weather_test_df])
 weather_train_df.reset_index(inplace=True )
pred=model.predict(test_imp)
 print(len(pred))
Id=pd.Series(range(1461,2920))
d = {'Id': Id, 'SalePrice': pred}
df = pd.DataFrame(data=d)
 df['SalePrice']=np.exp(df['SalePrice'])
 df
df.to_csv('submission.csv',index=False )<train_on_grid>
param_grid = {
'bootstrap': [True],
'max_depth': [5, 3,7,10,15],
'max_features': [5, 12,10,8],
'min_samples_split': [5, 10, 8,3],
'n_estimators': [100, 200,150,300]
}
rf = RandomForestRegressor()
grid_search_rf = GridSearchCV(estimator = rf, param_grid = param_grid,
cv = 5, n_jobs = -1, verbose = 2)
 grid_search_rf.fit(X_train,Y_train)
 grid_search_rf.best_params_<compute_test_metric>
del weather_test_df, leak_df,
gc.collect()
def evaluate(model, test_features, test_labels):
predictions = model.predict(test_features)
 errors = abs(predictions - test_labels)
 mape = 100 * np.mean(errors / test_labels)
 accuracy = 100 - mape
print('Model Performance')
 print('Average Error: {:0.4f} degrees.'.format(np.mean(errors)))
 print('Accuracy = {:0.2f}%.'.format(accuracy))
return accuracy<compute_train_metric>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
best_grid_rf = grid_search_rf.best_estimator_
pred_gs_rf=best_grid_rf.predict(X_test)
 scoreOfModel = best_grid_rf.score(X_test, Y_Test)
 print("Rqusared for the Model",scoreOfModel)
 pred_gs_rf=best_grid_rf.predict(X_test)
 print("RMSLE for the Model",np.sqrt(mean_squared_log_error(np.exp(Y_Test),np.exp(pred_gs_rf)))) 
<train_model>
building_meta_df[building_meta_df.site_id == 0]
xgb = xgboost.XGBRegressor(n_estimators=100, learning_rate=0.08, gamma=0, subsample=0.75,
colsample_bytree=1, max_depth=7)
 xgb.fit(X_train,Y_train )<train_on_grid>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
param_grid = {
'max_depth': [8,10,5],
'learning_rate': [0.05, 0.1, 0.3],
'colsample_bytree': [0.5, 0.7],
'lambda':[0.75,1],
'min_child_weight':[2,4,6],
'subsample': [0.5, 0.7, 0.8],
'n_estimators': [150, 200,250,300]
}
xg = xgboost.XGBRegressor()
grid_search_xg = GridSearchCV(estimator = xg, param_grid = param_grid,
cv = 3, n_jobs = -1, verbose = 2)
 grid_search_xg.fit(X_train,Y_train)
 best_grid_xg=grid_search_xg.best_params_<find_best_params>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
best_grid_xg=grid_search_xg.best_params_ 
<compute_test_metric>
preprocess(train_df )
best_grid_xg = grid_search_xg.best_estimator_
pred_gs_xg = best_grid_xg.predict(X_test)
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_gs_xg))))
print(explained_variance_score(pred_gs_xg,Y_Test))<train_on_grid>
if use_ucf and use_sort:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()
lasso=Lasso(max_iter=10000)
 parameters={'alpha': [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5]}
lasso_regressor=GridSearchCV(lasso,parameters,scoring='r2',cv=5)
 lasso_regressor.fit(X_train,Y_train)
 best_grid_lasso=lasso_regressor.best_params_
print(lasso_regressor.best_params_)
 print("R Squared value for the model",lasso_regressor.best_score_ )<compute_test_metric>
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group
lasso_model=Lasso(alpha=0.0001,max_iter=10000 ).fit(X_train,Y_train)
 pred_gs_lasso=lasso_model.predict(X_test)
 print("RMSLE for the model",np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_gs_lasso))))<train_on_grid>
weather_train_df.isna().sum()
ridge=Ridge(max_iter=10000)
 parameters={'alpha': [1e-15, 1e-10, 1e-8, 9e-4, 7e-4, 5e-4, 3e-4, 1e-4, 1e-3, 5e-2, 1e-2, 0.1, 0.3, 1, 3, 5]}
ridge_regressor=GridSearchCV(ridge,parameters,scoring='r2',cv=5)
 ridge_regressor.fit(X_train,Y_train)
 best_grid_ridge=ridge_regressor.best_params_
print(ridge_regressor.best_params_)
 print("R Squared value for the model",ridge_regressor.best_score_)
 ridge_model=Ridge(alpha=0.3,max_iter=10000 ).fit(X_train,Y_train)
 pred_gs_ridge=ridge_model.predict(X_test)
 print("RMSLE for the model",np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_gs_lasso))))
coef = pd.Series(ridge_model.coef_, index = train_mod.columns)
 print(coef )<compute_test_metric>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
pred_rf=best_grid_rf.predict(X_test)
 pred_xg=best_grid_xg.predict(X_test)
 pred_ls=lasso_model.predict(X_test)
 pred_rg=ridge_model.predict(X_test)
 pred_xg_ls_rg=(pred_xg+pred_ls+pred_rg)/3
print('RMSLE for Random Forest')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_rf))))
print('RMSLE for XG Boost')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_xg))))
print('RMSLE for Lasso')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_ls))))
print('RMSLE for Xgboost & Lasso & Ridge Blended')
 print(np.sqrt(mean_squared_log_error(np.exp(Y_Test), np.exp(pred_xg_ls_rg))))<save_to_csv>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
pred_test_rf=best_grid_rf.predict(test_imp)
 pred_test_xg=best_grid_xg.predict(test_imp)
 pred_test_ls=lasso_model.predict(test_imp)
 pred_test_rg=lasso_model.predict(test_imp)
 pred=(pred_test_ls+pred_test_xg+pred_test_rg)/3
print(len(pred))
Id=pd.Series(range(1461,2920))
d = {'Id': Id, 'SalePrice': pred}
df = pd.DataFrame(data=d)
 df['SalePrice']=np.exp(df['SalePrice'])
 df
df.to_csv('submission.csv',index=False )<set_options>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
print(check_output(["ls", ".. /input"] ).decode("utf8"))
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename)) 
<load_from_csv>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )<feature_engineering>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
train["SalePrice"] = np.log1p(train["SalePrice"] )<prepare_x_and_y>
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
all_data = pd.concat(( train, test)).reset_index(drop=True)
 all_data.drop(['SalePrice'], axis=1, inplace=True)
 print("all_data size is : {}".format(all_data.shape))<create_dataframe>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', ]
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head(20 )<data_type_conversions>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
all_data["PoolQC"] = all_data["PoolQC"].fillna("None")
 all_data["MiscFeature"] = all_data["MiscFeature"].fillna("None")
 all_data["Alley"] = all_data["Alley"].fillna("None")
 all_data["Fence"] = all_data["Fence"].fillna("None")
 all_data["FireplaceQu"] = all_data["FireplaceQu"].fillna("None")
 for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
all_data[col] = all_data[col].fillna('None' )<categorify>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))<data_type_conversions>
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0
for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
all_data[col] = all_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
all_data[col] = all_data[col].fillna('None')
 all_data["MasVnrType"] = all_data["MasVnrType"].fillna("None")
 all_data["MasVnrArea"] = all_data["MasVnrArea"].fillna(0)
 all_data['MSZoning'] = all_data['MSZoning'].fillna(all_data['MSZoning'].mode() [0] )<count_values>
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()
all_data['Utilities'].value_counts()<drop_column>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()
all_data = all_data.drop(['Utilities'], axis=1 )<data_type_conversions>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()
all_data["Functional"] = all_data["Functional"].fillna("Typ")
 all_data['Electrical'] = all_data['Electrical'].fillna(all_data['Electrical'].mode() [0])
 all_data['KitchenQual'] = all_data['KitchenQual'].fillna(all_data['KitchenQual'].mode() [0])
 all_data['Exterior1st'] = all_data['Exterior1st'].fillna(all_data['Exterior1st'].mode() [0])
 all_data['Exterior2nd'] = all_data['Exterior2nd'].fillna(all_data['Exterior2nd'].mode() [0])
 all_data['SaleType'] = all_data['SaleType'].fillna(all_data['SaleType'].mode() [0] )<data_type_conversions>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()
all_data['MSSubClass'] = all_data['MSSubClass'].fillna("None" )<sort_values>
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total / len(train_df)) )
all_data_na =(all_data.isnull().sum() / len(all_data)) * 100
all_data_na = all_data_na.drop(all_data_na[all_data_na == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :all_data_na})
 missing_data.head()<data_type_conversions>
del train_df, weather_train_df, building_meta_df
gc.collect()
all_data['MSSubClass'] = all_data['MSSubClass'].apply(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str )<categorify>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))
print('Shape all_data: {}'.format(all_data.shape))<feature_engineering>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )
all_data['TotalSF'] = all_data['TotalBsmtSF'] + all_data['1stFlrSF'] + all_data['2ndFlrSF']<sort_values>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
numeric_feats = all_data.dtypes[all_data.dtypes != "object"].index
skewed_feats = all_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False )<define_variables>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
skewed_features = skewness.index<compute_test_metric>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
lam = 0.15
for feat in skewed_features:
all_data[feat] = boxcox1p(all_data[feat], lam )<categorify>
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )
all_data = pd.get_dummies(all_data)
 print(all_data.shape )<split>
leak_score = 0
leak_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values
train = all_data[:ntrain]
test = all_data[ntrain:]<import_modules>
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
print('UCF score = ', np.sqrt(leak_score / len(leak_df)) )