Kaggle competition: Bike Sharing Demand
<SOS> metric: RMSLE Kaggle data source: bike-sharing-demand<drop_column>
register_matplotlib_converters()
del weather_test_df, leak_df,
gc.collect()<feature_engineering>
print('Test log and exp')
 test_count = 100
print('original value', test_count)
 x = np.log1p(test_count)
 print('log1p', x)
 print('expm1', np.expm1(x))
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )<filter>
df = pd.read_csv('.. /input/bike-sharing-demand/train.csv',parse_dates=['datetime'],index_col=0)
 df_test = pd.read_csv('.. /input/bike-sharing-demand/test.csv',parse_dates=['datetime'],index_col=0)
building_meta_df[building_meta_df.site_id == 0]<filter>
columns = ['count', 'season', 'holiday', 'workingday', 'weather', 'temp',
'atemp', 'humidity', 'windspeed', 'year', 'month', 'day', 'dayofweek','hour']
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )<feature_engineering>
def add_features(df):
df['year'] = df.index.year
df['month'] = df.index.month
df['day'] = df.index.day
df['dayofweek'] = df.index.dayofweek
df['hour'] = df.index.hour
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek 
<categorify>
add_features(df)
 add_features(df_test )
preprocess(train_df )<sort_values>
group_hour = df.groupby(['hour'])
 average_by_hour = group_hour['count'].mean()
if use_ucf and use_sort:
train_df = train_df.sort_values('month')
 train_df = train_df.reset_index()<categorify>
df["count"] = df["count"].map(np.log1p )
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_median = df_group.median().astype(np.float16)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 del df_group<count_missing_values>
group_year_month = df.groupby(['year','month'] )
weather_train_df.isna().sum()<groupby>
average_year_month = group_year_month['count'].mean()
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<groupby>
group_workingday_hour = df.groupby(['workingday','hour'])
 average_workingday_hour = group_workingday_hour['count'].mean()
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))<groupby>
df.to_csv('bike_all.csv',index=True,index_label='datetime',columns=columns )
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )<data_type_conversions>
np.random.seed(5)
 l = list(df.index)
 np.random.shuffle(l)
 df = df.loc[l]
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]<categorify>
rows = df.shape[0]
train = int (.7 * rows)
 test = rows-train
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()<feature_engineering>
df.iloc[:train].to_csv('bike_train.csv'
,index=False,header=False
,columns=columns )
train_df = reduce_mem_usage(train_df, use_float16=True)
 building_meta_df = reduce_mem_usage(building_meta_df, use_float16=True)
 weather_train_df = reduce_mem_usage(weather_train_df, use_float16=True )<define_variables>
df.iloc[train:].to_csv('bike_validation.csv'
,index=False,header=False
,columns=columns )
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', ] 
<prepare_x_and_y>
df_test.to_csv('bike_test.csv',index=True,index_label='datetime' )
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train<train_model>
with open('bike_train_column_list.txt','w')as f:
f.write(','.join(columns))
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log<choose_model_class>
!pip install xgboost==0.90
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed)
 oof_total = 0<split>
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error, mean_absolute_error
import xgboost as xgb
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof0 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof0 * len(y_train)
 del X_train, y_train
gc.collect()<split>
column_list_file = 'bike_train_column_list.txt'
train_file = 'bike_train.csv'
validation_file = 'bike_validation.csv'
test_file = 'bike_test.csv'
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof1 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof1 * len(y_train)
 del X_train, y_train
gc.collect()<split>
df_train = pd.read_csv(train_file,names=columns)
 df_validation = pd.read_csv(validation_file,names=columns )
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=num_rounds, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof2 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof2 * len(y_train)
 del X_train, y_train
gc.collect()<split>
X_train = df_train.iloc[:,1:]
y_train = df_train.iloc[:,0].ravel()
X_validation = df_validation.iloc[:,1:]
y_validation = df_validation.iloc[:,0].ravel()
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=num_rounds,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 sns.distplot(y_valid_pred_total)
 oof3 = mean_squared_error(y_train, y_valid_pred_total)
 oof_total += oof3 * len(y_train)
 del X_train, y_train
gc.collect()<compute_test_metric>
regressor = xgb.XGBRegressor(max_depth=5,n_estimators=150 )
print('oof score meter0 =', np.sqrt(oof0))
print('oof score meter1 =', np.sqrt(oof1))
print('oof score meter2 =', np.sqrt(oof2))
print('oof score meter3 =', np.sqrt(oof3))
print('oof score total =', np.sqrt(oof_total / len(train_df)) )<drop_column>
regressor.fit(X_train,y_train, eval_set = [(X_train, y_train),(X_validation, y_validation)] )
del train_df, weather_train_df, building_meta_df
gc.collect()<categorify>
eval_result = regressor.evals_result()
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()<load_from_disk>
training_rounds = range(len(eval_result['validation_0']['rmse']))
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )<merge>
df = pd.read_csv(validation_file,names=columns )
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test<predict_on_test>
result = regressor.predict(X_test )
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total 
<feature_engineering>
df['count_predicted'] = result
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )<save_to_csv>
df[df['count_predicted'] < 0]
if not debug:
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<feature_engineering>
def adjust_count(x):
if x < 0:
return 0
else:
return x
leak_score = 0
leak_df = pd.read_pickle(ucf_root/'site0.pkl')
 leak_df['meter_reading'] = leak_df.meter_reading_scraped
leak_df.drop(['meter_reading_original','meter_reading_scraped'], axis=1, inplace=True)
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[leak_df.timestamp.dt.year > 2016]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0
for bid in leak_df.building_id.unique() :
temp_df = leak_df[(leak_df.building_id == bid)]
for m in temp_df.meter.unique() :
v0 = sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'].values
v1 = temp_df[temp_df.meter==m].meter_reading.values
leak_score += mean_squared_error(np.log1p(v0), np.log1p(v1)) * len(v0)
 sample_submission.loc[(test_df.building_id == bid)&(test_df.meter==m), 'meter_reading'] = temp_df[temp_df.meter==m].meter_reading.values 
<save_to_csv>
df['count_predicted'] = df['count_predicted'].map(adjust_count )
if not debug:
sample_submission.to_csv('submission_ucf_replaced.csv', index=False, float_format='%.4f' )<compute_test_metric>
df[df['count_predicted'] < 0]
print('UCF score = ', np.sqrt(leak_score / len(leak_df)) )<load_from_csv>
df['count'] = df['count'].map(np.expm1)
 df['count_predicted'] = df['count_predicted'].map(np.expm1 )
INPUT = ".. /input/ashrae-energy-prediction/"
df_train = pd.read_csv(f"{INPUT}train.csv")
 df_test = pd.read_csv(f"{INPUT}test.csv")
 bldg_metadata = pd.read_csv(f"{INPUT}building_metadata.csv")
 weather_train = pd.read_csv(f"{INPUT}weather_train.csv")
 weather_test = pd.read_csv(f"{INPUT}weather_test.csv")
 sample = pd.read_csv(f"{INPUT}sample_submission.csv" )<drop_column>
value_counts =(residuals > 0 ).value_counts(sort=False)
 print(' Under Estimation: {0:0.2f}'.format(value_counts[True]/len(residuals)))
 print(' Over Estimation: {0:0.2f}'.format(value_counts[False]/len(residuals)) )
df_test = df_test.drop(columns=['row_id'] )<categorify>
print("RMSE: {0:0.2f}".format(mean_squared_error(df['count'],df['count_predicted'])**.5))
df_train = reduce_mem_usage(df=df_train)
 df_test = reduce_mem_usage(df=df_test)
 weather_train = reduce_mem_usage(df=weather_train)
 weather_test = reduce_mem_usage(df=weather_test )<merge>
def compute_rmsle(y_true, y_pred):
if type(y_true)!= np.ndarray:
y_true = np.array(y_true)
 if type(y_pred)!= np.ndarray:
y_pred = np.array(y_pred)
 return(np.average(( np.log1p(y_pred)- np.log1p(y_true)) **2)**.5 )
df_train = df_train.merge(bldg_metadata, on='building_id', how='left')
 df_test = df_test.merge(bldg_metadata, on='building_id', how='left')
 df_train = df_train.merge(weather_train, on=['site_id', 'timestamp'], how='left')
 df_test = df_test.merge(weather_test, on=['site_id', 'timestamp'], how='left' )<set_options>
print('RMSLE')
 print(compute_rmsle(100,50),
compute_rmsle(1000,500),
compute_rmsle(10000,5000))
del weather_train, weather_test, bldg_metadata
gc.collect()<data_type_conversions>
print('RMSLE')
 print(compute_rmsle(100,25),
compute_rmsle(1000,250),
compute_rmsle(10000,2500))
df_train['timestamp'] = pd.to_datetime(arg=df_train['timestamp'])
 df_test['timestamp'] = pd.to_datetime(arg=df_test['timestamp'] )<feature_engineering>
print('RMSE')
 print(mean_squared_error([100],[50])**.5,
mean_squared_error([1000],[500])**.5,
mean_squared_error([10000],[5000])**.5 )
df_train['year'] = df_train['timestamp'].dt.year
df_train['month'] = df_train['timestamp'].dt.month
df_train['day'] = df_train['timestamp'].dt.day
df_train['hour'] = df_train['timestamp'].dt.hour
df_test['year'] = df_test['timestamp'].dt.year
df_test['month'] = df_test['timestamp'].dt.month
df_test['day'] = df_test['timestamp'].dt.day
df_test['hour'] = df_test['timestamp'].dt.hour
df_train['dayofweek'] = df_train['timestamp'].dt.dayofweek
df_test['dayofweek'] = df_test['timestamp'].dt.dayofweek
df_train['day-and-night'] = np.where(( df_train['hour'] < 6), 2, 1)
 df_test['day-and-night'] = np.where(( df_test['hour'] < 6), 2, 1)
 df_train.loc[df_train['month'].isin([12, 1, 2]), 'season'] = 1
df_train.loc[df_train['month'].isin([3, 4, 5]), 'season'] = 2
df_train.loc[df_train['month'].isin([6, 7, 8]), 'season'] = 3
df_train.loc[df_train['month'].isin([9, 10, 11]), 'season'] = 4
df_test.loc[df_test['month'].isin([12, 1, 2]), 'season'] = 1
df_test.loc[df_test['month'].isin([3, 4, 5]), 'season'] = 2
df_test.loc[df_test['month'].isin([6, 7, 8]), 'season'] = 3
df_test.loc[df_test['month'].isin([9, 10, 11]), 'season'] = 4<drop_column>
print('RMSE')
 print(mean_squared_error([100],[25])**.5,
mean_squared_error([1000],[250])**.5,
mean_squared_error([10000],[2500])**.5 )
df_train = reduce_mem_usage(df=df_train)
 df_test = reduce_mem_usage(df=df_test )<merge>
print("RMSLE: {0}".format(compute_rmsle(df['count'],df['count_predicted'])) )
df_train['age'] = df_train['year'] - df_train['year_built']
df_test['age'] = df_test['year'] - df_test['year_built']
new_df = df_train.groupby(by=['building_id'], as_index=False)['timestamp'].min()
new_df = new_df.rename(columns = {'timestamp': 'start_ts'})
 df_train = df_train.merge(new_df, on = 'building_id', how='left')
 df_test = df_test.merge(new_df, on = 'building_id', how='left')
 df_train['hours_passed'] =(df_train['timestamp'] - df_train['start_ts'] ).dt.total_seconds() /3600
df_test['hours_passed'] =(df_test['timestamp'] - df_test['start_ts'] ).dt.total_seconds() /3600<drop_column>
df_test = pd.read_csv(test_file,parse_dates=['datetime'] )
df_train = reduce_mem_usage(df=df_train)
 df_test = reduce_mem_usage(df=df_test )<data_type_conversions>
X_test = df_test.iloc[:,1:]
<feature_engineering>
result = regressor.predict(X_test )
df_train = df_train.query('not(site_id==0 & timestamp<"2016-05-21 00:00:00")')
 df_train = df_train.loc[df_train['meter_reading'] > 0, :]
cols = ['floor_count', 'air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed']
df_train.loc[:, cols] = df_train.loc[:, cols].interpolate(axis=0)
 df_test.loc[:, cols] = df_test.loc[:, cols].interpolate(axis=0 )<data_type_conversions>
np.expm1(result )
cat_cols = ['meter', 'primary_use', 'site_id', 'building_id', 'year', 'month', 'day', 'hour', 'dayofweek', 'season', 'day-and-night']
for col in cat_cols:
df_train[col] = df_train[col].astype('category')
 df_test[col] = df_test[col].astype('category' )<drop_column>
df_test["count"] = np.expm1(result )
df_train = df_train.reset_index(drop=True )<prepare_x_and_y>
df_test[df_test["count"] < 0]
df_val = df_train.loc[df_train['timestamp'] >= '2016-11-01 00:00:00']
df_train = df_train.loc[df_train['timestamp'] < '2016-11-01 00:00:00']
y_train = df_train['meter_reading']
y_val = df_val['meter_reading']
y_train = np.log1p(y_train)
 y_val = np.log1p(y_val)
 df_train = df_train.drop(columns=['meter_reading'])
 df_val = df_val.drop(columns=['meter_reading'])
 df_train = df_train.drop(columns=['timestamp', 'start_ts'])
 df_val = df_val.drop(columns=['timestamp', 'start_ts'])
 df_test = df_test.drop(columns=['timestamp', 'start_ts'] )<set_options>
df_test[['datetime','count']].to_csv('predicted_count_log.csv',index=False )