Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<load_from_csv>
py.init_notebook_mode(connected=True)
<load_from_csv>
RAW_DATA_DIR = '/kaggle/input/ashrae-energy-prediction/'
weather_dtypes = {
'site_id': np.uint8,
'air_temperature': np.float32,
'cloud_coverage': np.float32,
'dew_temperature': np.float32,
'precip_depth_1_hr': np.float32,
'sea_level_pressure': np.float32,
'wind_direction': np.float32,
'wind_speed': np.float32,
}
weather_train = pd.read_csv(pjoin(RAW_DATA_DIR, 'weather_train.csv'),dtype=weather_dtypes,
parse_dates=['timestamp'])
 weather_test = pd.read_csv(pjoin(RAW_DATA_DIR, 'weather_test.csv'),dtype=weather_dtypes,
parse_dates=['timestamp'])
 weather = pd.concat([weather_train,weather_test],ignore_index=True)
 del weather_train, weather_test
weather_key = ['site_id', 'timestamp']
temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key ).sort_values(by=weather_key ).copy()
del weather
<load_from_csv>
temp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')
 df_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)
 site_ids_offsets = pd.Series(df_2d.values.argmax(axis=1)- 14)
 site_ids_offsets.index.name = 'site_id'
def timestamp_align(df):
df['offset'] = df.site_id.map(site_ids_offsets)
 df['timestamp_aligned'] =(df.timestamp - pd.to_timedelta(df.offset, unit='H'))
df['timestamp'] = df['timestamp_aligned']
del df['timestamp_aligned']
return df
<load_pretrained>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
train_df_stft = pd.read_pickle('.. /input/volcano-features/train_df_stft.pkl')
 time = train_df_stft['time_to_eruption']
train_df_stft = train_df_stft.drop(['segment_id', 'time_to_eruption'], axis=1)
 train_df_sd = pd.read_pickle('.. /input/volcano-features/train_df_sd.pkl')
 test_df_stft = pd.read_pickle('.. /input/volcano-features/test_df_stft.pkl')
 test_df_stft = test_df_stft.drop(['segment_id'],axis = 1)
 test_df_sd = pd.read_pickle('.. /input/volcano-features/test_df_sd.pkl' )<create_dataframe>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
pca = PCA(n_components=0.999,svd_solver="full")
 train_pca = pca.fit_transform(train_df_sd)
 test_pca = pca.transform(test_df_sd)
 train_pca = pd.DataFrame(train_pca)
 test_pca = pd.DataFrame(test_pca )<concatenate>
building_site_dict = dict(zip(building_meta_df['building_id'], building_meta_df['site_id']))
site_meter_raw = train_df[['building_id', 'meter', 'timestamp', 'meter_reading']].copy()
site_meter_raw['site_id'] = site_meter_raw.building_id.map(building_site_dict)
 del site_meter_raw['building_id']
site_meter_to_plot = site_meter_raw.copy()
site_meter_to_plot["hour"] = site_meter_to_plot["timestamp"].dt.hour
elec_to_plot = site_meter_to_plot[site_meter_to_plot.meter == 0]
train_df = pd.concat([train_df_stft,train_pca] ,axis=1)
 test_df = pd.concat([test_df_stft,test_pca],axis = 1 )<load_from_csv>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'])
 building_meta_df[building_meta_df.site_id == 0]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 debug = False
preprocess(train_df)
 df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16)
 train_df['building_mean'] = train_df['building_id'].map(building_mean)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 train_df['building_min'] = train_df['building_id'].map(building_min)
 train_df['building_max'] = train_df['building_id'].map(building_max)
 train_df['building_std'] = train_df['building_id'].map(building_std)
 weather_train_df = timestamp_align(weather_train_df)
 weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72)
 primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
reduce_mem_usage(train_df, use_float16=True)
 reduce_mem_usage(building_meta_df, use_float16=True)
 reduce_mem_usage(weather_train_df, use_float16=True )
input_df = pd.read_csv('.. /input/predict-volcanic-eruptions-ingv-oe/train.csv')
 input_df = input_df.sort_values("time_to_eruption")
 fold_list = [1,2,3,4,5]
folds = []
for i in range(int(( input_df.shape[0]-1)/5)) :
random.shuffle(fold_list)
 folds.extend(fold_list)
 folds = folds + [1]
input_df['fold'] = folds 
<train_model>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']
predictions = np.zeros(len(test_df))
for fold in range(1,6):
train_index_list = input_df[input_df['fold'] != fold].index
test_index_list = input_df[input_df['fold'] == fold].index
X_train = train_df.iloc[train_index_list]
y_train = time[train_index_list]
X_val = train_df.iloc[test_index_list]
y_val = time[test_index_list]
model = xgboost.XGBRegressor(n_estimators=100000,tree_method='gpu_hist',max_depth=8,learning_rate=0.05,alpha=0.1,SUBSAMPLE=0.6)
 eval_set = [(X_val, y_val)]
model.fit(X_train, y_train,early_stopping_rounds=5,eval_metric='mae', eval_set=eval_set, verbose=False)
 print(model.evals_result() ['validation_0']['mae'][-5:])
 predictions += model.predict(test_df)
 predictions = predictions/5
sample_submission_df1=pd.read_csv('.. /input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')
 sample_submission_df1['time_to_eruption']=predictions
sample_submission_df1.to_csv('xgb_5fldst_ft_sdpca_stft.csv',index=False )<load_from_csv>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed )
train = pd.read_csv('.. /input/ingv-tsfresh-7730/train.csv', sep = ';')
 train.set_index('Unnamed: 0', inplace = True)
 test = pd.read_csv('.. /input/ingv-tsfresh-7730/test.csv', sep = ';')
 test.set_index('Unnamed: 0', inplace = True )<define_variables>
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
test_index = test.index<create_dataframe>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
train_rf = train.copy()<data_type_conversions>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
train_rf = train_rf.fillna(0 )<prepare_x_and_y>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
x = train_rf.drop('time_to_eruption', axis=1)
 y = train_rf['time_to_eruption']<train_model>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_mean'] = test_df['building_id'].map(building_mean)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 test_df['building_min'] = test_df['building_id'].map(building_min)
 test_df['building_max'] = test_df['building_id'].map(building_max)
 test_df['building_std'] = test_df['building_id'].map(building_std)
 print('preprocessing weather...')
 weather_test_df = timestamp_align(weather_test_df)
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
model = RandomForestRegressor(random_state=1, max_depth=7)
 model.fit(x,y )<features_selection>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )
selected_feature = feature_scores[:350].index<concatenate>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
target = train['time_to_eruption']
all_data = pd.concat([train, test], ignore_index = True)
 all_data.head()<concatenate>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
all_data = pd.concat([all_data[selected_feature], all_data['time_to_eruption']], axis=1)
 all_data.head()<count_missing_values>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
<count_missing_values><EOS> 
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')
 sample_submission.head()