Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<correct_missing_values>
py.init_notebook_mode(connected=True)
all_data = all_data.fillna(all_data.mode() )<normalization>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
header = all_data.columns
scaler = StandardScaler()
all_data[header] = scaler.fit_transform(all_data)
 all_data.head()<drop_column>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
all_data = all_data.drop('time_to_eruption', axis=1)
 all_data.var()<count_missing_values>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
missing_values = missing_values_table(all_data)
 missing_values<create_dataframe>
building_meta_df[building_meta_df.site_id == 0]
pca = PCA()
all_data_pca = pca.fit_transform(all_data)
 all_data_pca = pd.DataFrame(all_data_pca)
 all_data_pca.head()<split>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
train = all_data_pca[:train.shape[0]]
test = all_data_pca[train.shape[0]:]<init_hyperparams>
debug = False
Y = target
X = train
X_test = test
n_fold = 5
cv = KFold(n_splits=n_fold, shuffle=True, random_state=123)
 oof = np.zeros(len(X))
cat_prediction = np.zeros(len(X_test))
mae, r2 = [], []
PARAMS = {
'random_seed': 123,
'eval_metric': 'MAE',
'task_type': 'GPU',
'bagging_temperature': 0.41010395885331385,
'border_count': 186,
'depth': 10,
'iterations': 700,
'l2_leaf_reg': 30,
'learning_rate': 0.067,
'random_strength': 3.230824361824754e-06,
}
for fold_n,(train_index, valid_index)in enumerate(cv.split(X)) :
print('
Fold', fold_n, 'started at', ctime())
 X_train = X.iloc[train_index,:]
X_valid = X.iloc[valid_index,:]
Y_train = Y.iloc[train_index]
Y_valid = Y.iloc[valid_index]
best_model = CatBoostRegressor(**PARAMS, thread_count = -1)
 train_dataset = Pool(data=X_train,
label=Y_train, 
)
 eval_dataset = Pool(data=X_valid,
label=Y_valid, 
)
 best_model.fit(train_dataset,
use_best_model=True,
verbose = False,
plot = True,
eval_set=eval_dataset,
early_stopping_rounds=70)
 y_pred = best_model.predict(Pool(data=X_valid))
mae.append(mean_absolute_error(Y_valid, y_pred))
r2.append(r2_score(Y_valid, y_pred))
print('MAE: ', mean_absolute_error(Y_valid, y_pred))
print('R2: ', r2_score(Y_valid, y_pred))
cat_prediction += best_model.predict(Pool(data=X_test))
cat_prediction /= n_fold
print('='*45)
 print('CV mean MAE: {0:.4f}, std: {1:.4f}.'.format(np.mean(mae), np.std(mae)))
 print('CV mean R2: {0:.4f}, std: {1:.4f}.'.format(np.mean(r2), np.std(r2)) )<create_dataframe>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
<prepare_x_and_y>
preprocess(train_df )
<save_to_csv>
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16)
 train_df['building_mean'] = train_df['building_id'].map(building_mean)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 train_df['building_min'] = train_df['building_id'].map(building_min)
 train_df['building_max'] = train_df['building_id'].map(building_max)
 train_df['building_std'] = train_df['building_id'].map(building_std )
submission = pd.DataFrame()
submission['segment_id'] = test_index
submission['time_to_eruption'] = cat_prediction
submission.to_csv('submission.csv', header=True, index=False )<import_modules>
weather_train_df.isna().sum()
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import seaborn as sns
import os
from tqdm import tqdm
import gc
import pickle
import scipy
import scipy.signal<load_from_csv>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
train = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/train.csv")
 sample_submission = pd.read_csv(".. /input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv" )<count_missing_values>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
train.isna().sum(axis=0 ).sum()<load_from_csv>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
def generate_feature_timedomain() :
def helper(path):
data = []
for file in tqdm(os.listdir(path)) :
tmp = []
file_path = os.path.join(path, file)
 d = pd.read_csv(file_path)
 tmp.append(eval(file[:-4]))
tmp += d.mean(axis=0 ).values.astype('float32' ).tolist()
tmp += d.std(axis=0 ).values.astype('float32' ).tolist()
tmp += d.min(axis=0 ).values.astype('float32' ).tolist()
tmp += d.max(axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.05, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.1, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.2, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.4, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.6, axis=0 ).values.astype('float32' ).tolist()
tmp += d.quantile(0.8, axis=0 ).values.astype('float32' ).tolist()
for col in d:
d[col+'_5000'] = d[col].shift(5000)
 d[col+'_10000'] = d[col].shift(10000)
 d[col+'_20000'] = d[col].shift(20000)
 d[col+'_30000'] = d[col].shift(30000)
 for col in d.columns[:10]:
col1 = col+'_5000'
col2 = col+'_10000'
col3 = col+'_20000'
col4 = col+'_30000'
tmp1 = d.loc[:, [col, col1]].dropna()
tmp2 = d.loc[:, [col, col2]].dropna()
tmp3 = d.loc[:, [col, col3]].dropna()
tmp4 = d.loc[:, [col, col4]].dropna()
tmp += [tmp1[col].corr(tmp1[col1]),
tmp2[col].corr(tmp2[col2]),
tmp3[col].corr(tmp3[col3]),
tmp4[col].corr(tmp4[col4])]
data.append(tmp)
 return data
print('train_part: ')
 train_part_fea = helper('.. /input/predict-volcanic-eruptions-ingv-oe/train')
 print('test_part: ')
 test_part_fea = helper('.. /input/predict-volcanic-eruptions-ingv-oe/test')
 return train_part_fea, test_part_fea<load_from_csv>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
def generate_feature_freq_domain() :
fs = 100
n = 256
N = 60001
max_f = 20
delta_f = fs / n
delta_t = n / fs / 2
def helper(fs, n, N, max_f, delta_f, path):
data = []
for file in tqdm(os.listdir(path)) :
tmp = []
file_path = os.path.join(path, file)
 d = pd.read_csv(file_path)
 tmp.append(eval(file[:-4]))
for i in range(d.shape[1]):
if d.iloc[:, i].isna().sum() > 1000:
tmp += [np.nan] * 7 * 65
tmp += [np.nan] * 10
else:
f, t, Z = scipy.signal.stft(d.iloc[:, i].fillna(0 ).values, fs = fs, window = 'hann', nperseg = n)
 f = f[:round(max_f/delta_f)+1]
Z_half = np.abs(Z[:round(Z.shape[0]//2)+1] ).T
tmp += Z_half.min(axis=0 ).astype('float32' ).tolist()
tmp += Z_half.max(axis=0 ).astype('float32' ).tolist()
tmp += Z_half.std(axis=0 ).astype('float32' ).tolist()
tmp += Z_half.mean(axis=0 ).astype('float32' ).tolist()
tmp += np.quantile(Z_half, 0.25, axis=0 ).astype('float32' ).tolist()
tmp += np.quantile(Z_half, 0.5, axis=0 ).astype('float32' ).tolist()
tmp += np.quantile(Z_half, 0.75, axis=0 ).astype('float32' ).tolist()
Z = np.abs(Z[:round(max_f/delta_f)+1] ).T
th = Z.mean() * 1
Z_pow = Z.copy()
Z_pow[Z < th] = 0
Z_num = Z_pow.copy()
Z_num[Z >= th] = 1
Z_pow_sum = Z_pow.sum(axis = 0)
 Z_num_sum = Z_num.sum(axis = 0)
 A_pow = Z_pow_sum[round(10/delta_f):].sum()
A_num = Z_num_sum[round(10/delta_f):].sum()
BH_pow = Z_pow_sum[round(5/delta_f):round(8/delta_f)].sum()
BH_num = Z_num_sum[round(5/delta_f):round(8/delta_f)].sum()
BL_pow = Z_pow_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
BL_num = Z_num_sum[round(1.5/delta_f):round(2.5/delta_f)].sum()
C_pow = Z_pow_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
C_num = Z_num_sum[round(0.6/delta_f):round(1.2/delta_f)].sum()
D_pow = Z_pow_sum[round(2/delta_f):round(4/delta_f)].sum()
D_num = Z_num_sum[round(2/delta_f):round(4/delta_f)].sum()
tmp += [A_pow, A_num, BH_pow, BH_num, BL_pow, BL_num, C_pow, C_num, D_pow, D_num]
data.append(tmp)
 return data
print('train_part: ')
 train_part_fea = helper(fs, n, N, max_f, delta_f, path='.. /input/predict-volcanic-eruptions-ingv-oe/train')
 print('test_part: ')
 test_part_fea = helper(fs, n, N, max_f, delta_f, path='.. /input/predict-volcanic-eruptions-ingv-oe/test')
 return train_part_fea, test_part_fea<feature_engineering>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
train_part_fea, test_part_fea = generate_feature_timedomain()<load_pretrained>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
with open('train_part_fea_time_domain.pkl', 'wb')as f:
pickle.dump(train_part_fea, f)
 with open('test_part_fea_time_domain.pkl', 'wb')as f:
pickle.dump(test_part_fea, f )<feature_engineering>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']
train_part_fea_freq, test_part_fea_freq = generate_feature_freq_domain()<load_pretrained>
def create_X_y(train_df, target_meter):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
with open('train_part_fea_freq_domain.pkl', 'wb')as f:
pickle.dump(train_part_fea_freq, f)
 with open('test_part_fea_freq_domain.pkl', 'wb')as f:
pickle.dump(test_part_fea_freq, f )<load_pretrained>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
<define_variables>
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed )
base_colname = ['sensor_'+str(i)for i in range(1, 11)]
fea_colname = ['segment_id'] + [j + '_mean' for j in base_colname] + [j + '_std' for j in base_colname] + \
[j + '_min' for j in base_colname] + [j + '_max' for j in base_colname] + \
[j + '_5_quant' for j in base_colname] + [j + '_10_quant' for j in base_colname] + \
[j + '_20_quant' for j in base_colname] + [j + '_40_quant' for j in base_colname] + \
[j + '_60_quant' for j in base_colname] + [j + '_80_quant' for j in base_colname] + \
[j + i for j in base_colname for i in ['_5000_self_corr', '_10000_self_corr',
'_20000_self_corr', '_30000_self_corr']]
train = pd.merge(train, pd.DataFrame(train_part_fea, columns=fea_colname), on='segment_id', how='left')
 sample_submission = pd.merge(sample_submission, pd.DataFrame(test_part_fea, columns=fea_colname), on='segment_id', how='left' )<concatenate>
target_meter = 0
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models0 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.7)
 y_valid_pred_total[valid_idx] = y_pred_valid
models0.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
fea_freq_colname = ['segment_id']
for i in base_colname:
for j in range(65):
for s in ['min','max', 'std', 'mean', '25_quant', '50_quant', '75_quant']:
fea_freq_colname.append(i+'_freq'+str(j)+'_'+s)
 fea_freq_colname.extend([i + ss for ss in ['_A_pow', '_A_num', '_BH_pow', '_BH_num', '_BL_pow',
'_BL_num', '_C_pow', '_C_num', '_D_pow', '_D_num']])
 train = pd.merge(train, pd.DataFrame(train_part_fea_freq, columns=fea_freq_colname), on='segment_id', how='left')
 sample_submission = pd.merge(sample_submission, pd.DataFrame(test_part_fea_freq,
columns=fea_freq_colname), on='segment_id', how='left' )<load_pretrained>
target_meter = 1
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models1 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.05, bf=0.5)
 y_valid_pred_total[valid_idx] = y_pred_valid
models1.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
train.to_pickle('train.pkl')
 sample_submission.to_pickle('test.pkl' )<split>
target_meter = 2
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models2 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,
num_rounds=1000, lr=0.05, bf=0.8)
 y_valid_pred_total[valid_idx] = y_pred_valid
models2.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
X_train, X_val, y_train, y_val = train_test_split(train.drop(['segment_id', 'time_to_eruption'], axis=1 ).values,
train['time_to_eruption'].values,
test_size=0.25, random_state=42 )<create_dataframe>
target_meter = 3
X_train, y_train = create_X_y(train_df, target_meter=target_meter)
 y_valid_pred_total = np.zeros(X_train.shape[0])
 gc.collect()
print('target_meter', target_meter, X_train.shape)
 cat_features = [X_train.columns.get_loc(cat_col)for cat_col in category_cols]
print('cat_features', cat_features)
 models3 = []
for train_idx, valid_idx in kf.split(X_train, y_train):
train_data = X_train.iloc[train_idx,:], y_train[train_idx]
valid_data = X_train.iloc[valid_idx,:], y_train[valid_idx]
print('train', len(train_idx), 'valid', len(valid_idx))
model, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, num_rounds=1000,
lr=0.03, bf=0.9)
 y_valid_pred_total[valid_idx] = y_pred_valid
models3.append(model)
 gc.collect()
if debug:
break
sns.distplot(y_train)
 del X_train, y_train
gc.collect()
train_data = lgb.Dataset(X_train, label=y_train)
 val_data = lgb.Dataset(X_val, y_val, reference=train_data,)
 params = { 'num_leaves': 85,
'n_estimators': 1000,
'min_data_in_leaf': 10,
'objective':'mae',
'max_depth': -1,
'learning_rate': 0.01,
'max_bins': 2048,
"boosting": "gbdt",
"feature_fraction": 0.91,
"bagging_freq": 1,
"bagging_fraction": 0.91,
"bagging_seed": 42,
"metric": 'mae',
"lambda_l1": 0.1,
"verbosity": -1,
"nthread": -1,
"random_state": 42}
model = lgb.train(params=params, train_set=train_data, valid_sets=[train_data, val_data], valid_names=['train', 'val'],
early_stopping_rounds=50 )<save_to_csv>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_mean'] = test_df['building_id'].map(building_mean)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 test_df['building_min'] = test_df['building_id'].map(building_min)
 test_df['building_max'] = test_df['building_id'].map(building_max)
 test_df['building_std'] = test_df['building_id'].map(building_std)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
submission = pd.DataFrame({'segment_id': sample_submission['segment_id'].values,
'time_to_eruption': model.predict(sample_submission.iloc[:, 2:].values)})
 submission.to_csv('submission.csv', index=False )<import_modules>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import os
from sklearn.model_selection import train_test_split
from lightgbm import LGBMRegressor
from sklearn.metrics import mean_squared_error as mse
from sklearn.metrics import mean_absolute_error as mae 
<load_from_csv>
def create_X(test_df, target_meter):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 X_test = target_test_df[feature_cols + category_cols]
return X_test
sample_submission = pd.read_csv('/kaggle/input/predict-volcanic-eruptions-ingv-oe/sample_submission.csv')
 sample_submission.head(10 )<load_from_csv>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
train_set = pd.read_csv('/kaggle/input/volcano-stft-data/output_train_set.csv', index_col=0 ).reset_index(drop=True)
 train_set.head(10 )<load_from_csv>
sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)
 sample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)
 sample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)
 sample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test3 )
test_set = pd.read_csv('/kaggle/input/volcano-stft-data/output_test_set.csv', index_col=0 ).reset_index(drop=True)
 test_set.head(10 )<filter>
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )