Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<import_modules>
path_data = "/kaggle/input/ashrae-energy-prediction/"
path_train = path_data + "train.csv"
path_test = path_data + "test.csv"
path_building = path_data + "building_metadata.csv"
path_weather_train = path_data + "weather_train.csv"
path_weather_test = path_data + "weather_test.csv"
plt.style.use("seaborn")
 sns.set(font_scale=1)
 myfavouritenumber = 0
seed = myfavouritenumber
random.seed(seed )
import numpy as np
import pandas as pd
import os<import_modules>
df_train = pd.read_csv(path_train)
 building = pd.read_csv(path_building)
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 weather_train = pd.read_csv(path_weather_train )
from sklearn.model_selection import train_test_split
from sklearn import preprocessing<normalization>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def normalize(X_train, X_valid, X_test, normalize_opt, excluded_feat):
feats = [f for f in X_train.columns if f not in excluded_feat]
if normalize_opt is not None:
if normalize_opt == 'min_max':
scaler = preprocessing.MinMaxScaler()
scaler = scaler.fit(X_train[feats])
 X_train[feats] = scaler.transform(X_train[feats])
 X_valid[feats] = scaler.transform(X_valid[feats])
 X_test[feats] = scaler.transform(X_test[feats])
 return X_train, X_valid, X_test<load_from_csv>
df_train = reduce_mem_usage(df_train, use_float16=True)
 building = reduce_mem_usage(building, use_float16=True)
 weather_train = reduce_mem_usage(weather_train, use_float16=True )
PATH_DATASET = '/kaggle/input/vulcanic-preprocessing/'
train_sample = pd.read_csv(f'{PATH_DATASET}/train_sample.csv')
 targets = pd.read_csv(f'{PATH_DATASET}/targets.csv')
 test = pd.read_csv(f'{PATH_DATASET}/test.csv' ).iloc[:,1:]<install_modules>
def prepare_data(X, building_data, weather_data, test=False):

 X = X.merge(building_data, on="building_id", how="left")
 X = X.merge(weather_data, on=["site_id", "timestamp"], how="left")
 X.timestamp = pd.to_datetime(X.timestamp, format="%Y-%m-%d %H:%M:%S")
 X.square_feet = np.log1p(X.square_feet)
 if not test:
X.sort_values("timestamp", inplace=True)
 X.reset_index(drop=True, inplace=True)
 gc.collect()
holidays = ["2016-01-01", "2016-01-18", "2016-02-15", "2016-05-30", "2016-07-04",
"2016-09-05", "2016-10-10", "2016-11-11", "2016-11-24", "2016-12-26",
"2017-01-01", "2017-01-16", "2017-02-20", "2017-05-29", "2017-07-04",
"2017-09-04", "2017-10-09", "2017-11-10", "2017-11-23", "2017-12-25",
"2018-01-01", "2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04",
"2018-09-03", "2018-10-08", "2018-11-12", "2018-11-22", "2018-12-25",
"2019-01-01"]
X["hour"] = X.timestamp.dt.hour
X["weekday"] = X.timestamp.dt.weekday
X["is_holiday"] =(X.timestamp.dt.date.astype("str" ).isin(holidays)).astype(int)
 drop_features = ["timestamp", "sea_level_pressure", "wind_direction", "wind_speed"]
X.drop(drop_features, axis=1, inplace=True)
 if test:
row_ids = X.row_id
X.drop("row_id", axis=1, inplace=True)
 return X, row_ids
else:
y = np.log1p(X.meter_reading)
 X.drop("meter_reading", axis=1, inplace=True)
 return X, y
!pip install pywick<import_modules>
X_train, y_train = prepare_data(df_train, building, weather_train)
 del df_train, weather_train
gc.collect()
import torch
from sklearn import preprocessing
from torch.nn import functional as F
from torch import nn
from pytorch_lightning.core.lightning import LightningModule
from pywick.optimizers.nadam import Nadam
from torch.utils.data import TensorDataset, DataLoader
from torchvision import transforms
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint
from sklearn.metrics import mean_squared_error as mse<define_variables>
X_half_1 = X_train[:int(X_train.shape[0] / 2)]
X_half_2 = X_train[int(X_train.shape[0] / 2):]
y_half_1 = y_train[:int(X_train.shape[0] / 2)]
y_half_2 = y_train[int(X_train.shape[0] / 2):]
categorical_features = ["building_id", "site_id", "meter", "primary_use", "hour", "weekday"]
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)
 watchlist_1 = [d_half_1, d_half_2]
watchlist_2 = [d_half_2, d_half_1]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 40,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse"
}
print("Building model with first half and validating on second half:")
 model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)
 print("Building model with second half and validating on first half:")
 model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200 )
NUM_MODELS = 20
BATCH_SIZE = 9999
NUM_EPOCHS = 1500
PATH_MODEL = '/kaggle/working/models/'
PATH_DATA = '/kaggle/input/predict-volcanic-eruptions-ingv-oe/'<choose_model_class>
del X_train, y_train, X_half_1, X_half_2, y_half_1, y_half_2, d_half_1, d_half_2, watchlist_1, watchlist_2, df_fimp_1, df_fimp_2, df_fimp
gc.collect()
class VolcanicLSTM(LightningModule):
def __init__(self, num_features):
super().__init__()
self.bn = nn.BatchNorm1d(num_features=num_features)
 self.lstm = nn.LSTM(input_size=num_features, hidden_size=128, num_layers=1)
 self.conv1 = nn.Conv1d(in_channels=128, out_channels=128, kernel_size=2, padding=1, stride=2)
 self.conv2 = nn.Conv1d(in_channels=128, out_channels=84, kernel_size=2, padding=1, stride=2)
 self.conv3 = nn.Conv1d(in_channels=84, out_channels=64, kernel_size=2, padding=1, stride=2)
 self.flat = nn.Flatten()
self.lin1 = nn.Linear(in_features=64, out_features=64)
 self.lin2 = nn.Linear(in_features=64, out_features=32)
 self.lin3 = nn.Linear(in_features=32, out_features=1)
 def forward(self, x):
batch_size, _, _ = x.size()
x = x.view(batch_size, -1)
 x = self.bn(x)
 x = torch.unsqueeze(x, 1)
 x, _ = self.lstm(x)
 x = x.permute(0, 2, 1)
 x = self.conv1(x)
 x = F.relu(x)
 x = self.conv2(x)
 x = F.relu(x)
 x = self.conv3(x)
 x = F.relu(x)
 x = self.flat(x)
 x = self.lin1(x)
 x = F.relu(x)
 x = self.lin2(x)
 x = F.relu(x)
 x = self.lin3(x)
 x = F.relu(x)
 return x
def configure_optimizers(self):
return Nadam(self.parameters() , lr=0.005)
 def training_step(self, batch, batch_idx):
x, y = batch
preds = self(x)
 loss = F.l1_loss(preds, y)
 return loss
def validation_step(self, batch, batch_idx):
x, y = batch
preds = self(x)
 loss = F.l1_loss(preds, y)
 self.log('val_loss', loss)
 return loss
def train_dataloader(self):
tensor_x = torch.Tensor(train_x)
 tensor_y = torch.Tensor(train_y)
 dataset = TensorDataset(tensor_x, tensor_y)
 return DataLoader(dataset, batch_size=BATCH_SIZE)
 def val_dataloader(self):
tensor_x = torch.Tensor(valid_x)
 tensor_y = torch.Tensor(valid_y)
 dataset = TensorDataset(tensor_x, tensor_y)
 return DataLoader(dataset, batch_size=BATCH_SIZE)
 def test_dataloader(self):
tensor_x = torch.Tensor(test_scaled)
 dataset = TensorDataset(tensor_x)
 return DataLoader(dataset, batch_size=BATCH_SIZE )<split>
df_test = pd.read_csv(path_test)
 weather_test = pd.read_csv(path_weather_test)
 df_test = reduce_mem_usage(df_test)
 weather_test = reduce_mem_usage(weather_test)
 X_test, row_ids = prepare_data(df_test, building, weather_test, test=True )
submission = pd.read_csv(f'{PATH_DATA}/sample_submission.csv')
 sub_final = np.zeros(len(submission))
avg_valid_mae = 0
num_used_models = 0
i = 0
while i < NUM_MODELS:
print('
Running model ', i)
 train_x, valid_x, train_y, valid_y = train_test_split(train_sample, targets, test_size=0.2, random_state=0)
 train_x, valid_x, test_scaled = normalize(train_x.copy() , valid_x.copy() , test.copy() , 'min_max', [])
 train_x = train_x.values.reshape(train_x.shape[0], 1, train_x.shape[1])
 valid_x = valid_x.values.reshape(valid_x.shape[0], 1, valid_x.shape[1])
 train_y = train_y.to_numpy()
valid_y = valid_y.to_numpy()
test_scaled = test_scaled.values.reshape(test_scaled.shape[0], 1, test_scaled.shape[1])
 if not os.path.exists(f'{PATH_MODEL}/{str(i)}'):
os.makedirs(f'{PATH_MODEL}/{str(i)}')
 model = VolcanicLSTM(num_features=train_x.shape[-1])
 checkpoint_callback = ModelCheckpoint(
 monitor='val_loss',
dirpath=f'{PATH_MODEL}/{str(i)}',
filename='best_model-{epoch}' 
)
 trainer = Trainer(gpus=1, callbacks=[checkpoint_callback], min_epochs=1, max_epochs=NUM_EPOCHS, progress_bar_refresh_rate=0)
 trainer.fit(model)
 print(checkpoint_callback.best_model_path)
 test_model = VolcanicLSTM.load_from_checkpoint(checkpoint_callback.best_model_path, num_features=train_x.shape[-1])
 valid_preds = torch.squeeze(test_model(torch.Tensor(valid_x)) ).detach().numpy()
mae = mse(valid_y, valid_preds, squared=False)
 print(f'{i} MAE: {mae:.0f}')
 if mae < 10000000:
avg_valid_mae += mae
sub_final += torch.squeeze(test_model(torch.Tensor(test_scaled)) ).detach().numpy()
num_used_models += 1
i += 1<save_to_csv>
del df_test, building, weather_test
gc.collect()
avg_valid_mae /= num_used_models
sub_final /= num_used_models
print(f'
Number of used models: {num_used_models}')
 print(f'
Average validation MAE for used models: {avg_valid_mae:.0f}')
 submission['time_to_eruption'] = sub_final
submission.to_csv('submission.csv', index=False )<import_modules>
pred = np.expm1(model_half_1.predict(X_test, num_iteration=model_half_1.best_iteration)) / 2
del model_half_1
gc.collect()
pred += np.expm1(model_half_2.predict(X_test, num_iteration=model_half_2.best_iteration)) / 2
del model_half_2
gc.collect()
<load_from_csv><EOS> 
submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission.to_csv("submission.csv", index=False )