Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<normalization>
DATA_PATH = ".. /input/ashrae-energy-prediction/"
def add_gaussian_noise(x, sigma):
x += np.random.randn(*x.shape)* sigma
x = np.clip(x, 0., 1.)
return x
class Transform:
def __init__(self, affine=True, crop=True, size=(64, 64),
normalize=True, train=True, threshold=40.,
sigma=-1.) :
self.affine = affine
self.crop = crop
self.size = size
self.normalize = normalize
self.train = train
self.threshold = threshold / 255.
 self.sigma = sigma / 255.
 def __call__(self, example):
if self.train:
x, y = example
else:
x = example
if self.affine:
x = affine_image(x)
 if self.crop:
x = crop_char_image(x, threshold=self.threshold)
 if self.size is not None:
x = resize(x, size=self.size)
 if self.sigma > 0.:
x = add_gaussian_noise(x, sigma=self.sigma)
 if self.normalize:
x =(x.astype(np.float32)- 0.0692)/ 0.2051
if x.ndim == 2:
x = x[None, :, :]
x = x.astype(np.float32)
 if self.train:
y = y.astype(np.int64)
 return x, y
else:
return x<concatenate>
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )
def residual_add(lhs, rhs):
lhs_ch, rhs_ch = lhs.shape[1], rhs.shape[1]
if lhs_ch < rhs_ch:
out = lhs + rhs[:, :lhs_ch]
elif lhs_ch > rhs_ch:
out = torch.cat([lhs[:, :rhs_ch] + rhs, lhs[:, rhs_ch:]], dim=1)
 else:
out = lhs + rhs
return out 
<import_modules>
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df
class LazyLoadModule(nn.Module):

 _lazy_buffer_keys: List[str] = []
_lazy_parameter_keys: List[str] = []
def __init__(self):
super(LazyLoadModule, self ).__init__()
for k in self._lazy_buffer_keys:
self.register_buffer(k, torch.tensor([]))
for k in self._lazy_parameter_keys:
self.register_parameter(k, None)
 self._register_load_state_dict_pre_hook(self._hook)
 def _hook(self, state_dict, prefix, local_metadata, strict, missing_keys,
unexpected_keys, error_msgs):
for key in self._lazy_buffer_keys:
self.register_buffer(key, state_dict[prefix + key])
 for key in self._lazy_parameter_keys:
self.register_parameter(key, Parameter(state_dict[prefix + key])) 
<import_modules>
weather_df = fill_weather_dataset(weather_df )
class LazyLinear(LazyLoadModule):

 __constants__ = ['bias', 'in_features', 'out_features']
_lazy_parameter_keys = ['weight']
def __init__(self, in_features, out_features, bias=True):
super(LazyLinear, self ).__init__()
self.in_features = in_features
self.out_features = out_features
if bias:
self.bias = Parameter(torch.Tensor(out_features))
else:
self.register_parameter('bias', None)
 if in_features is not None:
self.weight = Parameter(torch.Tensor(out_features, in_features))
self.reset_parameters()
def reset_parameters(self):
init.kaiming_uniform_(self.weight, a=math.sqrt(5))
if self.bias is not None:
fan_in, _ = init._calculate_fan_in_and_fan_out(self.weight)
 bound = 1 / math.sqrt(fan_in)
 init.uniform_(self.bias, -bound, bound)
 def forward(self, input):
if self.weight is None:
self.in_features = input.shape[-1]
self.weight = Parameter(torch.Tensor(self.out_features, self.in_features))
self.reset_parameters()
self.to(input.device)
 return F.linear(input, self.weight, self.bias)
 def extra_repr(self):
return 'in_features={}, out_features={}, bias={}'.format(
 self.in_features, self.out_features, self.bias is not None 
)
<choose_model_class>
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )
class LinearBlock(nn.Module):
def __init__(self, in_features, out_features, bias=True,
use_bn=True, activation=F.relu, dropout_ratio=-1, residual=False,):
super(LinearBlock, self ).__init__()
if in_features is None:
self.linear = LazyLinear(in_features, out_features, bias=bias)
 else:
self.linear = nn.Linear(in_features, out_features, bias=bias)
 if use_bn:
self.bn = nn.BatchNorm1d(out_features)
 if dropout_ratio > 0.:
self.dropout = nn.Dropout(p=dropout_ratio)
 else:
self.dropout = None
self.activation = activation
self.use_bn = use_bn
self.dropout_ratio = dropout_ratio
self.residual = residual
def __call__(self, x):
h = self.linear(x)
 if self.use_bn:
h = self.bn(h)
 if self.activation is not None:
h = self.activation(h)
 if self.residual:
h = residual_add(h, x)
 if self.dropout_ratio > 0:
h = self.dropout(h)
 return h<import_modules>
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()
class PretrainedCNN(nn.Module):
def __init__(self, model_name='se_resnext101_32x4d',
in_channels=1, out_dim=10, use_bn=True,
pretrained=None):
super(PretrainedCNN, self ).__init__()
self.conv0 = nn.Conv2d(
 in_channels, 3, kernel_size=3, stride=1, padding=1, bias=True)
 self.base_model = pretrainedmodels.__dict__[model_name](pretrained=pretrained)
 activation = F.leaky_relu
self.do_pooling = True
if self.do_pooling:
inch = self.base_model.last_linear.in_features
else:
inch = None
hdim = 512
lin1 = LinearBlock(inch, hdim, use_bn=use_bn, activation=activation, residual=False)
 lin2 = LinearBlock(hdim, out_dim, use_bn=use_bn, activation=None, residual=False)
 self.lin_layers = Sequential(lin1, lin2)
 def forward(self, x):
h = self.conv0(x)
 h = self.base_model.features(h)
 if self.do_pooling:
h = torch.sum(h, dim=(-1, -2))
else:
bs, ch, height, width = h.shape
h = h.view(bs, ch*height*width)
 for layer in self.lin_layers:
h = layer(h)
 return h<train_model>
train_df = features_engineering(train_df )
def accuracy(y, t):
pred_label = torch.argmax(y, dim=1)
 count = pred_label.shape[0]
correct =(pred_label == t ).sum().type(torch.float32)
 acc = correct / count
return acc
class BengaliClassifier(nn.Module):
def __init__(self, predictor, n_grapheme=168, n_vowel=11, n_consonant=7):
super(BengaliClassifier, self ).__init__()
self.n_grapheme = n_grapheme
self.n_vowel = n_vowel
self.n_consonant = n_consonant
self.n_total_class = self.n_grapheme + self.n_vowel + self.n_consonant
self.predictor = predictor
self.metrics_keys = [
'loss', 'loss_grapheme', 'loss_vowel', 'loss_consonant',
'acc_grapheme', 'acc_vowel', 'acc_consonant']
def forward(self, x, y=None):
pred = self.predictor(x)
 if isinstance(pred, tuple):
assert len(pred)== 3
preds = pred
else:
assert pred.shape[1] == self.n_total_class
preds = torch.split(pred, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 loss_grapheme = F.cross_entropy(preds[0], y[:, 0])
 loss_vowel = F.cross_entropy(preds[1], y[:, 1])
 loss_consonant = F.cross_entropy(preds[2], y[:, 2])
 loss = loss_grapheme + loss_vowel + loss_consonant
metrics = {
'loss': loss.item() ,
'loss_grapheme': loss_grapheme.item() ,
'loss_vowel': loss_vowel.item() ,
'loss_consonant': loss_consonant.item() ,
'acc_grapheme': accuracy(preds[0], y[:, 0]),
'acc_vowel': accuracy(preds[1], y[:, 1]),
'acc_consonant': accuracy(preds[2], y[:, 2]),
}
return loss, metrics, pred
def calc(self, data_loader):
device: torch.device = next(self.parameters() ).device
self.eval()
output_list = []
with torch.no_grad() :
for batch in tqdm(data_loader):
batch = batch.to(device)
 pred = self.predictor(batch)
 output_list.append(pred)
 output = torch.cat(output_list, dim=0)
 preds = torch.split(output, [self.n_grapheme, self.n_vowel, self.n_consonant], dim=1)
 return preds
def predict_proba(self, data_loader):
preds = self.calc(data_loader)
 return [F.softmax(p, dim=1)for p in preds]
def predict(self, data_loader):
preds = self.calc(data_loader)
 pred_labels = [torch.argmax(p, dim=1)for p in preds]
return pred_labels 
<choose_model_class>
target = np.log1p(train_df["meter_reading"])
 features = train_df.drop('meter_reading', axis = 1)
 del train_df
gc.collect()
def build_predictor(arch, out_dim, model_name=None):
if arch == 'pretrained':
predictor = PretrainedCNN(in_channels=1, out_dim=out_dim, model_name=model_name)
 else:
raise ValueError("[ERROR] Unexpected value arch={}".format(arch))
return predictor
def build_classifier(arch, load_model_path, n_total, model_name='', device='cuda:0'):
if isinstance(device, str):
device = torch.device(device)
 predictor = build_predictor(arch, out_dim=n_total, model_name=model_name)
 print('predictor', type(predictor))
classifier = BengaliClassifier(predictor)
 if load_model_path:
predictor.load_state_dict(torch.load(load_model_path))
else:
print("[WARNING] Unexpected value load_model_path={}" 
.format(load_model_path))
classifier.to(device)
 return classifier 
<load_pretrained>
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend"]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(features):
train_features = features.loc[train_index]
train_target = target.loc[train_index]
test_features = features.loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()
def prepare_image(datadir, featherdir, data_type='train',
submission=False, indices=[0, 1, 2, 3]):
assert data_type in ['train', 'test']
if submission:
image_df_list = [pd.read_parquet(datadir / f'{data_type}_image_data_{i}.parquet')
 for i in indices]
else:
image_df_list = [pd.read_feather(featherdir / f'{data_type}_image_data_{i}.feather')
 for i in indices]
print('image_df_list', len(image_df_list))
HEIGHT = 137
WIDTH = 236
images = [df.iloc[:, 1:].values.reshape(-1, HEIGHT, WIDTH)for df in image_df_list]
del image_df_list
gc.collect()
images = np.concatenate(images, axis=0)
 return images 
<predict_on_test>
del features, target
gc.collect()
def predict_core(test_images, image_size, threshold,
arch, n_total, model_name, load_model_path, batch_size=512, device='cuda:0', **kwargs):
classifier = build_classifier(arch, load_model_path, n_total, model_name, device=device)
 test_dataset = BengaliAIDataset(
 test_images, None,
transform=Transform(affine=False, crop=True, size=(image_size, image_size),
threshold=threshold, train=False))
print('test_dataset', len(test_dataset))
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
 test_pred_proba = classifier.predict_proba(test_loader)
 return test_pred_proba 
<define_variables>
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )
device = torch.device(device)
 n_grapheme = 168
n_vowel = 11
n_consonant = 7
n_total = n_grapheme + n_vowel + n_consonant
print('n_total', n_total )<define_variables>
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()
traindir = '/kaggle/input/bengaliaicv19-trainedmodels/'
data_type = 'test'
test_preds_list = []
for i in range(4):
indices = [i]
test_images = prepare_image(
 datadir, featherdir, data_type=data_type, submission=submission, indices=indices)
 n_dataset = len(test_images)
 print(f'n_dataset={n_dataset}')
 model_preds_list = []
for j in range(4):
train_args_dict = load_json(os.path.join(traindir, f'args_{j}.json'))
train_args_dict.update({
'load_model_path': os.path.join(traindir, f'predictor_{j}.pt'),
'device': device,
'batch_size': batch_size,
'debug': debug,
})
 print(f'j {j} updated train_args_dict {train_args_dict}')
 test_preds = predict_core(
 test_images=test_images, n_total=n_total,
**train_args_dict)
 model_preds_list.append(test_preds)
 proba0 = torch.mean(torch.stack([test_preds[0] for test_preds in model_preds_list], dim=0), dim=0)
 proba1 = torch.mean(torch.stack([test_preds[1] for test_preds in model_preds_list], dim=0), dim=0)
 proba2 = torch.mean(torch.stack([test_preds[2] for test_preds in model_preds_list], dim=0), dim=0)
 p0 = torch.argmax(proba0, dim=1 ).cpu().numpy()
p1 = torch.argmax(proba1, dim=1 ).cpu().numpy()
p2 = torch.argmax(proba2, dim=1 ).cpu().numpy()
print('p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 test_preds_list.append([p0, p1, p2])
 if debug:
break
del test_images
gc.collect()<concatenate>
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df)
p0 = np.concatenate([test_preds[0] for test_preds in test_preds_list], axis=0)
 p1 = np.concatenate([test_preds[1] for test_preds in test_preds_list], axis=0)
 p2 = np.concatenate([test_preds[2] for test_preds in test_preds_list], axis=0)
 print('concat:', 'p0', p0.shape, 'p1', p1.shape, 'p2', p2.shape)
 row_id = []
target = []
for i in tqdm(range(len(p0))):
row_id += [f'Test_{i}_grapheme_root', f'Test_{i}_vowel_diacritic',
f'Test_{i}_consonant_diacritic']
target += [p0[i], p1[i], p2[i]]
submission_df = pd.DataFrame({'row_id': row_id, 'target': target})
 submission_df.to_csv('submission.csv', index=False)
<load_from_csv>
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()
train = pd.read_csv(datadir/'train.csv' )<create_dataframe>
test_df = features_engineering(test_df )
pred_df = pd.DataFrame({
'grapheme_root': p0,
'vowel_diacritic': p1,
'consonant_diacritic': p2
} )<define_variables>
results = []
for model in models:
if results == []:
results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 else:
results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)
 del model
gc.collect()
train_labels = train[['grapheme_root', 'vowel_diacritic', 'consonant_diacritic']].values<data_type_conversions>
del test_df, models
gc.collect()
<predict_on_test><EOS> 
results_df = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
results_df.to_csv("submission.csv", index=False)