Kaggle competition: ASHRAE - Great Energy Predictor III
<SOS> metric: RMSLE Kaggle data source: ashrae-great-energy-predictor-iii<load_from_csv>
py.init_notebook_mode(connected=True)
%%time
target = []
row_id = []
for i in range(4):
data = pd.read_parquet(f'.. /input/bengaliai-cv19/test_image_data_{i}.parquet')
 dataset = DatasetRetriever(data)
 data_loader = DataLoader(dataset, batch_size=256, num_workers=4, shuffle=False, sampler=SequentialSampler(dataset))
for idx,(image_ids, images)in tqdm(enumerate(data_loader), total=len(data_loader)) :
roots, vowels, consonants = make_prediction(images)
 for image_id, root, vowel, consonant in zip(image_ids, roots, vowels, consonants):
row_id.append(image_id + '_consonant_diacritic')
 target.append(consonant)
 row_id.append(image_id + '_grapheme_root')
 target.append(root)
 row_id.append(image_id + '_vowel_diacritic')
 target.append(vowel )<save_to_csv>
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df
df_submission = pd.DataFrame(
 {
'row_id': row_id,
'target': target
},
columns=['row_id','target'] 
)
 df_submission.to_csv('submission.csv', index=False)
 df_submission.head(10 )<import_modules>
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 weather_train_df = pd.read_feather(root/'weather_train.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )
import torch
import numpy as np
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
from torchvision import transforms as torchtransforms
import cv2
import torch.nn as nn
from tqdm import tqdm_notebook as tqdm
import torchvision
import torch.nn as nn
import torch.nn.functional as F
import time<define_variables>
train_df['date'] = train_df['timestamp'].dt.date
train_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'] )
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print(device)
 modelpath = "/kaggle/input/se-resnext50-32x4d-fold2/se_resnext50_32x4d_fold2.pkl"
root_path="/kaggle/input/bengaliai-cv19"<categorify>
building_meta_df[building_meta_df.site_id == 0]
simple_transform_valid = torchtransforms.Compose([
torchtransforms.ToTensor() ,
torchtransforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
 ] )<choose_model_class>
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")' )
__all__ = ['SENet', 'se_resnext50_32x4d']
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):
super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AvgPool2d(7, stride=1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def se_resnext50_32x4d(num_classes=1000, pretrained='imagenet'):
model = SENet(SEResNeXtBottleneck, [3, 4, 6, 3], groups=32, reduction=16,
dropout_p=None, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model<load_pretrained>
debug = False
model = se_resnext50_32x4d(pretrained=None)
 model.avg_pool = nn.AdaptiveAvgPool2d(1)
 model.last_linear = nn.Linear(model.last_linear.in_features, 186)
 modelvalue = torch.load(modelpath, map_location='cuda:0')
 newmodelvalue = {}
for kv in modelvalue:
newmodelvalue[kv[4:]]=modelvalue[kv]
model.load_state_dict(newmodelvalue)
 model = model.to(device )<load_from_csv>
def preprocess(df):
df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df["month"] = df["timestamp"].dt.month
df["dayofweek"] = df["timestamp"].dt.dayofweek
allpathes=[]
allpreds_root = []
allpreds_vowel = []
allpreds_consonant = []
tAllBegin = time.time()
for i in range(4):
test_csv = pd.read_parquet(os.path.join(root_path, f'test_image_data_{i}.parquet'))
tBegin = time.time()
dataloaders = make_loader(data_folder = test_csv,
batch_size=8,
num_workers = 2,
is_shuffle = False)
 pathes, logit1, logit2, logit3 = getmodeleval(model, dataloaders)
 preds_root = np.argmax(logit1, axis=1)
 preds_vowel = np.argmax(logit2, axis=1)
 preds_consonant = np.argmax(logit3, axis=1)
 allpathes.extend(pathes)
 allpreds_root.extend(preds_root.tolist())
 allpreds_vowel.extend(preds_vowel.tolist())
 allpreds_consonant.extend(preds_consonant.tolist())
 tEnd = time.time()
print(i, int(round(tEnd * 1000)) - int(round(tBegin * 1000)) , "ms")
 tAllEnd = time.time()
print(len(allpathes), len(allpreds_root), len(allpreds_vowel), len(allpreds_consonant), int(round(tAllEnd * 1000)) - int(round(tAllBegin * 1000)) , "ms" )<load_from_csv>
preprocess(train_df )
row_id=[]
target=[]
for idx, image_id in enumerate(allpathes):
target.extend([allpreds_consonant[idx]])
 target.extend([allpreds_root[idx]])
 target.extend([allpreds_vowel[idx]])
 row_id.extend([str(image_id)+ '_consonant_diacritic'])
 row_id.extend([str(image_id)+ '_grapheme_root'])
 row_id.extend([str(image_id)+ '_vowel_diacritic'])
 submission_df = pd.read_csv(root_path + '/sample_submission.csv')
 submission_df.target = np.hstack(np.array(target ).astype(np.int))
submission_df.set_index('row_id', inplace=True)
 df_submit = pd.read_csv(".. /input/subm-0/submission_seresnext50-mixup-4cls-224.csv")
 df_submit.set_index('row_id', inplace=True)
 for name, row in df_submit.iterrows() :
submission_df.loc[name] = row
submission_df.reset_index(inplace=True)
 print(submission_df.head(10))
submission_df.to_csv('submission.csv', index=False)
 print("================end ======================" )<load_from_csv>
df_group = train_df.groupby('building_id')['meter_reading_log1p']
building_mean = df_group.mean().astype(np.float16)
 building_median = df_group.median().astype(np.float16)
 building_min = df_group.min().astype(np.float16)
 building_max = df_group.max().astype(np.float16)
 building_std = df_group.std().astype(np.float16)
 train_df['building_mean'] = train_df['building_id'].map(building_mean)
 train_df['building_median'] = train_df['building_id'].map(building_median)
 train_df['building_min'] = train_df['building_id'].map(building_min)
 train_df['building_max'] = train_df['building_id'].map(building_max)
 train_df['building_std'] = train_df['building_id'].map(building_std )
train = pd.read_csv(".. /input/train.csv",parse_dates=[2], low_memory=False)
 test = pd.read_csv(".. /input/test.csv",parse_dates=[3], low_memory=False)
 store = pd.read_csv(".. /input/store.csv", low_memory=False )<concatenate>
weather_train_df.isna().sum()
train.head().append(train.tail() )<concatenate>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
test.head().append(test.tail() )<count_missing_values>
weather_train_df = weather_train_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
display(train.isnull().sum() ,test.isnull().sum() ,store.isnull().sum() )<filter>
weather_train_df.groupby('site_id' ).apply(lambda group: group.isna().sum() )
test[pd.isnull(test.Open)]<filter>
def add_lag_feature(weather_df, window=3):
group_df = weather_df.groupby('site_id')
 cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']
rolled = group_df[cols].rolling(window=window, min_periods=0)
 lag_mean = rolled.mean().reset_index().astype(np.float16)
 lag_max = rolled.max().reset_index().astype(np.float16)
 lag_min = rolled.min().reset_index().astype(np.float16)
 lag_std = rolled.std().reset_index().astype(np.float16)
 for col in cols:
weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]
weather_df[f'{col}_max_lag{window}'] = lag_max[col]
weather_df[f'{col}_min_lag{window}'] = lag_min[col]
weather_df[f'{col}_std_lag{window}'] = lag_std[col]
store[pd.isnull(store.CompetitionDistance)]<data_type_conversions>
add_lag_feature(weather_train_df, window=3)
 add_lag_feature(weather_train_df, window=72 )
test.fillna(1, inplace=True)
<correct_missing_values>
primary_use_list = building_meta_df['primary_use'].unique()
primary_use_dict = {key: value for value, key in enumerate(primary_use_list)}
print('primary_use_dict: ', primary_use_dict)
 building_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)
 gc.collect()
store.fillna(0, inplace=True )<count_missing_values>
category_cols = ['building_id', 'site_id', 'primary_use']
feature_cols = ['square_feet', 'year_built'] + [
'hour', 'weekend',
'building_median'] + [
'air_temperature', 'cloud_coverage',
'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',
'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',
'air_temperature_max_lag72', 'air_temperature_min_lag72',
'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',
'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',
'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',
'wind_speed_mean_lag72', 'air_temperature_mean_lag3',
'air_temperature_max_lag3',
'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',
'dew_temperature_mean_lag3',
'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',
'wind_direction_mean_lag3', 'wind_speed_mean_lag3']
display(train.isnull().sum() ,test.isnull().sum() ,store.isnull().sum() )<merge>
weather_train_df['site_id'].nunique()
train = pd.merge(train, store, on='Store')
 test = pd.merge(test, store, on='Store' )<sort_values>
def create_X_y(train_df, target_meter, site_id):
target_train_df = train_df[train_df['meter'] == target_meter]
target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')
 target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')
 target_train_df = target_train_df[target_train_df['site_id'] == site_id]
X_train = target_train_df[feature_cols + category_cols]
y_train = target_train_df['meter_reading_log1p'].values
del target_train_df
return X_train, y_train
train = train.sort_values(['Date'],ascending = False)
 ho_test = train[:6*7*1115]
ho_train = train[6*7*1115:]<filter>
def fit_lgbm(train, val, devices=(-1,), seed=None, cat_features=None, num_rounds=1500, lr=0.1, bf=0.1):

 X_train, y_train = train
X_valid, y_valid = val
metric = 'l2'
params = {'num_leaves': 31,
'objective': 'regression',
'learning_rate': lr,
"boosting": "gbdt",
"bagging_freq": 5,
"bagging_fraction": bf,
"feature_fraction": 0.9,
"metric": metric,
}
device = devices[0]
if device == -1:
pass
else:
print(f'using gpu device_id {device}...')
 params.update({'device': 'gpu', 'gpu_device_id': device})
 params['seed'] = seed
early_stop = 20
verbose_eval = 20
d_train = lgb.Dataset(X_train, label=y_train, categorical_feature=cat_features)
 d_valid = lgb.Dataset(X_valid, label=y_valid, categorical_feature=cat_features)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 y_pred_valid = model.predict(X_valid, num_iteration=model.best_iteration)
 print('best_score', model.best_score)
 log = {'train/mae': model.best_score['training']['l2'],
'valid/mae': model.best_score['valid_1']['l2']}
return model, y_pred_valid, log
ho_test = ho_test[ho_test["Open"] != 0]
ho_test = ho_test[ho_test["Sales"] > 0]
ho_train = ho_train[ho_train["Open"] != 0]
ho_train = ho_train[ho_train["Sales"] > 0]<feature_engineering>
folds = 5
seed = 666
shuffle = False
kf = KFold(n_splits=folds, shuffle=shuffle, random_state=seed )
def features_create(data):
mappings = {'0':0, 'a':1, 'b':2, 'c':3, 'd':4}
data.StoreType.replace(mappings, inplace=True)
 data.Assortment.replace(mappings, inplace=True)
 data.StateHoliday.replace(mappings, inplace=True)
 data['Year'] = data.Date.dt.year
data['Month'] = data.Date.dt.month
data['Day'] = data.Date.dt.day
data['DayOfWeek'] = data.Date.dt.dayofweek
data['WeekOfYear'] = data.Date.dt.weekofyear
data['CompetitionOpen'] = 12 *(data.Year - data.CompetitionOpenSinceYear)+(data.Month - data.CompetitionOpenSinceMonth)
 data['PromoOpen'] = 12 *(data.Year - data.Promo2SinceYear)+(data.WeekOfYear - data.Promo2SinceWeek)/ 4.0
data['CompetitionOpen'] = data.CompetitionOpen.apply(lambda x: x if x > 0 else 0)
 data['PromoOpen'] = data.PromoOpen.apply(lambda x: x if x > 0 else 0)
 month2str = {1:'Jan', 2:'Feb', 3:'Mar', 4:'Apr', 5:'May', 6:'Jun', 7:'Jul', 8:'Aug', 9:'Sept', 10:'Oct', 11:'Nov', 12:'Dec'}
data['monthStr'] = data.Month.map(month2str)
 data.loc[data.PromoInterval == 0, 'PromoInterval'] = ''
data['IsPromoMonth'] = 0
for interval in data.PromoInterval.unique() :
if interval != '':
for month in interval.split(','):
data.loc[(data.monthStr == month)&(data.PromoInterval == interval), 'IsPromoMonth'] = 1
return data<feature_engineering>
features_create(ho_train)
 features_create(ho_test)
 features_create(test)
 print('Features creation finished' )<drop_column>
ho_train.drop(['Date','Customers','Open','PromoInterval','monthStr'],axis=1,inplace =True)
 ho_test.drop(['Date','Customers','Open','PromoInterval','monthStr'],axis=1,inplace =True )<prepare_x_and_y>
with open('.. /input/lgbm-by-meter-type-and-site-id/models.pickle', 'rb')as handle:
models = pickle.load(handle )
ho_xtrain = ho_train.drop(['Sales'],axis=1)
 ho_ytrain = np.log1p(ho_train.Sales)
 ho_xtest = ho_test.drop(['Sales'],axis=1)
 ho_ytest = np.log1p(ho_test.Sales )<drop_column>
print('loading...')
 test_df = pd.read_feather(root/'test.feather')
 weather_test_df = pd.read_feather(root/'weather_test.feather')
 print('preprocessing building...')
 test_df['date'] = test_df['timestamp'].dt.date
preprocess(test_df)
 test_df['building_mean'] = test_df['building_id'].map(building_mean)
 test_df['building_median'] = test_df['building_id'].map(building_median)
 test_df['building_min'] = test_df['building_id'].map(building_min)
 test_df['building_max'] = test_df['building_id'].map(building_max)
 test_df['building_std'] = test_df['building_id'].map(building_std)
 print('preprocessing weather...')
 weather_test_df = weather_test_df.groupby('site_id' ).apply(lambda group: group.interpolate(limit_direction='both'))
weather_test_df.groupby('site_id' ).apply(lambda group: group.isna().sum())
 add_lag_feature(weather_test_df, window=3)
 add_lag_feature(weather_test_df, window=72)
 print('reduce mem usage...')
 reduce_mem_usage(test_df, use_float16=True)
 reduce_mem_usage(weather_test_df, use_float16=True)
 gc.collect()
xtest =test.drop(['Id','Date','Open','PromoInterval','monthStr'],axis = 1 )<compute_test_metric>
with open('.. /input/make-test-tuples/test_tuples.pickle', 'rb')as handle:
test_tuples = pickle.load(handle )
def rmspe(y, yhat):
return np.sqrt(np.mean(( yhat/y-1)** 2))
def rmspe_xg(yhat, y):
y = np.expm1(y.get_label())
 yhat = np.expm1(yhat)
 return "rmspe", rmspe(y,yhat )<train_model>
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
reduce_mem_usage(sample_submission )
params = {"objective": "reg:linear",
"booster" : "gbtree",
"eta": 0.03,
"max_depth": 10,
"subsample": 0.9,
"colsample_bytree": 0.7,
"silent": 1,
"seed": 10
}
num_boost_round = 6000
dtrain = xgb.DMatrix(ho_xtrain, ho_ytrain)
 dvalid = xgb.DMatrix(ho_xtest, ho_ytest)
 watchlist = [(dtrain, 'train'),(dvalid, 'eval')]
print("Train a XGBoost model")
 start = time()
gbm = xgb.train(params, dtrain, num_boost_round, evals=watchlist,
early_stopping_rounds=100, feval=rmspe_xg, verbose_eval=True)
 end = time()
print('Training time is {:2f} s.'.format(end-start))
print("validating")
 ho_xtest.sort_index(inplace=True)
 ho_ytest.sort_index(inplace=True)
 yhat = gbm.predict(xgb.DMatrix(ho_xtest))
error = rmspe(np.expm1(ho_ytest), np.expm1(yhat))
print('RMSPE: {:.6f}'.format(error))<merge>
def create_X(test_df, target_meter, site_id):
target_test_df = test_df[test_df['meter'] == target_meter]
target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')
 target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')
 target_test_df = target_test_df[target_test_df['site_id']==site_id]
X_test = target_test_df[feature_cols + category_cols]
return X_test, target_test_df['row_id']
res = pd.DataFrame(data = ho_ytest)
 res['Prediction']=yhat
res = pd.merge(ho_xtest,res, left_index= True, right_index=True)
 res['Ratio'] = res.Prediction/res.Sales
res['Error'] =abs(res.Ratio-1)
 res['Weight'] = res.Sales/res.Prediction
res.head()<create_dataframe>
def pred(X_test, models, batch_size=1000000):
iterations =(X_test.shape[0] + batch_size -1)// batch_size
print('iterations', iterations)
 y_test_pred_total = np.zeros(X_test.shape[0])
 for i, model in enumerate(models):
print(f'predicting {i}-th model')
 for k in tqdm(range(iterations)) :
y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size], num_iteration=model.best_iteration)
 y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test
y_test_pred_total /= len(models)
 return y_test_pred_total
col_1 = ['Sales','Prediction']
col_2 = ['Ratio']
L=np.random.randint(low=1,high = 1115, size = 3)
 print('Mean Ratio of predition and real sales data is {}: store all'.format(res.Ratio.mean()))
for i in L:
s1 = pd.DataFrame(res[res['Store']==i],columns = col_1)
 s2 = pd.DataFrame(res[res['Store']==i],columns = col_2)
 s1.plot(title = 'Comparation of predition and real sales data: store {}'.format(i),figsize=(12,4))
s2.plot(title = 'Ratio of predition and real sales data: store {}'.format(i),figsize=(12,4))
print('Mean Ratio of predition and real sales data is {}: store {}'.format(s2.Ratio.mean() ,i)) 
<sort_values>
y_tests = np.empty(( 4,16), dtype=list)
 y_tests_locs = np.empty(( 4,16), dtype=list )
res.sort_values(['Error'],ascending=False,inplace= True)
 res[:10]<compute_test_metric>
for tpl in test_tuples:
site_id = tpl[0]
target_meter = tpl[1]
sample_submission.loc[y_tests_locs[target_meter, site_id], 'meter_reading'] = \
np.expm1(y_tests[target_meter, site_id])
L=range(1115)
 W_ho=[]
W_test=[]
for i in L:
s1 = pd.DataFrame(res[res['Store']==i+1],columns = col_1)
 s2 = pd.DataFrame(xtest[xtest['Store']==i+1])
 W1=[(0.990+(i/1000)) for i in range(20)]
S =[]
for w in W1:
error = rmspe(np.expm1(s1.Sales), np.expm1(s1.Prediction*w))
S.append(error)
 Score = pd.Series(S,index=W1)
 BS = Score[Score.values == Score.values.min() ]
a=np.array(BS.index.values)
 b_ho=a.repeat(len(s1))
b_test=a.repeat(len(s2))
W_ho.extend(b_ho.tolist())
 W_test.extend(b_test.tolist() )<compute_test_metric>
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )