Kaggle competition: M5 Forecasting - Accuracy
<SOS> metric: M5_WRMSSE Kaggle data source: m5-forecasting-accuracy<feature_engineering>
from datetime import datetime, timedelta
import gc
import numpy as np, pandas as pd
import lightgbm as lgb
concat_sub['is_iceberg_base'] = sub_base['is_iceberg']
concat_sub['is_iceberg'] = np.where(np.all(concat_sub.iloc[:,1:6] > cutoff_lo, axis=1),
concat_sub['is_iceberg_max'],
np.where(np.all(concat_sub.iloc[:,1:6] < cutoff_hi, axis=1),
concat_sub['is_iceberg_min'],
concat_sub['is_iceberg_base']))
concat_sub[['id', 'is_iceberg']].to_csv('stack_minmax_bestbase.csv',
index=False, float_format='%.6f' )<categorify>
CAL_DTYPES={"event_name_1": "category",
"event_name_2": "category",
"event_type_1": "category",
"event_type_2": "category",
"weekday": "category",
'wm_yr_wk': 'int16',
"wday": "int16",
"month": "int16",
"year": "int16",
"snap_CA": "float32",
'snap_TX': 'float32',
'snap_WI': 'float32' }
PRICE_DTYPES = {"store_id": "category",
"item_id": "category",
"wm_yr_wk": "int16",
"sell_price":"float32" }
def eval_wrapper(yhat, y):
y = np.array(y)
 y = y.astype(int)
 yhat = np.array(yhat)
 yhat = np.clip(np.round(yhat), np.min(y), np.max(y)).astype(int)
 return quadratic_weighted_kappa(yhat, y)
 def get_params() :
params = {}
params["objective"] = "reg:linear"
params["eta"] = 0.05
params["min_child_weight"] = 240
params["subsample"] = 0.9
params["colsample_bytree"] = 0.67
params["silent"] = 1
params["max_depth"] = 6
plst = list(params.items())
 return plst
def apply_offset(data, bin_offset, sv, scorer=eval_wrapper):
data[1, data[0].astype(int)==sv] = data[0, data[0].astype(int)==sv] + bin_offset
score = scorer(data[1], data[2])
 return score
columns_to_drop = ['Id', 'Response', 'Medical_History_10','Medical_History_24']
xgb_num_rounds = 700
num_classes = 8
eta_list = [0.05] * 200
eta_list = eta_list + [0.02] * 500
print("Load the data using pandas")
 train = pd.read_csv(".. /input/train.csv")
 test = pd.read_csv(".. /input/test.csv")
 all_data = train.append(test)
 all_data['Product_Info_2_char'] = all_data.Product_Info_2.str[0]
all_data['Product_Info_2_num'] = all_data.Product_Info_2.str[1]
all_data['Product_Info_2'] = pd.factorize(all_data['Product_Info_2'])[0]
all_data['Product_Info_2_char'] = pd.factorize(all_data['Product_Info_2_char'])[0]
all_data['Product_Info_2_num'] = pd.factorize(all_data['Product_Info_2_num'])[0]
all_data['BMI_Age'] = all_data['BMI'] * all_data['Ins_Age']
med_keyword_columns = all_data.columns[all_data.columns.str.startswith('Medical_Keyword_')]
all_data['Med_Keywords_Count'] = all_data[med_keyword_columns].sum(axis=1)
 print('Eliminate missing values')
 all_data.fillna(-1, inplace=True)
 all_data['Response'] = all_data['Response'].astype(int)
 train = all_data[all_data['Response']>0].copy()
test = all_data[all_data['Response']<1].copy()
test['Response'] = test['Response']-1
xgtrain = xgb.DMatrix(train.drop(columns_to_drop, axis=1), train['Response'].values)
 xgtest = xgb.DMatrix(test.drop(columns_to_drop, axis=1), label=test['Response'].values)
 plst = get_params()
print(plst)
 model = xgb.train(plst, xgtrain, xgb_num_rounds, learning_rates=eta_list)
 train_preds = model.predict(xgtrain, ntree_limit=model.best_iteration)
 print('Train score is:', eval_wrapper(train_preds, train['Response']))
test_preds = model.predict(xgtest, ntree_limit=model.best_iteration)
 train_preds = np.clip(train_preds, 1, 8)
 test_preds = np.clip(test_preds, 1, 8)
 offsets = np.array([0.1, -1, -2, -1, -0.8, 0.02, 0.8, 1])
 data = np.vstack(( train_preds, train_preds, train['Response'].values))
for j in range(num_classes):
data[1, data[0].astype(int)==j] = data[0, data[0].astype(int)==j] + offsets[j]
for j in range(num_classes):
train_offset = lambda x: -apply_offset(data, x, j)
 offsets[j] = fmin_powell(train_offset, offsets[j])
 data = np.vstack(( test_preds, test_preds, test['Response'].values))
for j in range(num_classes):
data[1, data[0].astype(int)==j] = data[0, data[0].astype(int)==j] + offsets[j]
final_test_preds = np.round(np.clip(data[1], 1, 8)).astype(int)
 xgtest = xgb.DMatrix(test.drop(columns_to_drop, axis=1), label=final_test_preds)
 test_preds2 = model.predict(xgtest, ntree_limit=model.best_iteration)
 test_preds2 = np.clip(test_preds2, 1, 8)
 data = np.vstack(( test_preds2, test_preds2, test['Response'].values))
for j in range(num_classes):
data[1, data[0].astype(int)==j] = data[0, data[0].astype(int)==j] + offsets[j]
final_test_preds = np.round(np.clip(data[1], 1, 8)).astype(int)
 preds_out = pd.DataFrame({"Id": test['Id'].values, "Response": final_test_preds})
 preds_out = preds_out.set_index('Id')
 preds_out.to_csv('xgb_offset_submission.csv' )<load_from_disk>
pd.options.display.max_columns = 50
train_df=pd.read_json('.. /input/train.json')
 test_df=pd.read_json('.. /input/test.json' )<feature_engineering>
NDAYS = 28
MAX_LAGS = 57
TRAINING_LAST = 1913
FDAY = datetime(2016,4, 25)
 FDAY
train_df["price_t"] =train_df["price"]/train_df["bedrooms"]
test_df["price_t"] = test_df["price"]/test_df["bedrooms"]
train_df["room_sum"] = train_df["bedrooms"]+train_df["bathrooms"]
test_df["room_sum"] = test_df["bedrooms"]+test_df["bathrooms"]
train_df["num_photos"] = train_df["photos"].apply(len)
 test_df["num_photos"] = test_df["photos"].apply(len)
 train_df["num_features"] = train_df["features"].apply(len)
 test_df["num_features"] = test_df["features"].apply(len)
 train_df["num_description_words"] = train_df["description"].apply(lambda x: len(x.split(" ")))
 test_df["num_description_words"] = test_df["description"].apply(lambda x: len(x.split(" ")))
 features_to_use=["bathrooms", "bedrooms", "latitude", "longitude", "price","price_t","num_photos", "num_features", "num_description_words","listing_id"]<train_model>
def create_calendar_df() :
calendar_df = pd.read_csv(".. /input/m5-forecasting-accuracy/calendar.csv", dtype = CAL_DTYPES)
 calendar_df["date"] = pd.to_datetime(calendar_df["date"])
 for col, col_dtype in CAL_DTYPES.items() :
if col_dtype == "category":
calendar_df[col] = calendar_df[col].cat.codes.astype("int16")
 calendar_df[col] -= calendar_df[col].min()
return calendar_df
def create_prices_df() :
prices_df = pd.read_csv(".. /input/m5-forecasting-accuracy/sell_prices.csv", dtype = PRICE_DTYPES)
 for col, col_dtype in PRICE_DTYPES.items() :
if col_dtype == "category":
prices_df[col] = prices_df[col].cat.codes.astype("int16")
 prices_df[col] -= prices_df[col].min()
return prices_df
def create_dt(is_train = True, nrows = None, first_day = 1200):
start_day = max(1 if is_train else TRAINING_LAST - MAX_LAGS, first_day)
 numeric_cols = [f"d_{day}" for day in range(start_day, TRAINING_LAST + 1)]
category_cols = ['id', 'item_id', 'dept_id','store_id', 'cat_id', 'state_id']
dtype = {numcol:"float32" for numcol in numeric_cols}
dtype.update({col: "category" for col in category_cols if col != "id"})
 sales_df = pd.read_csv(".. /input/m5-forecasting-accuracy/sales_train_validation.csv",
nrows = nrows, usecols = category_cols + numeric_cols, dtype = dtype)
 for col in category_cols:
if col != "id":
sales_df[col] = sales_df[col].cat.codes.astype("int16")
 sales_df[col] -= sales_df[col].min()
if not is_train:
for day in range(TRAINING_LAST + 1, TRAINING_LAST + NDAYS +1):
sales_df[f"d_{day}"] = np.nan
sales_df = pd.melt(sales_df,
id_vars = category_cols,
value_vars = [col for col in sales_df.columns if col.startswith("d_")],
var_name = "d",
value_name = "sales")
 calendar_df = create_calendar_df()
prices_df = create_prices_df()
sales_df = sales_df.merge(calendar_df, on= "d", copy = False)
 sales_df = sales_df.merge(prices_df, on = ["store_id", "item_id", "wm_yr_wk"], copy = False)
 return sales_df
def runXGB(train_X, train_y, test_X, test_y=None, feature_names=None, seed_val=0, num_rounds=1000):
param = {}
param['objective'] = 'multi:softprob'
param['eta'] = 0.03
param['max_depth'] = 6
param['silent'] = 1
param['num_class'] = 3
param['eval_metric'] = "mlogloss"
param['min_child_weight'] = 1
param['subsample'] = 0.7
param['colsample_bytree'] = 0.7
param['seed'] = seed_val
num_rounds = num_rounds
plst = list(param.items())
 xgtrain = xgb.DMatrix(train_X, label=train_y)
 if test_y is not None:
xgtest = xgb.DMatrix(test_X, label=test_y)
 watchlist = [(xgtrain,'train'),(xgtest, 'test')]
model = xgb.train(plst, xgtrain, num_rounds, watchlist, early_stopping_rounds=20)
 else:
xgtest = xgb.DMatrix(test_X)
 model = xgb.train(plst, xgtrain, num_rounds)
 pred_test_y = model.predict(xgtest)
 return pred_test_y, model<categorify>
FIRST_DAY = 350
categorical = ["display_address", "manager_id", "building_id", "street_address"]
for f in categorical:
if train_df[f].dtype=='object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[f].values)+ list(test_df[f].values))
train_df[f] = lbl.transform(list(train_df[f].values))
test_df[f] = lbl.transform(list(test_df[f].values))
features_to_use.append(f )<feature_engineering>
def create_features(dt):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags ]
for lag, lag_col in zip(lags, lag_cols):
dt[lag_col] = dt[["id","sales"]].groupby("id")["sales"].shift(lag)
 wins = [7, 28]
for win in wins :
for lag,lag_col in zip(lags, lag_cols):
dt[f"rmean_{lag}_{win}"] = dt[["id", lag_col]].groupby("id")[lag_col].transform(lambda x : x.rolling(win ).mean())
 date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(dt["date"].dt, date_feat_func ).astype("int16" )
train_df['features'] = train_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
test_df['features'] = test_df["features"].apply(lambda x: " ".join(["_".join(i.split(" ")) for i in x]))
print(train_df["features"].head())
 tfidf = CountVectorizer(stop_words='english', max_features=200)
 tr_sparse = tfidf.fit_transform(train_df["features"])
 te_sparse = tfidf.transform(test_df["features"] )<prepare_x_and_y>
df.dropna(inplace = True)
 df.shape
train_X = sparse.hstack([train_df[features_to_use], tr_sparse] ).tocsr()
test_X = sparse.hstack([test_df[features_to_use], te_sparse] ).tocsr()
target_num_map = {'high':0, 'medium':1, 'low':2}
train_y = np.array(train_df['interest_level'].apply(lambda x: target_num_map[x]))
print(train_X.shape, test_X.shape )<compute_train_metric>
cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id'] + ["event_name_1", "event_name_2", "event_type_1", "event_type_2"]
cols_to_remove = ["id", "date", "sales","d", "wm_yr_wk", "weekday"]
train_cols = df.columns[~df.columns.isin(cols_to_remove)]
X_train = df[train_cols]
y_train = df["sales"]
cv_scores = []
kf = model_selection.KFold(n_splits=5, shuffle=True, random_state=2016)
 for dev_index, val_index in kf.split(range(train_X.shape[0])) :
dev_X, val_X = train_X[dev_index,:], train_X[val_index,:]
dev_y, val_y = train_y[dev_index], train_y[val_index]
preds, model = runXGB(dev_X, dev_y, val_X, val_y)
 cv_scores.append(log_loss(val_y, preds))
print(cv_scores)
 break<save_to_csv>
%%time
np.random.seed(777)
 fake_valid_inds = np.random.choice(X_train.index.values, 2_000_000, replace = False)
 train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
 train_data = lgb.Dataset(X_train.loc[train_inds] , label = y_train.loc[train_inds],
categorical_feature=cat_feats, free_raw_data=False)
 fake_valid_data = lgb.Dataset(X_train.loc[fake_valid_inds], label = y_train.loc[fake_valid_inds],
categorical_feature=cat_feats,
free_raw_data=False )
preds, model = runXGB(train_X, train_y, test_X, num_rounds=1000)
 out_df = pd.DataFrame(preds)
 out_df.columns = ["high", "medium", "low"]
out_df["listing_id"] = test_df.listing_id.values
out_df.to_csv("xgb_starter2.csv", index=False )<import_modules>
del df, X_train, y_train, fake_valid_inds,train_inds ; gc.collect()
ToTensor, Normalize, Compose, Resize, CenterCrop, RandomCrop,
RandomHorizontalFlip, RandomGrayscale)
 print(os.listdir(".. /input"))<define_variables>
params = {
"objective" : "poisson",
"metric" :"rmse",
"force_row_wise" : True,
"learning_rate" : 0.075,
"sub_row" : 0.75,
"bagging_freq" : 1,
"lambda_l2" : 0.1,
"metric": ["rmse"],
'verbosity': 1,
'num_iterations' : 1200,
'num_leaves': 128,
"min_data_in_leaf": 100,
}
ON_KAGGLE = True
N_CLASSES = 1103
DATA_ROOT = Path('.. /input/imet-2019-fgvc6' if ON_KAGGLE else '.. /data')
 RUN_ROOT = '.. /input/seresnext101-folds/' if ON_KAGGLE else '.. /data/results/'
use_sample = False
use_cuda = cuda.is_available()
SIZE = 352
train_root = DATA_ROOT / 'train'
test_root = DATA_ROOT / 'test'<choose_model_class>
%%time
m_lgb = lgb.train(params, train_data, valid_sets = [fake_valid_data], verbose_eval=20 )
class SEModule(nn.Module):
def __init__(self, channels, reduction):
super(SEModule, self ).__init__()
self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.fc1 = nn.Conv2d(channels, channels // reduction, kernel_size=1,
padding=0)
 self.relu = nn.ReLU(inplace=True)
 self.fc2 = nn.Conv2d(channels // reduction, channels, kernel_size=1,
padding=0)
 self.sigmoid = nn.Sigmoid()
def forward(self, x):
module_input = x
x = self.avg_pool(x)
 x = self.fc1(x)
 x = self.relu(x)
 x = self.fc2(x)
 x = self.sigmoid(x)
 return module_input * x
class Bottleneck(nn.Module):

 def forward(self, x):
residual = x
out = self.conv1(x)
 out = self.bn1(out)
 out = self.relu(out)
 out = self.conv2(out)
 out = self.bn2(out)
 out = self.relu(out)
 out = self.conv3(out)
 out = self.bn3(out)
 if self.downsample is not None:
residual = self.downsample(x)
 out = self.se_module(out)+ residual
out = self.relu(out)
 return out
class SEBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes * 2, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm2d(planes * 2)
 self.conv2 = nn.Conv2d(planes * 2, planes * 4, kernel_size=3,
stride=stride, padding=1, groups=groups,
bias=False)
 self.bn2 = nn.BatchNorm2d(planes * 4)
 self.conv3 = nn.Conv2d(planes * 4, planes * 4, kernel_size=1,
bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNetBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None):
super(SEResNetBottleneck, self ).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False,
stride=stride)
 self.bn1 = nn.BatchNorm2d(planes)
 self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1,
groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(planes)
 self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SEResNeXtBottleneck(Bottleneck):

 expansion = 4
def __init__(self, inplanes, planes, groups, reduction, stride=1,
downsample=None, base_width=4):
super(SEResNeXtBottleneck, self ).__init__()
width = math.floor(planes *(base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False,
stride=1)
 self.bn1 = nn.BatchNorm2d(width)
 self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride,
padding=1, groups=groups, bias=False)
 self.bn2 = nn.BatchNorm2d(width)
 self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm2d(planes * 4)
 self.relu = nn.ReLU(inplace=True)
 self.se_module = SEModule(planes * 4, reduction=reduction)
 self.downsample = downsample
self.stride = stride
class SENet(nn.Module):
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):

 super(SENet, self ).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [ 
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)) , 
('bn1', nn.BatchNorm2d(64)) , 
('relu1', nn.ReLU(inplace=True)) , 
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)) , 
('bn2', nn.BatchNorm2d(64)) , 
('relu2', nn.ReLU(inplace=True)) , 
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)) , 
('bn3', nn.BatchNorm2d(inplanes)) , 
('relu3', nn.ReLU(inplace=True)) ,
]
else:
layer0_modules = [ 
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)) , 
('bn1', nn.BatchNorm2d(inplanes)) , 
('relu1', nn.ReLU(inplace=True)) ,
]
layer0_modules.append(( 'pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
 self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
 block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0 
)
 self.layer2 = self._make_layer(
 block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer3 = self._make_layer(
 block,
planes=256,
blocks=layers[2],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.layer4 = self._make_layer(
 block,
planes=512,
blocks=layers[3],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding 
)
 self.avg_pool = nn.AdaptiveAvgPool2d(1)
 self.dropout = nn.Dropout(dropout_p)if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
 def _make_layer(self, block, planes, blocks, groups, reduction, stride=1,
downsample_kernel_size=1, downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
 nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=downsample_kernel_size, stride=stride,
padding=downsample_padding, bias=False),
nn.BatchNorm2d(planes * block.expansion), 
)
 layers = []
layers.append(block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
return nn.Sequential(*layers)
 def features(self, x):
x = self.layer0(x)
 x = self.layer1(x)
 x = self.layer2(x)
 x = self.layer3(x)
 x = self.layer4(x)
 return x
def logits(self, x):
x = self.avg_pool(x)
 if self.dropout is not None:
x = self.dropout(x)
 x = x.view(x.size(0), -1)
 x = self.last_linear(x)
 return x
def forward(self, x):
x = self.features(x)
 x = self.logits(x)
 return x
def initialize_pretrained_model(model, num_classes, settings):
assert num_classes == settings['num_classes'], \
'num_classes should be {}, but is {}'.format(
 settings['num_classes'], num_classes)
 model.load_state_dict(model_zoo.load_url(settings['url']))
model.input_space = settings['input_space']
model.input_size = settings['input_size']
model.input_range = settings['input_range']
model.mean = settings['mean']
model.std = settings['std']
def se_resnext101(num_classes=1000, pretrained=None):
model = SENet(SEResNeXtBottleneck, [3, 4, 23, 3], groups=32, reduction=16,
dropout_p=0.5, inplanes=64, input_3x3=False,
downsample_kernel_size=1, downsample_padding=0,
num_classes=num_classes)
 return model 
<find_best_params>
m_lgb.save_model("model.lgb" )
model = se_resnext101()
model.last_linear = nn.Linear(model.last_linear.in_features, N_CLASSES)
 all_params = list(model.parameters())
 use_cuda = cuda.is_available()
if use_cuda:
model = model.cuda()<load_pretrained>
def create_lag_features_for_test(dt, day):
lags = [7, 28]
lag_cols = [f"lag_{lag}" for lag in lags]
for lag, lag_col in zip(lags, lag_cols):
dt.loc[dt.date == day, lag_col] = \
dt.loc[dt.date ==day-timedelta(days=lag), 'sales'].values
windows = [7, 28]
for window in windows:
for lag in lags:
df_window = dt[(dt.date <= day-timedelta(days=lag)) &(dt.date > day-timedelta(days=lag+window)) ]
df_window_grouped = df_window.groupby("id" ).agg({'sales':'mean'} ).reindex(dt.loc[dt.date==day,'id'])
 dt.loc[dt.date == day,f"rmean_{lag}_{window}"] = \
df_window_grouped.sales.values
def load_model(model, root: str, fold: int, use_cuda: bool):

 best_model_path = root + 'best-metric_fold'+str(fold)+'.pt'
if use_cuda:
state_dict = torch.load(best_model_path)
 else:
state_dict = torch.load(best_model_path, map_location='cpu')
 model.load_state_dict(state_dict['model'])
 print('Loaded model from epoch {epoch}, step {step:,}'.format(**state_dict))
model.eval()<create_dataframe>
def create_date_features_for_test(dt):
date_features = {
"wday": "weekday",
"week": "weekofyear",
"month": "month",
"quarter": "quarter",
"year": "year",
"mday": "day",
}
for date_feat_name, date_feat_func in date_features.items() :
if date_feat_name in dt.columns:
dt[date_feat_name] = dt[date_feat_name].astype("int16")
 else:
dt[date_feat_name] = getattr(
 dt["date"].dt, date_feat_func ).astype("int16" )
class TTADataset(Dataset):
def __init__(self, root: Path, df: pd.DataFrame,
image_transform: Callable, tta: int):
super().__init__()
self._root = root
self._df = df
self._image_transform = image_transform
self._tta = tta
def __len__(self):
return len(self._df)* self._tta
def __getitem__(self, idx: int):
item = self._df.iloc[idx % len(self._df)]
image = load_image(item, self._root)
 width, height = image.size
if height < 320:
ratio = 400/height
image = image.resize(( int(width * ratio), int(height * ratio)) , Image.ANTIALIAS)
 if width < 320:
ratio = 400/width
image = image.resize(( int(width * ratio), int(height * ratio)) , Image.ANTIALIAS)
 image = self._image_transform(image)
 image = tensor_transform(image)
 return image, item.id<predict_on_test>
%%time
alphas = [1.028, 1.023, 1.018]
weights = [1/len(alphas)]*len(alphas)
 te0 = create_dt(False)
 create_date_features_for_test(te0)
 for icount,(alpha, weight)in enumerate(zip(alphas, weights)) :
te = te0.copy()
cols = [f"F{i}" for i in range(1, 29)]
for tdelta in range(0, 28):
day = FDAY + timedelta(days=tdelta)
 print(tdelta, day.date())
 tst = te[(te.date >= day - timedelta(days=MAX_LAGS))
&(te.date <= day)].copy()
create_lag_features_for_test(tst, day)
 tst = tst.loc[tst.date == day, train_cols]
te.loc[te.date == day, "sales"] = \
alpha * m_lgb.predict(tst)
 te_sub = te.loc[te.date >= FDAY, ["id", "sales"]].copy()
te_sub["F"] = [f"F{rank}" for rank in te_sub.groupby("id")[
"id"].cumcount() +1]
te_sub = te_sub.set_index(["id", "F"] ).unstack() [
"sales"][cols].reset_index()
te_sub.fillna(0., inplace=True)
 te_sub.sort_values("id", inplace=True)
 te_sub.reset_index(drop=True, inplace=True)
 te_sub.to_csv(f"submission_{icount}.csv", index=False)
 if icount == 0:
sub = te_sub
sub[cols] *= weight
else:
sub[cols] += te_sub[cols]*weight
print(icount, alpha, weight )
def predict(model, root: Path, predict_df: pd.DataFrame, save_root: str,
image_transform, batch_size: int, tta: int, workers: int, use_cuda: bool):

 valid_loader = DataLoader(
 TTADataset(root, predict_df, image_transform, tta),
shuffle=False,
batch_size=batch_size,
num_workers=workers, 
)
 print(f'{len(valid_loader.dataset):,} in valid')
 all_outputs, all_ids = [], []
with torch.no_grad() :
for inputs, ids in tqdm.tqdm(valid_loader, desc='Predict'):
if use_cuda:
inputs = inputs.cuda()
outputs = torch.sigmoid(model(inputs))
all_outputs.append(outputs.data.cpu().numpy())
 all_ids.extend(ids)
 df = pd.DataFrame(
 data=np.concatenate(all_outputs),
index=all_ids,
columns=map(str, range(N_CLASSES)))
 df = mean_df(df)
 print('probs: ', df.head(10))
return df
def binarize_prediction(probabilities, threshold: float, argsorted=None,
min_labels=1, max_labels=10):

 assert probabilities.shape[1] == N_CLASSES
if argsorted is None:
argsorted = probabilities.argsort(axis=1)
 max_mask = _make_mask(argsorted, max_labels)
 min_mask = _make_mask(argsorted, min_labels)
 prob_mask = probabilities > threshold
return(max_mask & prob_mask)| min_mask
def _make_mask(argsorted, top_n: int):
mask = np.zeros_like(argsorted, dtype=np.uint8)
 col_indices = argsorted[:, -top_n:].reshape(-1)
 row_indices = [i // top_n for i in range(len(col_indices)) ]
mask[row_indices, col_indices] = 1
return mask<predict_on_test>
sub.id.nunique() , sub["id"].str.contains("validation$" ).sum()
<train_model><EOS> 
sub2 = sub.copy()
sub2["id"] = sub2["id"].str.replace("validation$", "evaluation")
 sub = pd.concat([sub, sub2], axis=0, sort=False)
 sub.to_csv("submission.csv",index=False )