Kaggle competition: Sberbank Russian Housing Market
<SOS> metric: RMSLE Kaggle data source: sberbank-russian-housing-market<init_hyperparams>
import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.AdamW(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
print("better model find at epoch:{}".format(self.epoch))
best_weight=self.model.model.state_dict()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
return best_weight
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
self.optimizer.zero_grad()
loss, _, _ = self.model(images, boxes, labels)
 loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 self.optimizer.step()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<init_hyperparams>
df_train = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 df_macro = pd.read_csv(".. /input/macro.csv", parse_dates=['timestamp'])
 df_train.head()
class TrainGlobalConfig:
num_workers = 4
batch_size = 4
n_epochs = 25
lr = 0.0001
folder = 'effdet5-cutmix-augmix'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<train_model>
df_train.loc[df_train['full_sq'].isnull() ,'full_sq'] = 54
tr_full_sq = df_train['full_sq'].values
df_train['full_sq'] = np.apply_along_axis(lambda x:(x+54)/2,0,tr_full_sq)
 df_test.loc[df_test['full_sq'].isnull() ,'full_sq'] = 54
te_full_sq = df_test['full_sq'].values
df_test['full_sq'] = np.apply_along_axis(lambda x:(x+54)/2,0,te_full_sq)
 df_train['price_per_m2'] = df_train['price_doc']/df_train['full_sq'].astype(float)
 y_train = df_train['price_per_m2'].values
id_test = df_test['id']
df_train.drop(['id', 'price_doc', 'price_per_m2'], axis=1, inplace=True)
 df_test.drop(['id'], axis=1, inplace=True)
 num_train = len(df_train)
 df_all = pd.concat([df_train, df_test])
 df_all = pd.merge_ordered(df_all, df_macro, on='timestamp', how='left')
 print(df_all.shape)
 month_year =(df_all.timestamp.dt.month + df_all.timestamp.dt.year * 100)
 month_year_cnt_map = month_year.value_counts().to_dict()
df_all['month_year_cnt'] = month_year.map(month_year_cnt_map)
 week_year =(df_all.timestamp.dt.weekofyear + df_all.timestamp.dt.year * 100)
 week_year_cnt_map = week_year.value_counts().to_dict()
df_all['week_year_cnt'] = week_year.map(week_year_cnt_map)
 df_all['month'] = df_all.timestamp.dt.month
df_all['dow'] = df_all.timestamp.dt.dayofweek
df_all['rel_floor'] = df_all['floor'] / df_all['max_floor'].astype(float)
 df_all['rel_kitch_sq'] = df_all['kitch_sq'] / df_all['full_sq'].astype(float)
 df_all.drop(['timestamp'], axis=1, inplace=True )
def run_training(train_dataset,validation_dataset,train_net=None):
device = torch.device('cuda:0')
 train_net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=train_net, device=device, config=TrainGlobalConfig)
 best_weight = fitter.fit(train_loader, val_loader)
 return best_weight<train_on_grid>
df_numeric = df_all.select_dtypes(exclude=['object'])
 df_obj = df_all.select_dtypes(include=['object'] ).copy()
for c in df_obj:
df_obj[c] = pd.factorize(df_obj[c])[0]
df_values = pd.concat([df_numeric, df_obj], axis=1 )
def get_train_net(eval_net):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net.load_state_dict(eval_net.model.state_dict())
 gc.collect()
net = DetBenchTrain(net, config)
 net = net.train()
return net.cuda() 
<choose_model_class>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(X_train, y_train, feature_names=df_columns)
 dtest = xgb.DMatrix(X_test, feature_names=df_columns )
def get_test_net(best_weigth):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net.load_state_dict(best_weigth)
 gc.collect()
net = DetBenchEval(net, config)
 net = net.eval()
return net.cuda()<split>
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_round )
<load_pretrained><EOS> 
y_pred = model.predict(dtest)
 y_pred = y_pred*df_test['full_sq']
df_sub = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_sub.to_csv('sub.csv', index=False )