Kaggle competition: Sberbank Russian Housing Market
<SOS> metric: RMSLE Kaggle data source: sberbank-russian-housing-market<predict_on_test>
%matplotlib inline
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv' )
test_df_pseudo = predict_test(test_data_loader, nets )<load_from_csv>
train.groupby('sub_area' ).sub_area.count()
def generate_train(test_df_pseudo, fold_=1):
print("Loading training set...")
 marking = pd.read_csv('.. /input/wheat-cleaned-dataset/clean_train_v2.csv')
 marking = marking[marking['area']> 300]
df_with_pseudo = pd.concat([marking, test_df_pseudo])
 index = list(set(marking.image_id))
fold = 0
val_index = index[len(index)*fold//5:len(index)*(fold+1)//5]
train_index = index[:len(index)*fold//5] + index[len(index)*(fold+1)//5:]
train_df_pseudo_fold = test_df_pseudo['image_id'].unique()
train_fold = np.concatenate(( train_index, train_df_pseudo_fold))
print('train dataset size:', len(train_fold))
val_fold = np.array(val_index)
 print('val dataset size:', len(val_fold))
return df_with_pseudo, train_fold, val_fold<split>
train.groupby('ecology' ).sub_area.count()
df_with_pesudo, train_fold, val_fold = generate_train(test_df_pseudo )<define_search_model>
train.groupby('state' ).sub_area.count()
def get_train_transforms() :
return A.Compose(
 [
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.RandomRotate90(p=0.5),
A.Resize(height=1024, width=1024, p=1),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=1024, width=1024, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
 )<define_variables>
Data_stat=train.groupby('state' ).sub_area.count().sort_values()
Data_stat
TRAIN_ROOT_PATH = '.. /input/global-wheat-detection/train'
class DatasetRetriever(Dataset):
def __init__(self, marking, image_ids, transforms=None, test=False, mixup=True):
super().__init__()
self.image_ids = image_ids
self.marking = marking
self.transforms = transforms
self.test = test
self.mixup = mixup
self.mosaic_border = [-1024//2, -1024//2]
def __getitem__(self, index: int):
image_id = self.image_ids[index]
origin_image = False
if self.test or random.random() > 0.5:
image, boxes = self.load_image_and_boxes(index)
 origin_image = True
else:
image, boxes = self.load_cutmix_image_and_boxes(index)
 if(random.random() < 0.9)and(self.test == False):
image, boxes = cutout(image, boxes)
 labels = torch.ones(( boxes.shape[0],), dtype=torch.int64)
 target = {}
target['boxes'] = boxes
target['labels'] = labels
target['image_id'] = torch.tensor([index])
 if self.transforms:
for i in range(10):
sample = self.transforms(**{
'image': image,
'bboxes': target['boxes'],
'labels': labels
})
 if len(sample['bboxes'])> 0:
image = sample['image']
target['labels'] = torch.ones(( len(sample['bboxes']),), dtype=torch.int64)
 target['boxes'] = torch.stack(tuple(map(torch.tensor, zip(*sample['bboxes'])))).permute(1, 0)
 target['boxes'][:,[0,1,2,3]] = target['boxes'][:,[1,0,3,2]]
break
return image, target, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]
def load_image_and_boxes(self, index):
image_id = self.image_ids[index]
if os.path.exists(f'{TRAIN_ROOT_PATH}/{image_id}.jpg'):
image = cv2.imread(f'{TRAIN_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 else:
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image = cv2.resize(image,(1024, 1024))
image /= 255.0
records = self.marking[self.marking['image_id'] == image_id]
boxes = records[['x', 'y', 'w', 'h']].values
boxes[:, 2] = boxes[:, 0] + boxes[:, 2]
boxes[:, 3] = boxes[:, 1] + boxes[:, 3]
return image, boxes
def load_cutmix_image_and_boxes(self, index, imsize=1024):

 w, h = imsize, imsize
s = imsize
xc, yc = [int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border]
indexes = [index] + [random.randint(0, self.image_ids.shape[0] - 1)for _ in range(3)]
result_image = np.full(( imsize*2, imsize*2, 3), 114/255, dtype=np.float32)
 result_boxes = []
for i, index in enumerate(indexes):
image, boxes = self.load_image_and_boxes(index)
 if i == 0:
x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc
x1b, y1b, x2b, y2b = w -(x2a - x1a), h -(y2a - y1a), w, h
elif i == 1:
x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc
x1b, y1b, x2b, y2b = 0, h -(y2a - y1a), min(w, x2a - x1a), h
elif i == 2:
x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = w -(x2a - x1a), 0, max(xc, w), min(y2a - y1a, h)
 elif i == 3:
x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h)
 x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h)
 result_image[y1a:y2a, x1a:x2a] = image[y1b:y2b, x1b:x2b]
padw = x1a - x1b
padh = y1a - y1b
boxes[:, 0] += padw
boxes[:, 1] += padh
boxes[:, 2] += padw
boxes[:, 3] += padh
result_boxes.append(boxes)
 result_boxes = np.concatenate(result_boxes, 0)
 np.clip(result_boxes[:, 0:], 0, 2 * s, out=result_boxes[:, 0:])
 result_boxes = result_boxes.astype(np.int32)
 result_boxes = result_boxes[np.where(( result_boxes[:, 2] - result_boxes[:, 0])*(result_boxes[:, 3] - result_boxes[:, 1])> 300.0)]
result_boxes = result_boxes[np.where(( result_boxes[:, 2] - result_boxes[:, 0])>= 10.0)]
result_boxes = result_boxes[np.where(( result_boxes[:, 3] - result_boxes[:, 1])>= 10.0)]
result_image, result_boxes = random_affine(result_image, targets=result_boxes)
 return result_image, result_boxes<create_dataframe>
train=train.fillna(-999)
 pd.isnull(train ).any()
train_dataset = DatasetRetriever(
 image_ids=train_fold,
marking=df_with_pesudo,
transforms=get_train_transforms() ,
test=False, 
)
 validation_dataset = DatasetRetriever(
 image_ids=val_fold,
marking=df_with_pesudo,
transforms=get_valid_transforms() ,
test=True,
)<normalization>
pd.isnull(test ).any()
class AverageMeter(object):

 def __init__(self):
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
self.count = 0
def update(self, val, n=1):
self.val = val
self.sum += val * n
self.count += n
self.avg = self.sum / self.count<define_variables>
test=test.fillna(-999 )
class TrainGlobalConfig:
num_workers = 2
batch_size = 2
use_amp = True
use_ema = True
n_epochs = 8
total_batch_size = 8
clip_grad = True
accumulate_steps = int(total_batch_size / batch_size)
 lr = 0.0035
folder = 'effdet7'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = CosineLRScheduler
scheduler_params = dict(
 t_initial=n_epochs,
t_mul=1.0,
lr_min = 1e-8, 
)
<choose_model_class>
y_train = train["price_doc"]
x_train = train.drop(["id", "price_doc","timestamp" ], axis=1)
 x_test = test.drop(["id","timestamp"], axis=1)
 id_test = test['id']
def is_parallel(model):
return type(model)in(nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)
 class ModelEMA:

 def __init__(self, model, decay=0.9999, updates=0):
self.ema = deepcopy(model.module if is_parallel(model)else model ).eval()
self.updates = updates
self.decay = lambda x: decay *(1 - math.exp(-x / 2000))
for p in self.ema.parameters() :
p.requires_grad_(False)
 def update(self, model):
with torch.no_grad() :
self.updates += 1
d = self.decay(self.updates)
 msd = model.module.state_dict() if is_parallel(model)else model.state_dict()
for k, v in self.ema.state_dict().items() :
if v.dtype.is_floating_point:
v *= d
v +=(1.- d)* msd[k].detach()
def update_attr(self, model, include=() , exclude=('process_group', 'reducer')) :
copy_attr(self.ema, model, include, exclude )<set_options>
x_train.dtypes.value_counts()
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
self.use_amp = config.use_amp
self.accumulate_steps = config.accumulate_steps
self.clip_grad = config.clip_grad
self.max_norm = 10.0
self.use_ema = config.use_ema
self.model_ema = None
self.model_ema_decay = 0.999
pg0, pg1, pg2 = [], [], []
for k, v in self.model.named_parameters() :
if v.requires_grad:
if '.bias' in k:
pg2.append(v)
 elif '.weight' in k and '.bn' not in k:
pg1.append(v)
 else:
pg0.append(v)
 self.optimizer = torch.optim.SGD(pg0, lr=config.lr, momentum=0.9, nesterov=True)
 self.optimizer.add_param_group({'params': pg1, 'weight_decay': 5e-4})
 self.optimizer.add_param_group({'params': pg2})
 self.model.to(self.device)
 if self.use_amp:
self.model, self.optimizer = amp.initialize(self.model, self.optimizer, opt_level="O1",verbosity=0)
 if self.use_ema:
self.model_ema = ModelEMA(self.model, decay=self.model_ema_decay)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
best_epoch = 0
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint.bin')
 t = time.time()
if self.use_ema:
summary_loss = self.validation(self.model_ema.ema.module if hasattr(self.model_ema.ema, 'module')else self.model_ema.ema,
validation_loader)
 else:
summary_loss = self.validation(self.model, validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 best_epoch = e
if self.config.validation_scheduler:
self.scheduler.step(e+1)
 self.epoch += 1
return best_epoch
def validation(self, model, val_loader):
model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images ).to(self.device ).float()
batch_size = images.shape[0]
target_input = {'bbox':[target['boxes'].to(self.device ).float() for target in targets],
'cls': [target['labels'].to(self.device ).float() for target in targets]}
loss = model(images, target_input['bbox'], target_input['cls'])
 loss, class_loss, box_loss = loss
summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
self.optimizer.zero_grad()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images ).to(self.device ).float()
batch_size = images.shape[0]
target_input = {'bbox':[target['boxes'].to(self.device ).float() for target in targets],
'cls': [target['labels'].to(self.device ).float() for target in targets]}
loss = self.model(images, target_input['bbox'], target_input['cls'])
 loss, class_loss, box_loss = loss
if self.use_amp:
with amp.scale_loss(loss, self.optimizer)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if(step+1)% self.accumulate_steps == 0:
if self.clip_grad:
if self.use_amp:
torch.nn.utils.clip_grad_norm_(amp.master_params(self.optimizer), self.max_norm)
 else:
torch.nn.utils.clip_grad_norm_(self.model.parameters() , self.max_norm)
 self.optimizer.step()
self.optimizer.zero_grad()
if self.model_ema is not None:
self.model_ema.update(self.model)
 summary_loss.update(loss.detach().item() , batch_size)
 if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
if self.use_ema:
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'state_dict_ema': self.model_ema.ema.module.state_dict() if hasattr(self.model_ema.ema, 'module')else self.model_ema.ema.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 else:
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<load_pretrained>
for c in x_train.columns:
if x_train[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_train[c].values))
x_train[c] = lbl.transform(list(x_train[c].values))
def collate_fn(batch):
return tuple(zip(*batch))
def run_training() :
device = torch.device('cuda:0')
 net.to(device)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=TrainGlobalConfig.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=TrainGlobalConfig.num_workers,
collate_fn=collate_fn, 
)
 val_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=TrainGlobalConfig.batch_size,
num_workers=TrainGlobalConfig.num_workers,
shuffle=False,
sampler=SequentialSampler(validation_dataset),
pin_memory=False,
collate_fn=collate_fn, 
)
 fitter = Fitter(model=net, device=device, config=TrainGlobalConfig)
 best_epoch = fitter.fit(train_loader, val_loader)
 return best_epoch<choose_model_class>
for c in x_test.columns:
if x_test[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_test[c].values))
x_test[c] = lbl.transform(list(x_test[c].values))
def get_net() :
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size = 1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(trained_weight_path)
 net.load_state_dict(checkpoint['state_dict_ema'])
 return DetBenchTrain(net, config)
 net = get_net()<train_model>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test)
best_epoch = None
if len(os.listdir(".. /input/global-wheat-detection/test/")) > 11:
best_epoch = run_training()<load_pretrained>
num_boost_rounds = 384
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round= num_boost_rounds )
if best_epoch is not None:
print("load retrained model {}".format(best_epoch))
net = create_model_efficientdet_test(compound_coef=7,
weight_path=f'./effdet7/best-checkpoint-{str(best_epoch ).zfill(3)}epoch.bin',
image_size=1024,
model_ema=True)
 else:
net = create_model_efficientdet_test(compound_coef=7,
weight_path=trained_weight_path,
image_size=1024,
model_ema=True )<categorify>
y_predict = model.predict(dtest)
 output = pd.DataFrame({'id': id_test, 'price_doc': y_predict})
 output.head()
output.to_csv('xgbSb.csv', index=False)
<categorify><EOS> 
pd.read_csv('.. /output/xgbSb.csv' )