Kaggle competition: Sberbank Russian Housing Market
<SOS> metric: RMSLE Kaggle data source: sberbank-russian-housing-market<load_pretrained>
import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt
test_dataset = TestDatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{TEST_DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
test_data_loader = DataLoader(
 test_dataset,
batch_size=4,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
macro_cols = ["balance_trade", "balance_trade_growth", "eurrub", "average_provision_of_build_contract",
"micex_rgbi_tr", "micex_cbi_tr", "deposits_rate", "mortgage_value", "mortgage_rate",
"income_per_cap", "rent_price_4+room_bus", "museum_visitis_per_100_cap", "apartment_build"]
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda() 
<categorify>
df_train = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 df_macro = pd.read_csv(".. /input/macro.csv", parse_dates=['timestamp'], usecols=['timestamp'] + macro_cols)
 df_train.head()
class BaseWheatTTA:

 image_size = 512
def augment(self,image):
raise NotImplementedError
def batch_augment(self,images):
raise NotImplementedError
def deaugment_boxes(self,boxes):
raise NotImplementedError
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self,image):
return image.flip(1)
 def batch_augment(self,images):
return images.flip(2)
 def deaugment_boxes(self,boxes):
boxes[:,[1,3]] = self.image_size - boxes[:,[3,1]]
return boxes
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTARotate180(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 2,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 2,(2, 3))
def deaugment_boxes(self, boxes):
boxes[:, [0,1,2,3]] = self.image_size - boxes[:, [2,3,0,1]]
return boxes
class TTARotate270(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 3,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 3,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = boxes[:, [1,3]]
res_boxes[:, [1,3]] = self.image_size - boxes[:, [2,0]]
return res_boxes
class TTAHSV_or_RBC(BaseWheatTTA):

 transform = A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.2,
val_shift_limit=0.2,p=0.5),
A.RandomBrightnessContrast(brightness_limit=0.2,
contrast_limit=0.2, p=0.5),
],p=1)
 def augment(self,image):
image = image.permute(1,2,0 ).cpu().numpy()
image = self.transform(image=image)['image']
return ToTensorV2()(image=image)['image']
def batch_augment(self, images):
batch_size = len(images)
 for i in range(batch_size):
images[i] = self.augment(images[i])
 return images
def deaugment_boxes(self,boxes):
return boxes
class TTAToGray(BaseWheatTTA):

 transform = A.ToGray()
def augment(self,image):
image = image.permute(1,2,0 ).cpu().numpy()
image = self.transform(image=image)['image']
return ToTensorV2()(image=image)['image']
def batch_augment(self, images):
batch_size = len(images)
 for i in range(batch_size):
images[i] = self.augment(images[i])
 return images
def deaugment_boxes(self,boxes):
return boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<categorify>
ylog_train_all = np.log1p(df_train['price_doc'].values)
 id_test = df_test['id']
df_train.drop(['id', 'price_doc'], axis=1, inplace=True)
 df_test.drop(['id'], axis=1, inplace=True)
 num_train = len(df_train)
 df_all = pd.concat([df_train, df_test])
 df_all = pd.merge_ordered(df_all, df_macro, on='timestamp', how='left')
 print(df_all.shape)
 month_year =(df_all.timestamp.dt.month + df_all.timestamp.dt.year * 100)
 month_year_cnt_map = month_year.value_counts().to_dict()
df_all['month_year_cnt'] = month_year.map(month_year_cnt_map)
 week_year =(df_all.timestamp.dt.weekofyear + df_all.timestamp.dt.year * 100)
 week_year_cnt_map = week_year.value_counts().to_dict()
df_all['week_year_cnt'] = week_year.map(week_year_cnt_map)
 df_all['month'] = df_all.timestamp.dt.month
df_all['dow'] = df_all.timestamp.dt.dayofweek
df_all['rel_floor'] = df_all['floor'] / df_all['max_floor'].astype(float)
 df_all['rel_kitch_sq'] = df_all['kitch_sq'] / df_all['full_sq'].astype(float)
 df_all.drop(['timestamp'], axis=1, inplace=True )
def process_det(index, det, score_threshold=0.25):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes =(boxes ).clip(min=0, max=511 ).astype(int)
 indexes = np.where(scores>score_threshold)
 boxes = boxes[indexes]
scores = scores[indexes]
return boxes, scores<categorify>
df_numeric = df_all.select_dtypes(exclude=['object'])
 df_obj = df_all.select_dtypes(include=['object'] ).copy()
for c in df_obj:
df_obj[c] = pd.factorize(df_obj[c])[0]
df_values = pd.concat([df_numeric, df_obj], axis=1 )
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() ,None],
[TTAVerticalFlip() ,None],
[TTARotate90() ,None], 
):
tta_transforms.append(TTACompose([tta_transform for tta_transform in tta_combination if tta_transform])) 
<predict_on_test>
dtrain_all = xgb.DMatrix(X_train_all, ylog_train_all, feature_names=df_columns)
 dtrain = xgb.DMatrix(X_train, ylog_train, feature_names=df_columns)
 dval = xgb.DMatrix(X_val, ylog_val, feature_names=df_columns)
 dtest = xgb.DMatrix(X_test, feature_names=df_columns )
def make_tta_predictions(images, score_threshold=best_score_threshold,nets=None):
with torch.no_grad() :
images = torch.stack(images ).float().cuda()
predictions = []
for net in nets:
for tta_transform in tta_transforms:
result = []
det = net(tta_transform.batch_augment(images.clone()), torch.tensor([1]*images.shape[0] ).float().cuda())
 for i in range(images.shape[0]):
boxes = det[i].detach().cpu().numpy() [:,:4]
scores = det[i].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = tta_transform.deaugment_boxes(boxes.copy())
 result.append({
'boxes': boxes,
'scores': scores[indexes],
})
 predictions.append(result)
 return predictions
def run_wbf(predictions, image_index,image_size=512,iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr, weights=None):
boxes = [(prediction[image_index]['boxes']/(image_size-1)).tolist() for prediction in predictions]
scores = [prediction[image_index]['scores'].tolist() for prediction in predictions]
labels = [np.ones(prediction[image_index]['scores'].shape[0] ).astype(int ).tolist() for prediction in predictions]
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=None, iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<import_modules>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 1.0,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
partial_model = xgb.train(xgb_params, dtrain, num_boost_round=1000, evals=[(dval, 'val')],
early_stopping_rounds=20, verbose_eval=20)
 num_boost_round = partial_model.best_iteration
@jit(nopython=True)
 def calculate_iou(gt, pr, form='pascal_voc')-> float:

 if form == 'coco':
gt = gt.copy()
pr = pr.copy()
gt[2] = gt[0] + gt[2]
gt[3] = gt[1] + gt[3]
pr[2] = pr[0] + pr[2]
pr[3] = pr[1] + pr[3]
dx = min(gt[2], pr[2])- max(gt[0], pr[0])+ 1
if dx < 0:
return 0.0
dy = min(gt[3], pr[3])- max(gt[1], pr[1])+ 1
if dy < 0:
return 0.0
overlap_area = dx * dy
union_area =(
(gt[2] - gt[0] + 1)*(gt[3] - gt[1] + 1)+ 
(pr[2] - pr[0] + 1)*(pr[3] - pr[1] + 1)-
overlap_area 
)
 return overlap_area / union_area
@jit(nopython=True)
 def find_best_match(gts, pred, pred_idx, threshold = 0.5, form = 'pascal_voc', ious=None)-> int:

 best_match_iou = -np.inf
best_match_idx = -1
for gt_idx in range(len(gts)) :
if gts[gt_idx][0] < 0:
continue
iou = -1 if ious is None else ious[gt_idx][pred_idx]
if iou < 0:
iou = calculate_iou(gts[gt_idx], pred, form=form)
 if ious is not None:
ious[gt_idx][pred_idx] = iou
if iou < threshold:
continue
if iou > best_match_iou:
best_match_iou = iou
best_match_idx = gt_idx
return best_match_idx
@jit(nopython=True)
 def calculate_precision(gts, preds, threshold = 0.5, form = 'coco', ious=None)-> float:

 n = len(preds)
 tp = 0
fp = 0
for pred_idx in range(n):
best_match_gt_idx = find_best_match(gts, preds[pred_idx], pred_idx,
threshold=threshold, form=form, ious=ious)
 if best_match_gt_idx >= 0:
tp += 1
gts[best_match_gt_idx] = -1
else:
fp += 1
fn =(gts.sum(axis=1)> 0 ).sum()
return tp /(tp + fp + fn)
 @jit(nopython=True)
 def calculate_image_precision(gts, preds, thresholds =(0.5,), form = 'coco')-> float:

 n_threshold = len(thresholds)
 image_precision = 0.0
ious = np.ones(( len(gts), len(preds)))* -1
for threshold in thresholds:
precision_at_threshold = calculate_precision(gts.copy() , preds, threshold=threshold,
form=form, ious=ious)
 image_precision += precision_at_threshold / n_threshold
return image_precision
def show_result(sample_id, preds, gt_boxes):
sample = cv2.imread(f'{TRAIN_DATA_ROOT_PATH}/{sample_id}.jpg', cv2.IMREAD_COLOR)
 sample = cv2.cvtColor(sample, cv2.COLOR_BGR2RGB)
 fig, ax = plt.subplots(1, 1, figsize=(16, 8))
for pred_box in preds:
cv2.rectangle(
 sample, 
(pred_box[0], pred_box[1]), 
(pred_box[2], pred_box[3]), 
(220, 0, 0), 2 
)
 for gt_box in gt_boxes:
cv2.rectangle(
 sample, 
(gt_box[0], gt_box[1]), 
(gt_box[2], gt_box[3]), 
(0, 0, 220), 2 
)
 ax.set_axis_off()
ax.imshow(sample)
 ax.set_title("RED: Predicted | BLUE - Ground-truth")
 iou_thresholds = numba.typed.List()
for x in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75]:
iou_thresholds.append(x )<predict_on_test>
num_boost_round = partial_model.best_iteration
def predict_test(data_loader,nets,detection_threshold = 0.5):
testdf_pseudo = []
for images, image_ids in data_loader:
predictions = make_tta_predictions(images,nets=nets)
 for i, image in enumerate(images):
boxes, scores, labels = run_wbf(predictions, image_index=i)
 boxes =(boxes*2 ).round().astype(np.int32 ).clip(min=0, max=1023)
 boxes = boxes[scores >= detection_threshold].astype(np.int32)
 scores = scores[scores >= detection_threshold]
image_id = image_ids[i]
boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'gzyou'+image_id,
'width':1024,
'height':1024,
'source': 'gzyou',
'x': box[0],
'y': box[1],
'w': box[2],
'h': box[3]
}
testdf_pseudo.append(result)
 print(len(testdf_pseudo))
test_df_pseudo = pd.DataFrame(testdf_pseudo, columns=['image_id', 'width', 'height', 'source', 'x', 'y', 'w', 'h'])
 return test_df_pseudo<compute_test_metric>
model = xgb.train(dict(xgb_params, silent=0), dtrain_all, num_boost_round=num_boost_round )
<load_pretrained><EOS> 
ylog_pred = model.predict(dtest)
 y_pred = np.exp(ylog_pred)- 1
df_sub = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_sub.to_csv('sub.csv', index=False )