Kaggle competition: Sberbank Russian Housing Market
<SOS> metric: RMSLE Kaggle data source: sberbank-russian-housing-market<categorify>
%matplotlib inline
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 macro = pd.read_csv('.. /input/macro.csv')
 id_test = test.id
train.sample(3)
def process_det(index,det,transform,score_threshold=0):
boxes = det[index].detach().cpu().numpy() [:,:4]
scores = det[index].detach().cpu().numpy() [:,4]
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = transform.deaugment_boxes(boxes.copy())
 boxes =(boxes*2 ).clip(min=0, max=1023 ).astype(int)
 scores = scores[indexes]
return boxes, scores<define_variables>
def calculate_final_score(all_predictions, score_threshold):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
image_id = all_predictions[i]['image_id']
for transform_number in range(len(tta_transforms)) :
pred_boxes = all_predictions[i][f'pred_boxes_transform{transform_number}'].copy()
scores = all_predictions[i][f'scores_transform{transform_number}'].copy()
indexes = np.where(scores>score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
rank = np.argsort(scores)[::-1]
scores = scores[rank]
pred_boxes = pred_boxes[rank]
image_precision = calculate_image_precision(gt_boxes, pred_boxes,thresholds=iou_thresholds,form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<compute_train_metric>
y_train = train["price_doc"]
x_train = train.drop(["id", "timestamp", "price_doc"], axis=1)
 x_test = test.drop(["id", "timestamp"], axis=1)
 for c in x_train.columns:
if x_train[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_train[c].values))
x_train[c] = lbl.transform(list(x_train[c].values))
for c in x_test.columns:
if x_test[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(x_test[c].values))
x_test[c] = lbl.transform(list(x_test[c].values))
def search_best_score_threshold(all_predictions):
best_final_score, best_score_threshold = 0, 0
for score_threshold in tqdm(np.arange(0, 1, 0.01), total=np.arange(0, 1, 0.01 ).shape[0]):
final_score = calculate_final_score(all_predictions, score_threshold)
 if final_score > best_final_score:
best_final_score = final_score
best_score_threshold = score_threshold
print('-'*30)
 print(f'[Best Score Threshold]: {best_score_threshold}')
 print(f'[OOF Score]: {best_final_score:.4f}')
 print('-'*30)
 return best_score_threshold<compute_train_metric>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(x_train, y_train)
 dtest = xgb.DMatrix(x_test )
if SCORE_OPTIM:
all_predictions = validate(model=test_net)
 gt_boxes = all_predictions[0]['gt_boxes'].copy()
pred_boxes = all_predictions[0]['pred_boxes_transform1'].copy()
scores = all_predictions[0]['scores_transform1'].copy()
image_id = all_predictions[0]['image_id']
indexes = np.where(scores>best_score_threshold)
 pred_boxes = pred_boxes[indexes]
scores = scores[indexes]
show_result(image_id, pred_boxes, gt_boxes)
 best_score_threshold = search_best_score_threshold(all_predictions)
<compute_train_metric>
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=20,
verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()
def calculate_final_score_wbf(
 all_predictions,
iou_thr,
skip_box_thr,
method,
sigma=0.5, 
):
final_scores = []
for i in range(len(all_predictions)) :
gt_boxes = all_predictions[i]['gt_boxes'].copy()
image_id = all_predictions[i]['image_id']
folds_boxes, folds_scores, folds_labels = [], [], []
for transform_number in range(len(tta_transforms)) :
pred_boxes = all_predictions[i][f'pred_boxes_transform{transform_number}'].copy()
scores = all_predictions[i][f'scores_transform{transform_number}'].copy()
folds_boxes.append(pred_boxes)
 folds_scores.append(scores)
 folds_labels.append(np.ones(pred_boxes.shape[0]))
if method == 'weighted_boxes_fusion':
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'nms':
boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr)
 elif method == 'soft_nms':
boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)
 elif method == 'non_maximum_weighted':
boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
raise
image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<compute_train_metric>
num_boost_rounds = len(cv_output)
 model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round= num_boost_rounds )
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, method, n_calls=10):
@use_named_args(space)
 def score(**params):
log('-'*5 + f'{method}' + '-'*5)
 log(params)
 final_score = calculate_final_score_wbf(all_predictions, method=method, **params)
 log(f'final_score = {final_score}')
 log('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<categorify>
y_predict = model.predict(dtest)
 output = pd.DataFrame({'id': id_test, 'price_doc': y_predict})
 output.head()
<predict_on_test><EOS> 
output.to_csv('xgbSub.csv', index=False )