Kaggle competition: Sberbank Russian Housing Market
results = []
flag = True
csv_pseudo = []
for shape,images, image_ids in data_loader:
boxes_all,scores_all,y = make_tta_predictions(images,score_threshold=0.4024215973594928)
 for i, image in enumerate(images):
h,w,_ = shape[i]
w_factor = float(w)/1024.0
h_factor = float(h)/1024.0
try:
boxes, scores, labels = run_wbf(boxes_all,scores_all, image_index=i,name='wbf',iou_thr=0.40813015995026714, skip_box_thr=0.4133694759009949,
weights=[0.4,0.05,0.05,0.25,0.25])
 except Exception as e:
boxes, scores, labels = run_wbf(boxes_all,scores_all, image_index=i,name='wbf',iou_thr=0.395, skip_box_thr=0.393)
 print(e)
 boxes = boxes.round().clip(min=0, max=1023)
 boxes[:, [0,2]]*=w_factor
boxes[:, [1,3]]*=h_factor
image_id = image_ids[i]
boxes=boxes.astype(np.int32)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
df = pd.DataFrame(boxes,columns=['x','y','w','h'])
 df['image_id'] = image_id
df['source'] = 'test'
csv_pseudo.append(df )<save_to_csv>
data = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 prices = data['price_doc']
features_raw = data.drop(['price_doc','id'], axis =1 )
df_pseudo = pd.concat(csv_pseudo)[['image_id','x','y','w','h','source']]
df_pseudo.to_csv("/kaggle/working/pseudo.csv",index=False)
 df_pseudo.to_csv("/kaggle/working/train_spike_kaggle.csv",index=False)
 df_pseudo.head()<define_variables>
features_raw.fillna(0, inplace =True)
 scaler = MinMaxScaler(( 0,1e6))
numerical = features_raw.select_dtypes(include=['float64','int64'] ).keys()
features_raw[numerical] = scaler.fit_transform(features_raw[numerical])
 print(features_raw.head(n = 1))
class Config:
root_data = '/kaggle/working/data/train/'
csv = '/kaggle/working/train.csv'
pseudo_csv = '/kaggle/working/pseudo.csv'
fold_number = 2
num_workers = 4
batch_size = 2
grad_step = 1
n_epochs = 2
optimizer = torch.optim.SGD
lr = 0.002
SchedulerClass = torch.optim.lr_scheduler.CosineAnnealingLR
scheduler_params = dict(
 T_max=800, 
)
 verbose = True
verbose_step = 1
TrainMultiScale = [1.0]
print(TrainMultiScale)
 net_name = 'tf_efficientdet_d7'
checkpoint_name = '.. /efficientdet/tf_efficientdet_d7_53-6d1d7a95.pth'
CocoFormat=False
class data_config:
real = 0.3
mosaic = 0.0
cutmix = 0.00
stylized = 0.0
scale = 0.2
hsv = 0.2
if __name__=="__main__":
class opt:
fold=2
epochs=2
resume = '.. /input/effd7pth/best-checkpoint-027epoch.bin'
train=0 if not PSEUDO else 1
config = Config()
config.fold_number = opt.fold
config.resume = opt.resume
folder = 'effdet7_pseudo'
if not os.path.exists(folder):
os.makedirs(folder)
 config.folder = folder
config.n_epochs = opt.epochs
if not config.CocoFormat:
marking,df_folds,spike = train_utils.get_k_fols(config)
 pseudo_csv = pd.read_csv(config.pseudo_csv)
 index_pseudo = list(set(pseudo_csv.image_id))
train_csv = pd.concat([marking,pseudo_csv])
 image_ids=list(df_folds[df_folds['fold'] != config.fold_number].index.values)
 image_ids = image_ids+index_pseudo+index_pseudo
if opt.train==0:
image_ids = image_ids[0:10]
train_dataset = datasets_utils.train_wheat(image_ids=np.array(image_ids),
marking=train_csv,
data_config = data_config,
transforms=datasets_utils.get_train_transforms() ,
test=False,
TRAIN_ROOT_PATH=config.root_data)
 validation_dataset = datasets_utils.DatasetRetrieverTest(image_ids=df_folds[df_folds['fold'] == config.fold_number].index.values,
marking=marking,
transforms=datasets_utils.get_valid_transforms() ,
test=True,
TRAIN_ROOT_PATH=config.root_data)
 train_loader = torch.utils.data.DataLoader(
 train_dataset,
batch_size=config.batch_size,
sampler=RandomSampler(train_dataset),
pin_memory=False,
drop_last=True,
num_workers=config.num_workers,
collate_fn=datasets_utils.collate_fn 
)
 validation_loader = torch.utils.data.DataLoader(
 validation_dataset,
batch_size=1,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=datasets_utils.collate_fn 
)
 if len(config.TrainMultiScale)==1:
net = train_utils.get_net(type_net=config.net_name,checkpoint_name=config.checkpoint_name,resume = config.resume)
 else:
print(config.TrainMultiScale)
 net = train_utils.get_net_multiscle(type_net=config.net_name,checkpoint_name=config.checkpoint_name,resume = config.resume,multiScale=config.TrainMultiScale)
 net.cuda()
fitter = train_utils.Fitter(model=net, config=config)
 fitter.fit(train_loader, validation_loader )<load_pretrained>
objects = features_raw.select_dtypes(include=['object'] ).keys()
lbl = LabelEncoder()
for col in objects:
lbl.fit(list(features_raw[col].values))
features_raw[col] = lbl.transform(list(features_raw[col].values))
features = features_raw
features = features.drop(features.filter(regex='_no', axis=1),axis=1)
 features.drop('timestamp',axis=1,inplace =True)
 encoded = list(features.columns)
 print("{} total features after one-hot encoding.".format(len(encoded)))
 print(encoded )
torch.cuda.empty_cache()
net = load_net('/kaggle/working/effdet7_pseudo/last-checkpoint.bin' )<import_modules>
minimum_price = np.min(prices)
 maximum_price = np.max(prices)
 mean_price = np.mean(prices)
 median_price = np.median(prices)
 std_price = np.std(prices)
 print("Statistics for russian housing dataset:
")
 print("Minimum price: Rub {:,.2f}".format(minimum_price))
print("Maximum price: Rub {:,.2f}".format(maximum_price))
print("Mean price: Rub {:,.2f}".format(mean_price))
print("Median price Rub {:,.2f}".format(median_price))
print("Standard deviation of prices: Rub {:,.2f}".format(std_price))
alfa_c =(std_price/99)*1.96
print("critical alfa score for sample size of 100: Rub {:,.2f}".format(alfa_c))
from sklearn.model_selection import StratifiedKFold
from skopt import gp_minimize, forest_minimize
from skopt.utils import use_named_args
from skopt.plots import plot_objective, plot_evaluations, plot_convergence, plot_regret
from skopt.space import Categorical, Integer, Real
import os<compute_train_metric>
def performance_metric(y_true, y_predict):

 score = r2_score(y_true, y_predict)
 return score
def make_tta_predictions_val(net,images, score_threshold=0.0):
with torch.no_grad() :
x = torch.stack(images ).float().cuda()
predictions = []
img_size = x.shape[-2:]
s = [0.83, 0.67]
y = []
all_x =[]
for i, xi in enumerate(( x,
x.flip(3),
x.flip(2), 
)) :
xp = xi
all_x.append(xp)
 y.append(net(xp, torch.tensor([1]*xp.shape[0] ).float().cuda() ,1.0))
y[1][..., 0] = img_size[1] - y[1][..., 0] - y[1][..., 2]
y[2][..., 1] = img_size[0] - y[2][..., 1] - y[2][..., 3]
y = np.array(y)
 boxes_all = []
scores_all=[]
for j in range(len(y)) :
boxes = y[j][0].cpu().numpy() [:,:4].copy()
scores = y[j][0].cpu().numpy() [:,4].copy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes = boxes.round().clip(min=0, max=1023)
 boxes_all.append(boxes/1024)
 scores_all.append(scores[indexes])
 return boxes_all,scores_all,y<predict_on_test>
X_train, X_test, y_train, y_test = train_test_split(features,prices, test_size =0.2, random_state = 33)
 print("Training and testing split was successful." )
if True:
all_predictions = []
for images, targets, image_ids in tqdm(validation_loader, total=len(validation_loader)) :
with torch.no_grad() :
fold_predictions = {}
for fold_number in range(2,3):
all_bbox,all_score,_ = make_tta_predictions(images,0.0)
 for i in range(1):
image_predictions = {
'image_id': image_ids[i],
'gt_boxes':(targets[i]['boxes'].cpu().numpy() ).clip(min=0, max=1023 ).astype(int),
}
for fold_number in range(2,3):
image_predictions[f'pred_boxes_fold{fold_number}'] = all_bbox
image_predictions[f'scores_fold{fold_number}'] = all_score
all_predictions.append(image_predictions)
 if not PSEUDO:
break<categorify>
def ModelLearning(X, y):

 cv = ShuffleSplit(n_splits = 10, test_size = 0.2, random_state = 0)
 train_sizes = np.rint(np.linspace(1, X.shape[0]*0.8 - 1, 9)).astype(int)
 fig = pl.figure(figsize=(10,7))
for k, depth in enumerate([1,3,6,10]):
regressor = DecisionTreeRegressor(max_depth = depth)
 sizes, train_scores, test_scores = curves(regressor, X, y, \
cv = cv, train_sizes = train_sizes, scoring = 'r2')
 train_std = np.std(train_scores, axis = 1)
 train_mean = np.mean(train_scores, axis = 1)
 test_std = np.std(test_scores, axis = 1)
 test_mean = np.mean(test_scores, axis = 1)
 ax = fig.add_subplot(2, 2, k+1)
 ax.plot(sizes, train_mean, 'o-', color = 'r', label = 'Training Score')
 ax.plot(sizes, test_mean, 'o-', color = 'g', label = 'Testing Score')
 ax.fill_between(sizes, train_mean - train_std, \
train_mean + train_std, alpha = 0.15, color = 'r')
 ax.fill_between(sizes, test_mean - test_std, \
test_mean + test_std, alpha = 0.15, color = 'g')
 ax.set_title('max_depth = %s'%(depth))
ax.set_xlabel('Number of Training Points')
 ax.set_ylabel('Score')
 ax.set_xlim([0, X.shape[0]*0.8])
 ax.set_ylim([-0.05, 1.05])
 ax.legend(bbox_to_anchor=(1.05, 2.05), loc='lower left', borderaxespad = 0.)
fig.suptitle('Decision Tree Regressor Learning Performances', fontsize = 16, y = 1.03)
 fig.tight_layout()
fig.show()
def calculate_final_score(
 all_predictions,
iou_thr,
skip_box_thr=0.0,
score_threshold = 0.2,
method='nms',
sigma=0.5,weights=None 
):
final_scores = []
for i in tqdm(range(len(all_predictions))):
gt_boxes = all_predictions[i]['gt_boxes'].copy().astype(float)/1024
image_id = all_predictions[i]['image_id']
folds_boxes, folds_scores, folds_labels = [], [], []
for fold_number in range(2,3):
folds_boxes = all_predictions[i][f'pred_boxes_fold{fold_number}'].copy() [0:3]
folds_scores = all_predictions[i][f'scores_fold{fold_number}'].copy() [0:3]
folds_boxes_new = []
folds_scores_new = []
for bb in range(len(folds_boxes)) :
s = folds_scores[bb].copy()
b = folds_boxes[bb].copy()
indexes = np.where(s > score_threshold)[0]
new_bbox = b[indexes]
new_score = s[indexes]
folds_boxes_new.append(new_bbox)
 folds_scores_new.append(new_score)
 folds_boxes = folds_boxes_new
folds_scores = folds_scores_new
folds_labels = [np.ones([row.shape[0]])for row in folds_scores_new]
if method == 'weighted_boxes_fusion':
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'nms':
try:
boxes, scores, labels = nms(folds_boxes, folds_scores, folds_labels, weights=weights, iou_thr=iou_thr)
 except:
boxes, scores, labels = weighted_boxes_fusion(folds_boxes, folds_scores, folds_labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 elif method == 'soft_nms':
boxes, scores, labels = soft_nms(folds_boxes, folds_scores, folds_labels, weights=weights, iou_thr=iou_thr, thresh=skip_box_thr, sigma=sigma)
 elif method == 'non_maximum_weighted':
boxes, scores, labels = non_maximum_weighted(folds_boxes, folds_scores, folds_labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
raise
image_precision = calculate_image_precision(gt_boxes, boxes, thresholds=iou_thresholds, form='pascal_voc')
 final_scores.append(image_precision)
 return np.mean(final_scores )<compute_train_metric>
ModelLearning(features, prices )
print('[WBF]: ', calculate_final_score(
 all_predictions,
iou_thr=0.43312889428044965,
skip_box_thr=0.393358169307333,
score_threshold = 0.4024215973594928,
method='weighted_boxes_fusion',weights=None 
))<train_on_grid>
ModelComplexity(X_train, y_train )
def log(text):
with open('opt.log', 'a+')as logger:
logger.write(f'{text}
')
 def optimize(space, all_predictions, method, n_calls=10):
@use_named_args(space)
 def score(**params):
print('-'*5 + f'{method}' + '-'*5)
 print(params)
 final_score = calculate_final_score(all_predictions, method=method, **params)
 print(f'final_score = {final_score}')
 print('-'*10)
 return -final_score
return gp_minimize(func=score, dimensions=space, n_calls=n_calls )<train_on_grid>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(features, prices )
space = [
Real(0.3, 0.75, name='iou_thr'),
Real(0.2, 0.5, name='score_threshold'),
Real(0.3, 0.6, name='skip_box_thr'),
]
if True:
opt_result = optimize(
 space,
all_predictions,
method='weighted_boxes_fusion',
n_calls=10 if not PSEUDO else 50,
)<find_best_params>
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=20,
verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()
best_iou_thr = opt_result.x[0]
best_score_thr = opt_result.x[1]
best_skip_box_thr = opt_result.x[2]<init_hyperparams>
num_boost_rounds = len(cv_output)
 model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round= num_boost_rounds )
results = []
flag = True
for shape,images, image_ids in data_loader:
boxes_all,scores_all,y = make_tta_predictions(images,score_threshold=best_score_thr)
 for i, image in enumerate(images):
h,w,_ = shape[i]
w_factor = float(w)/1024.0
h_factor = float(h)/1024.0
try:
boxes, scores, labels = run_wbf(boxes_all,scores_all, image_index=i,name='wbf',iou_thr=best_iou_thr, skip_box_thr=best_skip_box_thr,
weights=[0.4,0.05,0.05,0.25,0.25])
 except Exception as e:
print(e)
 try:
boxes, scores, labels = run_wbf(boxes_all,scores_all, image_index=i,name='wbf',iou_thr=0.43, skip_box_thr=0.4)
 except Exception as e:
print(e)
 boxes, scores = boxes[0]*1023, scores[0]
boxes = boxes.round().clip(min=0, max=1023)
 boxes[:, [0,2]]*=w_factor
boxes[:, [1,3]]*=h_factor
image_id = image_ids[i]
boxes=boxes.astype(np.int32)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
if flag:
plt.figure(figsize=[10,10])
 sample = norm(image.permute(1,2,0 ).cpu().numpy())
 for box in boxes:
cv2.rectangle(sample,(box[0], box[1]),(box[0]+box[2], box[1]+box[3]),(1, 0, 0), 1)
 plt.imshow(sample)
 plt.show()
flag=False
result = {
'image_id': image_id,
'PredictionString': format_prediction_string(boxes, scores)
 }
results.append(result )<save_to_csv>
test_data = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 id_test = test_data.id
test_features_raw = test_data.drop(['id'], axis =1)
 test_features_raw.fillna(0, inplace =True)
 test_features_raw[numerical] = scaler.transform(test_features_raw[numerical])
 for col in objects:
lbl.fit(list(test_features_raw[col].values))
test_features_raw[col] = lbl.transform(list(test_features_raw[col].values))
test_features = test_features_raw
test_features = test_features.drop(test_features.filter(regex='_no', axis=1),axis=1)
 test_features.drop('timestamp',axis=1,inplace =True)
 Test_matrix = xgb.DMatrix(test_features )
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<install_modules>
y_pred = model.predict(Test_matrix)
 df_submit = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_submit.to_csv('submit.csv', index=False )