Kaggle competition: Sberbank Russian Housing Market
<SOS> metric: RMSLE Kaggle data source: sberbank-russian-housing-market<define_variables>
color = sns.color_palette()
%matplotlib inline
warnings.filterwarnings("ignore")
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 50 )
WEIGHT_FILE1 = '.. /input/yolov5multiscale/best_yolov5x_fold0.pt'
WEIGHT_FILE2 = '.. /input/yolov5multiscale/best_yolov5x_fold1.pt'
DIR_TEST = '.. /input/global-wheat-detection/test/'<init_hyperparams>
train_zip = zipfile.ZipFile('/kaggle/input/sberbank-russian-housing-market/train.csv.zip')
 test_zip = zipfile.ZipFile('/kaggle/input/sberbank-russian-housing-market/test.csv.zip')
 sample_submit_zip = zipfile.ZipFile('/kaggle/input/sberbank-russian-housing-market/sample_submission.csv.zip')
 macro_zip = zipfile.ZipFile('/kaggle/input/sberbank-russian-housing-market/macro.csv.zip')
 df_train = pd.read_csv(train_zip.open('train.csv'))
df_test = pd.read_csv(test_zip.open('test.csv'))
sample_submit = pd.read_csv(sample_submit_zip.open('sample_submission.csv'))
df_macro = pd.read_csv(macro_zip.open('macro.csv'))
print("Train shape:", df_train.shape)
 print("Test shape:", df_test.shape )
%%time
def detect(save_img=False):
weights, imgsz = opt.weights,opt.img_size
source = '.. /input/global-wheat-detection/test/'
device = torch_utils.select_device(opt.device)
 half = False
models = []
for w in weights:
models.append(torch.load(w, map_location=device)['model'].to(device ).float().eval())
 dataset = LoadImages(source, img_size=1024)
 t0 = time.time()
img = torch.zeros(( 1, 3, imgsz, imgsz), device=device)
 all_path=[]
all_bboxex =[]
all_score =[]
for path, img, im0s, vid_cap in dataset:
img = torch.from_numpy(img ).to(device)
 img = img.half() if half else img.float()
img /= 255.0
if img.ndimension() == 3:
img = img.unsqueeze(0)
 t1 = torch_utils.time_synchronized()
bboxes_2 = []
score_2 = []
for model in models:
pred = model(img, augment=opt.augment)[0]
pred = non_max_suppression(pred, 0.4, opt.iou_thres, merge=True, classes=None, agnostic=False)
 t2 = torch_utils.time_synchronized()
bboxes = []
score = []
for i, det in enumerate(pred):
p, s, im0 = path, '', im0s
gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]
if det is not None and len(det):
det[:, :4] = scale_coords(img.shape[2:], det[:, :4], im0.shape ).round()
for c in det[:, -1].unique() :
n =(det[:, -1] == c ).sum()
for *xyxy, conf, cls in det:
if True:
xywh = torch.tensor(xyxy ).view(-1 ).numpy()
bboxes.append(xywh)
 score.append(conf)
 bboxes_2.append(bboxes)
 score_2.append(score)
 all_path.append(path)
 all_score.append(score_2)
 all_bboxex.append(bboxes_2)
 return all_path,all_score,all_bboxex
class opt:
weights_folds = [WEIGHT_FILE1]
weights_folds.sort()
weights = weights_folds
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()<compute_train_metric>
df_train['price_doc_log'] = np.log(df_train['price_doc'] )
def run_wbf(boxes,scores, image_size=1024, iou_thr=0.4, skip_box_thr=0.4, weights=None):
labels0 = [np.ones(len(scores[idx])) for idx in range(len(scores)) ]
boxes, scores, labels = weighted_boxes_fusion(boxes, scores, labels0, weights=None, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 return boxes, scores, labels<define_variables>
df_train[(df_train['floor'])== 33]
all_path,all_score,all_bboxex = res<categorify>
df_train.drop(df_train.index[7457], inplace=True )
results =[]
testdf_psuedo = []
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'kaushal28'+image_id,
'source': 'kaushal28',
'x': float(box[0]),
'y': float(box[1]),
'w': float(box[2]),
'h': float(box[3]),
'x_center': float(box[0])+ float(box[2])/2,
'y_center': float(box[1])+ float(box[3])/2,
'classes': 0
}
testdf_psuedo.append(result)
<prepare_output>
df_train['year'] = df_train['timestamp'].apply(lambda x: x[:4] ).astype(int)
 df_train['month'] = df_train['timestamp'].apply(lambda x: x[5:7] ).astype(int)
 df_test['year'] = df_test['timestamp'].apply(lambda x: x[:4] ).astype(int)
 df_test['month'] = df_test['timestamp'].apply(lambda x: x[5:7] ).astype(int )
test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'x', 'y', 'w', 'h', 'x_center','y_center','classes', 'source'])
 test_df_pseudo.head()<feature_engineering>
missingValues = df_train.columns[df_train.isnull().any() ].tolist()
pd.isnull(df_train[missingValues] ).sum().sort_values(ascending=False )
train_df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(train_df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
train_df[column] = bboxs[:,i]
train_df.drop(columns=['bbox'], inplace=True)
 train_df['x_center'] = train_df['x'] + train_df['w']/2
train_df['y_center'] = train_df['y'] + train_df['h']/2
train_df['classes'] = 0
train_df = train_df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes', 'source']]
train_df.head()<concatenate>
cols_fillna_mode = ['floor',
'product_type',
'num_room',
'state',
'hospital_beds_raion',
'build_count_brick',
'build_count_monolith',
'green_part_2000']
cols_fillna_mean = ['life_sq',
'metro_min_walk',
'metro_km_walk',
'railroad_station_walk_km',
'railroad_station_walk_min',
'cafe_sum_1500_min_price_avg',
'cafe_sum_1500_max_price_avg',
'cafe_avg_price_1500',
'cafe_sum_2000_max_price_avg',
'cafe_avg_price_2000']
train_df = pd.concat([train_df, test_df_pseudo], ignore_index=True)
 train_df<define_variables>
for col in cols_fillna_mode:
df_train[col].fillna(df_train[col].mode().iloc[0],inplace=True)
 df_test[col].fillna(df_train[col].mode().iloc[0],inplace=True)
 for col in cols_fillna_mean:
df_train[col].fillna(df_train[col].mean() ,inplace=True)
 df_test[col].fillna(df_train[col].mean() ,inplace=True )
%%writefile yolov5x.yaml
nc: 1
depth_multiple: 1.33
width_multiple: 1.25
anchors:
- [116,90, 156,198, 373,326]
- [30,61, 62,45, 59,119]
- [10,13, 16,30, 33,23]
backbone:
[[-1, 1, Focus, [64, 3]],
[-1, 1, Conv, [128, 3, 2]],
[-1, 3, BottleneckCSP, [128]],
[-1, 1, Conv, [256, 3, 2]],
[-1, 9, BottleneckCSP, [256]],
[-1, 1, Conv, [512, 3, 2]],
[-1, 9, BottleneckCSP, [512]],
[-1, 1, Conv, [1024, 3, 2]],
[-1, 1, SPP, [1024, [5, 9, 13]]],
]
head:
[[-1, 3, BottleneckCSP, [1024, False]],
[-1, 1, Conv, [512, 1, 1]],
[-1, 1, nn.Upsample, [None, 2, 'nearest']],
[[-1, 6], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [512, False]],
[-1, 1, Conv, [256, 1, 1]],
[-1, 1, nn.Upsample, [None, 2, 'nearest']],
[[-1, 4], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [256, False]],
[-1, 1, nn.Conv2d, [na *(nc + 5), 1, 1]],
[-2, 1, Conv, [256, 3, 2]],
[[-1, 14], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [512, False]],
[-1, 1, nn.Conv2d, [na *(nc + 5), 1, 1]],
[-2, 1, Conv, [512, 3, 2]],
[[-1, 10], 1, Concat, [1]],
[-1, 3, BottleneckCSP, [1024, False]],
[-1, 1, nn.Conv2d, [na *(nc + 5), 1, 1]],
[[], 1, Detect, [nc, anchors]],
]<define_variables>
numerical_features = df_train.dtypes[df_train.dtypes != "object"].index
categorical_features = df_train.dtypes[df_train.dtypes == "object"].index
print("Кол-во количественных признаков: ", len(numerical_features))
print("Кол-во категориальных признаков: ", len(categorical_features))
yolo_config = {
'epochs': 37,
'image_size': 1024,
'batch_size': 2,
'data_config': 'wheat_data_config.yaml',
'yolo_config': 'yolov5x.yaml',
'results_name': f'yolov5x_fold{FOLD}',
'weights': WEIGHT_FILE1
}<set_options>
df_train.isna().sum().sort_values(ascending=False )
if len(os.listdir(DIR_TEST)) > 10:
!(python train.py --img-size {yolo_config.get('image_size')} \
--batch-size {yolo_config.get('batch_size')} \
--epochs {yolo_config.get('epochs')} \
--data {yolo_config.get('data_config')} \
--cfg {yolo_config.get('yolo_config')} \
--name {yolo_config.get('results_name')} \
--weights {yolo_config.get('weights')} )<set_options>
df_train.drop(['id', 'price_doc', 'timestamp'], axis=1, inplace=True)
 id_test = df_test['id']
df_test.drop(['id', 'timestamp'], axis=1, inplace=True )
torch.cuda.empty_cache()
!rm -rf convertor<set_options>
numerical_features = df_train.dtypes[df_train.dtypes != "object"].index
categorical_features = df_train.dtypes[df_train.dtypes == "object"].index
print("Кол-во количественных признаков: ", len(numerical_features))
print("Кол-во категориальных признаков: ", len(categorical_features))
class opt:
weights_folds = [WEIGHT_FILE2]
weights_folds.sort()
weights = weights_folds
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()<define_variables>
encoder = OneHotEncoder(handle_unknown='error')
 encoder_cols_train = pd.DataFrame(encoder.fit_transform(df_train[categorical_features] ).toarray())
 encoder_cols_test = pd.DataFrame(encoder.transform(df_test[categorical_features] ).toarray() )
all_path,all_score,all_bboxex = res<categorify>
encoder_cols_train.columns = encoder.get_feature_names(categorical_features)
 encoder_cols_test.columns = encoder.get_feature_names(categorical_features)
 encoder_cols_train.index = df_train.index
encoder_cols_test.index = df_test.index
results =[]
testdf_psuedo = []
for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
for box in boxes:
result = {
'image_id': 'kaushal28'+image_id,
'source': 'kaushal28',
'x': float(box[0]),
'y': float(box[1]),
'w': float(box[2]),
'h': float(box[3]),
'x_center': float(box[0])+ float(box[2])/2,
'y_center': float(box[1])+ float(box[3])/2,
'classes': 0
}
testdf_psuedo.append(result )<prepare_output>
num_df_train = df_train.drop(categorical_features, axis=1)
 num_df_test = df_test.drop(categorical_features, axis=1 )
test_df_pseudo = pd.DataFrame(testdf_psuedo, columns=['image_id', 'x', 'y', 'w', 'h', 'x_center','y_center','classes', 'source'])
 test_df_pseudo.head()<feature_engineering>
df_train_encoded = pd.concat([num_df_train, encoder_cols_train], axis=1)
 df_test_encoded = pd.concat([num_df_test, encoder_cols_test], axis=1)
 print("Train dataset shape:", df_train_encoded.shape)
 print("Test dataset shape:", df_test_encoded.shape )
train_df = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(train_df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
train_df[column] = bboxs[:,i]
train_df.drop(columns=['bbox'], inplace=True)
 train_df['x_center'] = train_df['x'] + train_df['w']/2
train_df['y_center'] = train_df['y'] + train_df['h']/2
train_df['classes'] = 0
train_df = train_df[['image_id','x', 'y', 'w', 'h','x_center','y_center','classes', 'source']]
train_df.head()<concatenate>
df_train_encoded.median().sort_values(ascending=False )
train_df = pd.concat([train_df, test_df_pseudo], ignore_index=True)
 train_df<load_pretrained>
X = df_train_encoded.drop(['price_doc_log'], axis=1)
 y = df_train_encoded['price_doc_log']
print("X shape:", X.shape)
 print("y shape:", y.shape )
torch.cuda.empty_cache()
yolo_config = {
'epochs': 39,
'image_size': 1024,
'batch_size': 2,
'data_config': 'wheat_data_config.yaml',
'yolo_config': 'yolov5x.yaml',
'results_name': f'yolov5x_fold{FOLD}',
'weights': WEIGHT_FILE2
}<set_options>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=2022)
 X_test = df_test_encoded
if len(os.listdir(DIR_TEST)) > 10:
!(python train.py --img-size {yolo_config.get('image_size')} \
--batch-size {yolo_config.get('batch_size')} \
--epochs {yolo_config.get('epochs')} \
--data {yolo_config.get('data_config')} \
--cfg {yolo_config.get('yolo_config')} \
--name {yolo_config.get('results_name')} \
--weights {yolo_config.get('weights')} )<set_options>
tree = DecisionTreeRegressor(random_state=2022, max_depth=5, min_samples_split=20)
 tree.fit(X_train, y_train)
 tree_predictions_log = tree.predict(X_val)
 tree_predictions = np.exp(tree_predictions_log )
class opt:
weights_folds = list(glob.glob("weights/best_*")) if len(os.listdir(DIR_TEST)) > 10 else [WEIGHT_FILE1, WEIGHT_FILE2]
weights_folds.sort()
weights = weights_folds
img_size = 1024
conf_thres = 0.1
iou_thres = 0.94
augment = True
device = '0'
classes=None
agnostic_nms = True
opt.img_size = check_img_size(opt.img_size)
 with torch.no_grad() :
res = detect()<define_variables>
print('RMSLE:', np.sqrt(mean_squared_log_error(np.exp(y_val), tree_predictions)) )
all_path,all_score,all_bboxex = res<categorify>
predict = np.exp(tree.predict(X_test))
submission = pd.DataFrame({'id': id_test, 'price_doc': predict})
 submission.head()
results =[]
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 for row in range(len(all_path)) :
image_id = all_path[row].split("/")[-1].split(".")[0]
boxes = all_bboxex[row]
scores = all_score[row]
boxes, scores, labels = run_wbf(boxes,scores)
 boxes =(boxes*1024/1024 ).astype(np.int32 ).clip(min=0, max=1023)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
result = {'image_id': image_id,'PredictionString': format_prediction_string(boxes, scores)}
results.append(result)
 test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'] )<save_to_csv>
submission.to_csv('DecisionTree.csv', index=False )
test_df.to_csv('submission.csv', index=False)
 test_df.head()<load_pretrained>
dmatrix_train = xgb.DMatrix(X_train, y_train)
 dmatrix_val = xgb.DMatrix(X_val, y_val)
 dmatrix_test = xgb.DMatrix(X_test )
%load_ext autoreload
%autoreload 2<set_options>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 1.0,
'colsample_bytree': 0.7,
'objective': 'reg:squarederror',
'eval_metric': 'rmse',
'verbosity': 0
}
partial_model = xgb.train(xgb_params, dmatrix_train, num_boost_round=1000, evals=[(dmatrix_val, 'val')],
early_stopping_rounds=20, verbose_eval=20)
 num_boost_round = partial_model.best_iteration
sys.path.insert(0, ".. /input/weightedboxesfusion/")
 %matplotlib inline
gc.enable()
np.set_printoptions(suppress=True)
 print('PyTorch version', torch.__version__)
 SEED = 1120
image_width = 1024
image_height = 1024
batch=64
subdivisions=32
burn_in = 0
prob_threhold = 0.5
iou_thr = 0.55
skip_box_thr = 0.1
model_image_sizes = [1024, 1024, 1024, 1024, 1024]
dataset_folder = ".. /input/global-wheat-detection/"
train_image_folder = ".. /input/global-wheat-detection/train"
test_image_folder = ".. /input/global-wheat-detection/test"
meta_file = b"data/wheat.data"
model_cfg_files = [
b"./yolov4-v3-20000iter-fold2/yolov4-mish-416-wheat-v3-fold2.cfg",
]
model_weights_files = [
b"./yolov4-v3-20000iter-fold2/yolov4-mish-416-wheat-v3-fold2_best.weights",
]
USE_OPTIMIZE = len(os.listdir(test_image_folder)) == 10
if USE_OPTIMIZE:
iteration_offset = 0
pseudo_iterations = 10
else:
iteration_offset = 0
pseudo_iterations = 1000
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(SEED )<define_variables>
model = xgb.train(dict(xgb_params, verbose=1), dmatrix_train, num_boost_round=num_boost_round )
bad_images = [
"41c0123cc",
"a1321ca95",
"2cc75e9f5",
"42e6efaaa",
"409a8490c",
"d067ac2b1",
"d60e832a5",
"893938464",
]
bad_boxes = [
3687, 117344, 173, 113947, 52868, 2159, 2169, 121633, 121634, 147504,
118211, 52727, 147552
]<feature_engineering>
predict = np.exp(model.predict(dmatrix_val))
print('RMSLE:', np.sqrt(mean_squared_log_error(np.exp(y_val), predict)) )
marking_df = pd.read_csv(f"{dataset_folder}/train.csv")
 marking_df['bbox'] = marking_df.bbox.fillna('[0,0,0,0]')
 bboxs = np.stack(
 marking_df['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking_df[column] = bboxs[:, i]
marking_df.drop(columns=['bbox'], inplace=True)
 marking_df["id"] = marking_df.index.tolist()
marking_df['x_center'] = marking_df['x'] + marking_df['w'] / 2
marking_df['y_center'] = marking_df['y'] + marking_df['h'] / 2
marking_df['classes'] = 0<choose_model_class>
ylog_pred = model.predict(dmatrix_test)
 y_pred = np.exp(ylog_pred)
 submission = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 submission.head()
n_folds = 5
skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=SEED)
 skf<data_type_conversions>
submission.to_csv("XGB_new_clear_submission.csv", index=False )
df_folds = dataset_df[['image_id']].copy()
df_folds.loc[:, 'bbox_count'] = 1
df_folds = df_folds.groupby('image_id' ).count()
df_folds.loc[:, 'source'] = dataset_df[['image_id', 'source'
]].groupby('image_id' ).min() ['source']
df_folds.loc[:, 'stratify_group'] = np.char.add(
 df_folds['source'].values.astype(str),
df_folds['bbox_count'].apply(lambda x: f'_{x // 15}' ).values.astype(str))
df_folds.loc[:, 'fold'] = 0
for fold_number,(train_index, val_index)in enumerate(
 skf.split(X=df_folds.index, y=df_folds['stratify_group'])) :
df_folds.loc[df_folds.iloc[val_index].index, 'fold'] = fold_number
df_folds = df_folds.reset_index()<feature_engineering>
dmatrix_train = xgb.DMatrix(X_train, y_train)
 dmatrix_test = xgb.DMatrix(X_test )
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(
 j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings)
 def process_detection_result(bboxes, resize_width, prob_threhold=0.5):
scores = np.zeros(( len(bboxes)))
 numpy_bboxes = np.zeros(( len(bboxes), 4))
for i, bbox in enumerate(bboxes):
scores[i] = bbox[1]
cord = bbox[2]
x1 = cord[0] - cord[2] / 2
y1 = cord[1] - cord[3] / 2
x2 = cord[0] + cord[2] / 2
y2 = cord[1] + cord[3] / 2
x1 /= resize_width
y1 /= resize_width
x2 /= resize_width
y2 /= resize_width
x1 *= image_width
y1 *= image_width
x2 *= image_width
y2 *= image_width
x1 = int(round(max(min(x1, image_width - 1), 0)))
 y1 = int(round(max(min(y1, image_width - 1), 0)))
 x2 = int(round(max(min(x2, image_width - 1), 0)))
 y2 = int(round(max(min(y2, image_width - 1), 0)))
 numpy_bboxes[i, :] = np.array([x1, y1, x2, y2])
 indexes = np.where(scores > prob_threhold)
 numpy_bboxes = numpy_bboxes[indexes]
scores = scores[indexes]
return scores, numpy_bboxes<categorify>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:squarederror',
'eval_metric': 'rmse',
'verbosity': 0
}
class BaseWheatTTA:

 image_size = image_width
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
image = np.fliplr(image)
 return image
def batch_augment(self, images):
images = np.fliplr(images)
 return images
def deaugment_boxes(self, boxes):
boxes[:, [0, 2]] = self.image_size - boxes[:, [2, 0]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
image = np.flipud(image)
 return image
def batch_augment(self, images):
images = np.flipud(images)
 return images
def deaugment_boxes(self, boxes):
boxes[:, [3, 1]] = self.image_size - boxes[:, [1, 3]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
image = np.rot90(image, k=1, axes=(0, 1))
return image
def batch_augment(self, images):
images = np.rot90(images, k=1, axes=(1, 2))
return images
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0, 2]] = self.image_size - boxes[:, [1, 3]]
res_boxes[:, [1, 3]] = boxes[:, [2, 0]]
return res_boxes
class TTARotate180(BaseWheatTTA):
def augment(self, image):
tmp = np.rot90(image, k=1, axes=(0, 1))
tmp = np.rot90(tmp, k=1, axes=(0, 1))
return tmp
def batch_augment(self, images):
tmp = np.rot90(images, k=1, axes=(1, 2))
tmp = np.rot90(tmp, k=1, axes=(1, 2))
return tmp
def deaugment_boxes(self, boxes):
tmp = TTARotate90().deaugment_boxes(boxes)
 return TTARotate90().deaugment_boxes(tmp)
 class TTARotate270(BaseWheatTTA):
def augment(self, image):
tmp = TTARotate180().augment(image)
 tmp = np.rot90(tmp, k=1, axes=(0, 1))
return tmp
def batch_augment(self, images):
tmp = TTARotate180().batch_augment(images)
 tmp = np.rot90(tmp, k=1, axes=(1, 2))
return tmp
def deaugment_boxes(self, boxes):
tmp = TTARotate180().deaugment_boxes(boxes)
 return TTARotate90().deaugment_boxes(tmp)
 class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:, 0] = np.min(boxes[:, [0, 2]], axis=1)
 result_boxes[:, 2] = np.max(boxes[:, [0, 2]], axis=1)
 result_boxes[:, 1] = np.min(boxes[:, [1, 3]], axis=1)
 result_boxes[:, 3] = np.max(boxes[:, [1, 3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<load_pretrained>
cv_output = xgb.cv(xgb_params,
dmatrix_train,
num_boost_round=1000,
early_stopping_rounds=20,
verbose_eval=50,
show_stdv=False)
 num_boost_rounds = len(cv_output )
<data_type_conversions>
model = xgb.train(dict(xgb_params, verbose=1), dmatrix_train, num_boost_round=num_boost_rounds )
def run_wbf(preds, image_size=image_width,
iou_thr=0.55, skip_box_thr=0.7, weights=None):
boxes = [(p['boxes'] /(image_size-1)).tolist() for p in preds]
scores = [p['scores'].tolist() for p in preds]
labels = [np.ones(p['scores'].shape[0] ).astype(int ).tolist() for p in preds]
boxes, scores, labels = ensemble_boxes_wbf.weighted_boxes_fusion(
 boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<concatenate>
predict = np.exp(model.predict(dmatrix_test))
submission = pd.DataFrame({'id': id_test, 'price_doc': predict})
 submission.head()
tta_transforms = []
for tta_combination in product([TTAHorizontalFlip() , None],
[TTAVerticalFlip() , None],
[TTARotate90() , None]):
tta_transforms.append(
 TTACompose([
tta_transform for tta_transform in tta_combination if tta_transform
]))<predict_on_test>
submission.to_csv('XGB_CV.csv', index=False )
def detect(net, meta, darknet_image, target_image, prob_threshold=.5):
dn.copy_image_from_bytes(darknet_image, target_image.tobytes())
 return dn.detect_image(net,
meta,
darknet_image,
thresh=prob_threshold,
hier_thresh=.5,
nms=.45)
 def predict_test(batch_size=1, prob_threhold=0.5):
image_names = os.listdir(test_image_folder)
 image_pred_results = collections.defaultdict(list)
 for model_index, cfg in enumerate(model_cfg_files):
print(f"Generating inference for model cfg file {cfg}...... ")
 weights = model_weights_files[model_index]
net = dn.load_net_custom(cfg, weights, 0, batch_size)
 meta = dn.load_meta(meta_file)
 resize = model_image_sizes[model_index]
darknet_image = dn.make_image(resize, resize, 3)
 for name in image_names:
image_id = name.split('.')[0]
image = cv2.imread(f'{test_image_folder}/{image_id}.jpg')
 image = cv2.resize(image,(resize, resize))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 infer_start = time.time()
for tta in tta_transforms:
tta_image = tta.augment(image)
 preds = detect(net, meta, darknet_image, tta_image)
 scores, preds = process_detection_result(
 preds, resize, prob_threhold)
 preds = tta.deaugment_boxes(preds)
 preds = preds.clip(min=0, max=image_width - 1)
 image_pred_results[name].append({
'boxes': preds,
'scores': scores,
})
 print(
 f"Time spent on augmented inference for {name}: {time.time() - infer_start:2f} seconds" 
)
 del image, preds
gc.collect()
dn.free_network_ptr(net)
 del net, meta
gc.collect()
return image_pred_results
def ensemble(image_pred_results, iou_thr=0.5, skip_box_thr=0.1):
ensemble_results = {}
image_names = os.listdir(test_image_folder)
 for name in image_names:
print(f"Running ensemble by WBF for {name}...... ")
 image_id = name.split('.')[0]
tta_preds = image_pred_results[name]
boxes, scores, labels = run_wbf(tta_preds,
iou_thr=iou_thr,
skip_box_thr=skip_box_thr)
 boxes = boxes.round().astype(np.int32 ).clip(min=0, max=image_width - 1)
 boxes[:, 2] = boxes[:, 2] - boxes[:, 0]
boxes[:, 3] = boxes[:, 3] - boxes[:, 1]
ensemble_results[name] = {
'boxes': boxes,
'scores': scores,
}
return ensemble_results
def generate_submit(ensemble_results):
results = []
count = 0
image_names = os.listdir(test_image_folder)
 for name in image_names:
image_id = name.split('.')[0]
if len(image_names)< 11:
image = cv2.imread(f'{test_image_folder}/{image_id}.jpg')
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 r = ensemble_results[name]
boxes, scores = r["boxes"], r["scores"]
results.append({
'image_id':
image_id,
'PredictionString':
format_prediction_string(boxes, scores)
 })
 if len(image_names)< 11 and count < 10:
fig, ax = plt.subplots(1, 1, figsize=(20, 10))
for box, score in zip(boxes, scores):
cv2.rectangle(image,(box[0], box[1], box[2], box[3]),
color=(99, 228, 255),
thickness=4)
 cv2.putText(
 image,
f"{score:.2f}", 
(box[0], box[1] + box[3] - 5),
cv2.FONT_HERSHEY_SIMPLEX,
0.8, 
(255, 255, 255),
2,
cv2.LINE_AA)
 ax.imshow(image)
 count += 1
return results<predict_on_test>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=2022)
 X_test = df_test_encoded
pred_start = time.time()
test_preds = predict_test(batch_size=1, prob_threhold=prob_threhold)
 print(
 f"Time spent on generating submission predictions: {time.time() - pred_start:2f} seconds"
)<train_model>
pca = PCA(n_components=20 ).fit(X_train)
 X_train_pca=pca.transform(X_train)
 X_val_pca=pca.transform(X_val )
pred_start = time.time()
ensemble_results = ensemble(test_preds,
iou_thr=iou_thr,
skip_box_thr=skip_box_thr)
 print(f"Time spent on ensemble: {time.time() - pred_start:2f} seconds" )<save_to_csv>
dmatrix_train = xgb.DMatrix(X_train_pca, y_train)
 dmatrix_val = xgb.DMatrix(X_val_pca, y_val)
 dmatrix_test = xgb.DMatrix(X_test )
with open("./data/wheat.names", 'w')as out:
out.write("wheat
" )<concatenate>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 1.0,
'colsample_bytree': 0.7,
'objective': 'reg:squarederror',
'eval_metric': 'rmse',
'verbosity': 0
}
partial_model = xgb.train(xgb_params,
dmatrix_train,
num_boost_round=1000,
evals=[(dmatrix_val, 'val')],
early_stopping_rounds=20,
verbose_eval=20)
 num_boost_round = partial_model.best_iteration
!cat./data/wheat.names<save_to_csv>
model = xgb.train(dict(xgb_params, verbose=1), dmatrix_train, num_boost_round=num_boost_round )
with open(f"./data/wheat_pseudo.data", 'w')as out:
out.write("classes= 1
")
 out.write(f"train = data/train_wheat_pseudo.txt
")
 out.write(f"valid = data/valid_wheat_pseudo.txt
")
 out.write("names = data/wheat.names
")
 out.write("backup = backup" )<concatenate>
predict = np.exp(model.predict(dmatrix_val))
rmsle = np.sqrt(mean_squared_log_error(np.exp(y_val), predict))
print('RMSLE: {:.3f}'.format(rmsle))
!cat./data/wheat_pseudo.data<filter>
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=2022)
 X_test = df_test_encoded
fold_id = 0
fold_train_df, fold_val_df = df_folds[~(df_folds["fold"] == fold_id)].copy(
), df_folds[df_folds["fold"] == fold_id].copy()
assert fold_train_df.shape[0] + fold_val_df.shape[0] == df_folds.shape[0]
fold_train_df.shape[0], fold_val_df.shape[0]<save_to_csv>
train_dataset = cb.Pool(X_train, y_train)
 test_dataset = cb.Pool(X_val, y_val )
with open(f"./data/valid_wheat_pseudo.txt", 'w')as out:
image_ids_with_head = fold_val_df["image_id"].unique()
for image_id in image_ids_with_head:
sh.copy(f"{train_image_folder}/{image_id}.jpg",
f"./data/wheat/{image_id}.jpg")
 out.write(f"./data/wheat/{image_id}.jpg
" )<save_to_csv>
model = cb.CatBoostRegressor(loss_function='RMSE' )
for image_id in df_folds["image_id"].unique() :
df = dataset_df[dataset_df["image_id"] == image_id].copy()
with open(f"./data/wheat/{image_id}.txt", 'w')as out:
for index, row in df.iterrows() :
if row['x_center'] > 0 and row[
'y_center'] and row['w'] > 0 and row['h'] > 0:
out.write(f"{row['classes']} " +
f"{row['x_center']/image_width} " +
f"{row['y_center']/image_height} " +
f"{row['w']/image_width} " +
f"{row['h']/image_height}
" )<feature_engineering>
pred = np.exp(model.predict(X_val))
rmsle = np.sqrt(mean_squared_log_error(np.exp(y_val), pred))
print('RMSLE: {:.3f}'.format(rmsle))
for name in os.listdir(test_image_folder):
image_id = name.split('.')[0]
with open(f"./data/wheat/{image_id}.txt", 'w')as out:
r = ensemble_results[name]
boxes, scores = r["boxes"], r["scores"]
boxes[:, 0] = boxes[:, 0] + boxes[:, 2] / 2
boxes[:, 1] = boxes[:, 1] + boxes[:, 3] / 2
print(boxes.shape)
 for i in range(boxes.shape[0]):
box = boxes[i, :]
if box[0] > 0 and box[1] > 0 and box[2] > 0 and box[3] > 0:
out.write(f"0 " + f"{box[0]/image_width} " +
f"{box[1]/image_height} " +
f"{box[2]/image_width} " +
f"{box[3]/image_height}
" )<load_from_csv>
predict = np.exp(model.predict(X_test))
submission = pd.DataFrame({'id': id_test, 'price_doc': predict})
 submission.head()
<load_from_zip>
submission.to_csv('Catboost.csv', index=False )