Kaggle competition: Sberbank Russian Housing Market
<SOS> metric: RMSLE Kaggle data source: sberbank-russian-housing-market<save_to_csv>
import numpy as np
import pandas as pd
import xgboost as xgb
import matplotlib.pyplot as plt
test_df = pd.DataFrame(results, columns=['image_id', 'PredictionString'])
 test_df.to_csv('submission.csv', index=False)
 test_df.head()<set_options>
df_train = pd.read_csv(".. /input/train.csv", parse_dates=['timestamp'])
 df_test = pd.read_csv(".. /input/test.csv", parse_dates=['timestamp'])
 df_macro = pd.read_csv(".. /input/macro.csv", parse_dates=['timestamp'])
 df_train.head()
torch.cuda.is_available()<define_variables>
y_train = df_train['price_doc'].values
id_test = df_test['id']
df_train.drop(['id', 'price_doc'], axis=1, inplace=True)
 df_test.drop(['id'], axis=1, inplace=True)
 num_train = len(df_train)
 df_all = pd.concat([df_train, df_test])
 df_all = df_all.join(df_macro, on='timestamp', rsuffix='_macro')
 print(df_all.shape)
 month_year =(df_all.timestamp.dt.month + df_all.timestamp.dt.year * 100)
 month_year_cnt_map = month_year.value_counts().to_dict()
df_all['month_year_cnt'] = month_year.map(month_year_cnt_map)
 week_year =(df_all.timestamp.dt.weekofyear + df_all.timestamp.dt.year * 100)
 week_year_cnt_map = week_year.value_counts().to_dict()
df_all['week_year_cnt'] = week_year.map(week_year_cnt_map)
 df_all['month'] = df_all.timestamp.dt.month
df_all['dow'] = df_all.timestamp.dt.dayofweek
df_all['rel_floor'] = df_all['floor'] / df_all['max_floor'].astype(float)
 df_all['rel_kitch_sq'] = df_all['kitch_sq'] / df_all['full_sq'].astype(float)
 df_all.drop(['timestamp', 'timestamp_macro'], axis=1, inplace=True )
PSEUDO = len(glob.glob(".. /input/global-wheat-detection/test/*")) >10<install_modules>
factorize = lambda t: pd.factorize(t[1])[0]
df_obj = df_all.select_dtypes(include=['object'])
 X_all = np.c_[
df_all.select_dtypes(exclude=['object'] ).values,
np.array(list(map(factorize, df_obj.iteritems())) ).T
]
print(X_all.shape)
 X_train = X_all[:num_train]
X_test = X_all[num_train:]
!python -m pip install --upgrade.. /input/package/pip-20.1.1-py2.py3-none-any.whl<install_modules>
df_numeric = df_all.select_dtypes(exclude=['object'])
 df_obj = df_all.select_dtypes(include=['object'] ).copy()
for c in df_obj:
df_obj[c] = pd.factorize(df_obj[c])[0]
df_values = pd.concat([df_numeric, df_obj], axis=1 )
!pip install --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/nvidiaapex/ .<install_modules>
xgb_params = {
'eta': 0.05,
'max_depth': 5,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
dtrain = xgb.DMatrix(X_train, y_train, feature_names=df_columns)
 dtest = xgb.DMatrix(X_test, feature_names=df_columns )
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null
!pip install ".. /input/eff-omegaconf/omegaconf-2.0.0-py3-none-any.whl" > /dev/null<normalization>
model = xgb.train(dict(xgb_params, silent=0), dtrain, num_boost_round=num_boost_round )
def get_valid_transforms() :
return A.Compose([
A.Resize(height=1024, width=1024, p=1.0),
A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=255.0, always_apply=False, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<data_type_conversions>
y_pred = model.predict(dtest)
 df_sub = pd.DataFrame({'id': id_test, 'price_doc': y_pred})
 df_sub.head()
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 shape = image.shape
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return shape,image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
df_sub.to_csv('sub.csv', index=False )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_valid_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=2,
drop_last=False,
collate_fn=collate_fn
)<set_options>
df_test["timestamp"] = pd.to_datetime(df_test["timestamp"])
 df_test["year"] = df_test["timestamp"].dt.year
df_test["month"] = df_test["timestamp"].dt.month
df_test["yearmonth"] = 100*df_test.year + df_test.month
test_ids = pd.DataFrame({"yearmonth":df_test.yearmonth.values,
"id":id_test.values })
 test_data = test_ids.merge(df_sub,on="id")
 test_prices = test_data[["yearmonth","price_doc"]]
test_p = test_prices.groupby("yearmonth" ).median()
test_p.head()
class DetBenchEvalMultiScale(nn.Module):
def __init__(self, model, config,multiscale=[]):
super(DetBenchEvalMultiScale, self ).__init__()
self.config = config
self.model = model
self.multiscale = multiscale
self.anchors = []
for i,m in enumerate(multiscale):
self.anchors.append(Anchors(
 config.min_level, config.max_level,
config.num_scales, config.aspect_ratios,
config.anchor_scale, int(config.image_size*m)).cuda())
 self.size = config.image_size
def forward(self, x, image_scales,scale):
if scale not in self.multiscale:
print("scale not in 0.5,0.625,0.75,0.875,1.0,1.125,1.25,1.375,1.5")
 return None
s = np.where(np.array(self.multiscale)==scale)[0][0]
class_out, box_out = self.model(x)
 class_out, box_out, indices, classes = _post_process(self.config, class_out, box_out)
 batch_detections = []
for i in range(x.shape[0]):
detections = generate_detections(
 class_out[i], box_out[i], self.anchors[s].boxes, indices[i], classes[i], image_scales[i])
 batch_detections.append(detections)
 return torch.stack(batch_detections, dim=0 )<choose_model_class>
df_macro["timestamp"] = pd.to_datetime(df_macro["timestamp"])
 df_macro["year"] = df_macro["timestamp"].dt.year
df_macro["month"] = df_macro["timestamp"].dt.month
df_macro["yearmonth"] = 100*df_macro.year + df_macro.month
macmeds = df_macro.groupby("yearmonth" ).median()
df_train["timestamp"] = pd.to_datetime(df_train["timestamp"])
 df_train["year"] = df_train["timestamp"].dt.year
df_train["month"] = df_train["timestamp"].dt.month
df_train["yearmonth"] = 100*df_train.year + df_train.month
prices = pd.DataFrame({"yearmonth":df_train.yearmonth.values,
"price_doc":y_train })
 p = prices.groupby("yearmonth" ).median()
df = macmeds.join(p )
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d7')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=1024
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
net = torch.nn.DataParallel(net)
 checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['ema_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEvalMultiScale(net, config,[0.875,1.0,1.125,1.25])
 net.eval() ;
return net.cuda()
net = load_net('.. /input/effd7pth/best-checkpoint-027epoch.bin' )<normalization>
def almonZmatrix(X, maxlag, maxdeg):

 n = len(X)
 Z = ml.zeros(( len(X)-maxlag, maxdeg+1))
for t in range(maxlag, n):
Z[t-maxlag,0] = sum([X[t-lag] for lag in range(maxlag+1)])
 for j in range(1, maxdeg+1):
s = 0.0
for i in range(1, maxlag+1):
s +=(i)**j * X[t-i]
Z[t-maxlag,j] = s
return Z
def scale_img(img, ratio=1.0, same_shape=False):
h, w = img.shape[2:]
s =(int(h * ratio), int(w * ratio))
img = F.interpolate(img, size=s, mode='bilinear', align_corners=False)
 if not same_shape:
gs = 32
h, w = [math.ceil(x * ratio / gs)* gs for x in(h, w)]
return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447 )<define_search_space>
y_macro = df.price_doc.div(df.cpi ).apply(np.log ).loc[201108:201506]
nobs = 47
tblags = 5
mrlags = 5
ztb = almonZmatrix(df.balance_trade.loc[201103:201506].as_matrix() , tblags, 1)
 zmr = almonZmatrix(df.mortgage_rate.loc[201103:201506].as_matrix() , mrlags, 1)
 columns = ['tb0', 'tb1', 'mr0', 'mr1']
z = pd.DataFrame(np.concatenate(( ztb, zmr), axis=1), y_macro.index.values, columns)
 X_macro = sm.add_constant(z )
def make_tta_predictions(images, score_threshold=0.4024215973594928):
with torch.no_grad() :
x = torch.stack(images ).float().cuda()
predictions = []
img_size = x.shape[-2:]
s = [1.0,1.0,1.0,1.25,1.125]
y = []
all_x =[]
center = []
for i, xi in enumerate(( x,
x.flip(3),
x.flip(2),
scale_img(x,s[3]),
scale_img(x,s[4]), 
)) :
y.append(net(xi, torch.tensor([1]*xi.shape[0] ).float().cuda() ,s[i]))
y[1][..., 0] = img_size[1] - y[1][..., 0] - y[1][..., 2]
y[2][..., 1] = img_size[0] - y[2][..., 1] - y[2][..., 3]
y[3][..., :4]/=s[3]
y[4][..., :4]/=s[4]
y = np.array(y)
 boxes_all = []
scores_all=[]
for j in range(len(y)) :
boxes = y[j][0].cpu().numpy() [:,:4].copy()
scores = y[j][0].cpu().numpy() [:,4].copy()
indexes = np.where(scores > score_threshold)[0]
boxes = boxes[indexes]
boxes[:, 2] = boxes[:, 2] + boxes[:, 0]
boxes[:, 3] = boxes[:, 3] + boxes[:, 1]
boxes_all.append(boxes/1024)
 scores_all.append(scores[indexes])
 return boxes_all,scores_all,y<compute_train_metric>
macro_fit = sm.OLS(y_macro, X_macro ).fit()
def run_wbf(boxes,scores, image_index, image_size=1024, iou_thr=0.40813015995026714, skip_box_thr=0.4133694759009949, weights=[0.6,0.2,0.2],name='nms'):
labels = [np.ones([row.shape[0]])for row in scores]
if name=='wbf':
boxes, scores, labels = ensemble_boxes.ensemble_boxes_wbf.weighted_boxes_fusion(boxes, scores, labels, weights=weights, iou_thr=iou_thr, skip_box_thr=skip_box_thr)
 else:
boxes, scores, labels = ensemble_boxes.nms(boxes, scores, labels, weights=None, iou_thr=iou_thr)
 boxes = boxes*(image_size-1)
 return boxes, scores, labels<categorify>
test_cpi = df.cpi.loc[201507:201605]
test_index = test_cpi.index
ztb_test = almonZmatrix(df.balance_trade.loc[201502:201605].as_matrix() , tblags, 1)
 zmr_test = almonZmatrix(df.mortgage_rate.loc[201502:201605].as_matrix() , mrlags, 1)
 z_test = pd.DataFrame(np.concatenate(( ztb_test, zmr_test), axis=1), test_index, columns)
 X_macro_test = sm.add_constant(z_test)
 pred_lnrp = macro_fit.predict(X_macro_test)
 pred_p = np.exp(pred_lnrp)* test_cpi
def norm(img):
img = img.astype(float)
 img-=img.min()
img/=img.max()
return img<init_hyperparams>
adjust = pd.DataFrame(pred_p/test_p.price_doc, columns=["adjustment"])
 adjust
plt.figure(figsize=[20,20])
 for j,(shape,images, image_ids)in enumerate(data_loader):
h,w,_ = shape[0]
w_factor = w/1024
h_factor = h/1024
boxes_all,scores_all,y = make_tta_predictions(images,score_threshold= 0.4024215973594928)
 i = 0
sample = norm(images[i].permute(1,2,0 ).cpu().numpy())
 boxes, scores, labels = run_wbf(boxes_all,scores_all, image_index=i,name='wbf',iou_thr=0.43312889428044965, skip_box_thr= 0.393358169307333,
weights=[0.4,0.05,0.05,0.25,0.25])
 boxes = boxes.round().clip(min=0, max=1023)
 boxes[:, [0,2]]*=w_factor
boxes[:, [1,3]]*=h_factor
boxes = boxes.astype(np.int32)
 plt.subplot(3,3,j+1)
 for box in boxes:
cv2.rectangle(sample,(box[0], box[1]),(box[2], box[3]),(1, 1, 1), 3)
 plt.imshow(sample)
 if j==8:
break
plt.show()<define_variables>
combo = test_data.join(adjust, on='yearmonth')
 combo['adjusted'] = combo.price_doc * combo.adjustment
adjxgb_df = pd.DataFrame()
adjxgb_df['id'] = combo.id
adjxgb_df['price_doc'] = combo.adjusted
adjxgb_df.head()
def format_prediction_string(boxes, scores):
pred_strings = []
for j in zip(scores, boxes):
pred_strings.append("{0:.4f} {1} {2} {3} {4}".format(j[0], j[1][0], j[1][1], j[1][2], j[1][3]))
return " ".join(pred_strings )<init_hyperparams>
adjxgb_df.to_csv('adjusted_xgb_predicitons.csv', index=False )