Kaggle competition: Sberbank Russian Housing Market
!pip install --no-deps '.. /input/timm-package/timm-0.1.26-py3-none-any.whl' > /dev/null
!pip install --no-deps '.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl' > /dev/null<feature_engineering>
!git clone https://github.com/LenzDu/Kaggle-Competition-Sberbank Utils
sys.path.insert(0, ".. /input/timm-efficientdet-pytorch")
<define_variables>

 print('Data Cleaning...')
 trainDf = pd.read_csv('.. /input/train.csv' ).set_index('id')
 testDf = pd.read_csv('.. /input/test.csv' ).set_index('id')
 fix = pd.read_excel('.. /input/BAD_ADDRESS_FIX.xlsx' ).set_index('id')
 testDf['isTrain'] = 0
trainDf['isTrain'] = 1
allDf = pd.concat([trainDf,testDf])
 allDf.update(fix, filter_func = lambda x:np.array([True]*x.shape[0]))
macro = pd.read_csv('.. /input/macro.csv')
 allDf['timestamp'] = pd.to_datetime(allDf['timestamp'])
 allDf['apartment_name'] = allDf.sub_area + allDf['metro_km_avto'].astype(str)
 eco_map = {'excellent':4, 'good':3, 'satisfactory':2, 'poor':1, 'no data':0}
allDf['ecology'] = allDf['ecology'].map(eco_map)
 for c in allDf.columns:
if allDf[c].dtype == 'object':
lbl = preprocessing.LabelEncoder()
lbl.fit(list(allDf[c].values))
allDf[c] = lbl.transform(list(allDf[c].values))
allDf.loc[allDf.full_sq>2000,'full_sq'] = np.nan
allDf.loc[allDf.full_sq<3,'full_sq'] = np.nan
allDf.loc[allDf.life_sq>500,'life_sq'] = np.nan
allDf.loc[allDf.life_sq<3,'life_sq'] = np.nan
allDf.loc[allDf.life_sq>0.8*allDf.full_sq,'life_sq'] = np.nan
allDf.loc[allDf.kitch_sq>=allDf.life_sq,'kitch_sq'] = np.nan
allDf.loc[allDf.kitch_sq>500,'kitch_sq'] = np.nan
allDf.loc[allDf.kitch_sq<2,'kitch_sq'] = np.nan
allDf.loc[allDf.state>30,'state'] = np.nan
allDf.loc[allDf.build_year<1800,'build_year'] = np.nan
allDf.loc[allDf.build_year==20052009,'build_year'] = 2005
allDf.loc[allDf.build_year==4965,'build_year'] = np.nan
allDf.loc[allDf.build_year>2021,'build_year'] = np.nan
allDf.loc[allDf.num_room>15,'num_room'] = np.nan
allDf.loc[allDf.num_room==0,'num_room'] = np.nan
allDf.loc[allDf.floor==0,'floor'] = np.nan
allDf.loc[allDf.max_floor==0,'max_floor'] = np.nan
allDf.loc[allDf.floor>allDf.max_floor,'max_floor'] = np.nan
bad_index = allDf[allDf.price_doc/allDf.full_sq > 600000].index
bad_index = bad_index.append(allDf[allDf.price_doc/allDf.full_sq < 10000].index)
 allDf.drop(bad_index,0,inplace=True)
 print('Feature Engineering...')
 gc.collect()
allDf['year'] = allDf.timestamp.dt.year
allDf['weekday'] = allDf.timestamp.dt.weekday
allDf['w'] = 1
allDf.loc[allDf.price_doc==1000000,'w'] *= 0.5
allDf.loc[allDf.year==2015,'w'] *= 1.5
allDf['floor_by_max_floor'] = allDf.floor / allDf.max_floor
allDf['avg_room_size'] =(allDf.life_sq - allDf.kitch_sq)/ allDf.num_room
allDf['life_sq_prop'] = allDf.life_sq / allDf.full_sq
allDf['kitch_sq_prop'] = allDf.kitch_sq / allDf.full_sq
allDf['build_age'] = allDf.year - allDf.build_year
allDf = allDf.drop('build_year', 1)
 allDf['popu_den'] = allDf.raion_popul / allDf.area_m
allDf['gender_rate'] = allDf.male_f / allDf.female_f
allDf['working_rate'] = allDf.work_all / allDf.full_all
allDf.loc[allDf.preschool_quota==0,'preschool_quota'] = np.nan
allDf['preschool_ratio'] = allDf.children_preschool / allDf.preschool_quota
allDf['school_ratio'] = allDf.children_school / allDf.school_quota
allDf['square_full_sq'] =(allDf.full_sq - allDf.full_sq.mean())** 2
allDf['square_build_age'] =(allDf.build_age - allDf.build_age.mean())** 2
allDf['nan_count'] = allDf[['full_sq','build_age','life_sq','floor','max_floor','num_room']].isnull().sum(axis=1)
 allDf['full*maxfloor'] = allDf.max_floor * allDf.full_sq
allDf['full*floor'] = allDf.floor * allDf.full_sq
allDf['full/age'] = allDf.full_sq /(allDf.build_age + 0.5)
 allDf['age*state'] = allDf.build_age * allDf.state
allDf['main_road_diff'] = allDf['big_road2_km'] - allDf['big_road1_km']
allDf['rate_metro_km'] = allDf['metro_km_walk'] / allDf['ID_metro'].map(allDf.metro_km_walk.groupby(allDf.ID_metro ).mean().to_dict())
 allDf['rate_road1_km'] = allDf['big_road1_km'] / allDf['ID_big_road1'].map(allDf.big_road1_km.groupby(allDf.ID_big_road1 ).mean().to_dict())
 allDf['rate_road2_km'] = allDf['big_road2_km'] / allDf['ID_big_road2'].map(allDf.big_road2_km.groupby(allDf.ID_big_road2 ).mean().to_dict())
 allDf['rate_railroad_km'] = allDf['railroad_station_walk_km'] / allDf['ID_railroad_station_walk'].map(allDf.railroad_station_walk_km.groupby(allDf.ID_railroad_station_walk ).mean().to_dict())
 allDf.drop(['year','timestamp'], 1, inplace = True)
 trainDf = allDf[allDf.isTrain==1].drop(['isTrain'],1)
 testDf = allDf[allDf.isTrain==0].drop(['isTrain','price_doc', 'w'],1)
 outputFile = 'train_featured.csv'
trainDf.to_csv(outputFile,index=False)
 outputFile = 'test_featured.csv'
testDf.to_csv(outputFile,index=False)
best_score_threshold = 0.54<set_options>
import numpy as np
import pandas as pd
from sklearn.model_selection import ShuffleSplit, cross_val_score
from sklearn.ensemble import AdaBoostRegressor, RandomForestRegressor, ExtraTreesRegressor, GradientBoostingRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.preprocessing import Imputer
from sklearn.tree import DecisionTreeRegressor
from sklearn.linear_model import LinearRegression
import xgboost as xgb
import lightgbm as lgb
from sklearn.preprocessing import StandardScaler
SEED = 42
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
seed_everything(SEED )<define_variables>
from sklearn.model_selection import KFold
effdet_weights = ".. /input/effdetretrains/retrain.bin"<load_from_csv>
n_folds = 5
marking = pd.read_csv('.. /input/global-wheat-detection/train.csv')
 bboxs = np.stack(marking['bbox'].apply(lambda x: np.fromstring(x[1:-1], sep=',')))
 for i, column in enumerate(['x', 'y', 'w', 'h']):
marking[column] = bboxs[:,i]
marking.drop(columns=['bbox'], inplace=True )<feature_engineering>
class LGBregressor(object):
def __init__(self,params):
self.params = params
def fit(self, X, y, w):
y /= 10000000
split = int(X.shape[0] * 0.8)
 indices = np.random.permutation(X.shape[0])
 train_id, test_id = indices[:split], indices[split:]
x_train, y_train, w_train, x_valid, y_valid, w_valid = X[train_id], y[train_id], w[train_id], X[test_id], y[test_id], w[test_id],
d_train = lgb.Dataset(x_train, y_train, weight=w_train)
 d_valid = lgb.Dataset(x_valid, y_valid, weight=w_valid)
 partial_bst = lgb.train(self.params, d_train, 10000, valid_sets=d_valid, early_stopping_rounds=50, verbose_eval=500)
 num_round = partial_bst.best_iteration
d_all = lgb.Dataset(X, label = y, weight=w)
 self.bst = lgb.train(self.params, d_all, num_round)
 def predict(self, X):
return self.bst.predict(X)* 10000000
marking['area'] = marking['w'] * marking['h']
print(len(marking))
marking = marking[marking['area'] < 154200.0]
print(len(marking))
error_bbox = [100648.0, 145360.0, 149744.0, 119790.0, 106743.0]
marking = marking[~marking['area'].isin(error_bbox)]
marking = marking[marking['w'] >= 10.0]
marking = marking[marking['h'] >= 10.0]
print(len(marking))<define_search_model>
class XGBregressor(object):
def __init__(self, params):
self.params = params
def fit(self, X, y, w=None):
split = int(X.shape[0] * 0.8)
 indices = np.random.permutation(X.shape[0])
 train_id, test_id = indices[:split], indices[split:]
x_train, y_train, w_train, x_valid, y_valid, w_valid = X[train_id], y[train_id], w[train_id], X[test_id], y[test_id], w[test_id],
d_train = xgb.DMatrix(x_train, label=y_train, weight=w_train)
 d_valid = xgb.DMatrix(x_valid, label=y_valid, weight=w_valid)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
partial_bst = xgb.train(self.params, d_train, 10000, early_stopping_rounds=50, evals = watchlist, verbose_eval=500)
 num_round = partial_bst.best_iteration
d_all = xgb.DMatrix(X, label = y, weight=w)
 self.bst = xgb.train(self.params, d_all, num_round)
 def predict(self, X):
test = xgb.DMatrix(X)
 return self.bst.predict(test )
def get_train_transforms() :
return A.Compose(
 [
A.RandomSizedCrop(min_max_height=(800, 800), height=1024, width=1024, p=0.5),
A.OneOf([
A.HueSaturationValue(hue_shift_limit=0.2, sat_shift_limit= 0.1,
val_shift_limit=0.1, p=0.9),
A.RandomBrightnessContrast(brightness_limit=0.1,
contrast_limit=0.1, p=0.9),
],p=0.9),
A.ToGray(p=0.01),
A.HorizontalFlip(p=0.5),
A.VerticalFlip(p=0.5),
A.Resize(height=512, width=512, p=1),
A.Cutout(num_holes=10, max_h_size=64, max_w_size=64, fill_value=0, p=0.5),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_valid_transforms() :
return A.Compose(
 [
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
],
p=1.0,
bbox_params=A.BboxParams(
 format='pascal_voc',
min_area=0,
min_visibility=0,
label_fields=['labels'] 
)
)
 def get_test_transforms() :
return A.Compose([
A.Resize(height=512, width=512, p=1.0),
ToTensorV2(p=1.0),
], p=1.0 )<init_hyperparams>
class Ensemble(object):
def __init__(self, stacker, base_models):
self.stacker = stacker
self.base_models = base_models
def fit_predict(self, trainDf, testDf):
X = trainDf.drop(['price_doc', 'w'], 1 ).values
y = trainDf['price_doc'].values
w = trainDf['w'].values
T = testDf.values
X_fillna = trainDf.drop(['price_doc', 'w'], 1 ).fillna(-999 ).values
T_fillna = testDf.fillna(-999 ).values
kfold = KFold(n_splits=n_folds, shuffle=True)
 S_train = np.zeros(( X.shape[0], len(self.base_models)))
 S_test = np.zeros(( T.shape[0], len(self.base_models)))
 for i, clf in enumerate(self.base_models):
print('Training base model ' + str(i+1)+ '...')
 S_test_i = np.zeros(( T.shape[0], n_folds))
j=0
for train_idx, test_idx in kfold.split(X):
print('Training round...' + str(j+1)+ '...')
 if clf not in [xgb1,lgb1]:
X = X_fillna
T = T_fillna
X_train = X[train_idx]
y_train = y[train_idx]
w_train = w[train_idx]
X_holdout = X[test_idx]
clf.fit(X_train, y_train, w_train)
 y_pred = clf.predict(X_holdout)
 S_train[test_idx, i] = y_pred
S_test_i[:, j] = clf.predict(T)
 j=j+1
S_test[:, i] = S_test_i.mean(1)
 self.corr = pd.concat([pd.DataFrame(S_train),trainDf['price_doc']],1 ).corr()
print(S_train.shape)
 print(y.shape)
 self.stacker.fit(S_train, y, w)
 y_pred = self.stacker.predict(S_test)
 return y_pred
warnings.filterwarnings("ignore")
 class Fitter:
def __init__(self, model, device, config):
self.config = config
self.epoch = 0
self.base_dir = f'./{config.folder}'
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
 self.log_path = f'{self.base_dir}/log.txt'
self.best_summary_loss = 10**5
self.model = model
self.device = device
param_optimizer = list(self.model.named_parameters())
 no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},
{'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
self.optimizer = torch.optim.Adam(self.model.parameters() , lr=config.lr)
 self.scheduler = config.SchedulerClass(self.optimizer, **config.scheduler_params)
 self.log(f'Fitter prepared.Device is {self.device}')
 def fit(self, train_loader, validation_loader):
for e in range(self.config.n_epochs):
if self.config.verbose:
lr = self.optimizer.param_groups[0]['lr']
timestamp = datetime.utcnow().isoformat()
self.log(f'
{timestamp}
LR: {lr}')
 t = time.time()
summary_loss = self.train_one_epoch(train_loader)
 self.log(f'[RESULT]: Train.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 self.save(f'{self.base_dir}/last-checkpoint1.bin')
 t = time.time()
summary_loss = self.validation(validation_loader)
 self.log(f'[RESULT]: Val.Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}')
 if summary_loss.avg < self.best_summary_loss:
self.best_summary_loss = summary_loss.avg
self.model.eval()
self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch ).zfill(3)}epoch.bin')
 for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin')) [:-3]:
os.remove(path)
 if self.config.validation_scheduler:
self.scheduler.step(metrics=summary_loss.avg)
 self.epoch += 1
def validation(self, val_loader):
self.model.eval()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(val_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Val Step {step}/{len(val_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 with torch.no_grad() :
images = torch.stack(images)
 batch_size = images.shape[0]
images = images.to(self.device ).float()
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
loss, _, _ = self.model(images, boxes, labels)
 summary_loss.update(loss.detach().item() , batch_size)
 return summary_loss
def train_one_epoch(self, train_loader):
self.model.train()
summary_loss = AverageMeter()
t = time.time()
for step,(images, targets, image_ids)in enumerate(train_loader):
if self.config.verbose:
if step % self.config.verbose_step == 0:
print(
 f'Train Step {step}/{len(train_loader)}, ' + \
f'summary_loss: {summary_loss.avg:.5f}, ' + \
f'time: {(time.time() - t):.5f}', end='\r' 
)
 images = torch.stack(images)
 images = images.to(self.device ).float()
batch_size = images.shape[0]
boxes = [target['boxes'].to(self.device ).float() for target in targets]
labels = [target['labels'].to(self.device ).float() for target in targets]
accumulation_steps = 3
loss, _, _ = self.model(images, boxes, labels)
 loss = loss/accumulation_steps
loss.backward()
summary_loss.update(loss.detach().item() , batch_size)
 if(( step + 1)%accumulation_steps)== 0:
self.optimizer.step()
self.optimizer.zero_grad()
if self.config.step_scheduler:
self.scheduler.step()
return summary_loss
def save(self, path):
self.model.eval()
torch.save({
'model_state_dict': self.model.model.state_dict() ,
'optimizer_state_dict': self.optimizer.state_dict() ,
'scheduler_state_dict': self.scheduler.state_dict() ,
'best_summary_loss': self.best_summary_loss,
'epoch': self.epoch,
}, path)
 def load(self, path):
checkpoint = torch.load(path)
 self.model.model.load_state_dict(checkpoint['model_state_dict'])
 self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
 self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
 self.best_summary_loss = checkpoint['best_summary_loss']
self.epoch = checkpoint['epoch'] + 1
def log(self, message):
if self.config.verbose:
print(message)
 with open(self.log_path, 'a+')as logger:
logger.write(f'{message}
' )<init_hyperparams>
trainDf = pd.read_csv('/kaggle/working/train_featured.csv')
 testDf = pd.read_csv('/kaggle/working/test_featured.csv' )
class TrainGlobalConfig:
num_workers = 2
batch_size = 4
n_epochs = 1
lr = 0.0001
folder = 'plabel_model'
verbose = True
verbose_step = 1
step_scheduler = False
validation_scheduler = True
SchedulerClass = torch.optim.lr_scheduler.ReduceLROnPlateau
scheduler_params = dict(
 mode='min',
factor=0.5,
patience=1,
verbose=False,
threshold=0.0001,
threshold_mode='abs',
cooldown=0,
min_lr=1e-8,
eps=1e-08
)<data_type_conversions>
params1 = {'eta':0.05, 'max_depth':5, 'subsample':0.8, 'colsample_bytree':0.8, 'min_child_weight':1,
'gamma':0, 'silent':1, 'objective':'reg:linear', 'eval_metric':'rmse'}
xgb1 = XGBregressor(params1 )
DATA_ROOT_PATH = '.. /input/global-wheat-detection/test'
class DatasetRetriever(Dataset):
def __init__(self, image_ids, transforms=None):
super().__init__()
self.image_ids = image_ids
self.transforms = transforms
def __getitem__(self, index: int):
image_id = self.image_ids[index]
image = cv2.imread(f'{DATA_ROOT_PATH}/{image_id}.jpg', cv2.IMREAD_COLOR)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB ).astype(np.float32)
 image /= 255.0
if self.transforms:
sample = {'image': image}
sample = self.transforms(**sample)
 image = sample['image']
return image, image_id
def __len__(self)-> int:
return self.image_ids.shape[0]<load_pretrained>
params2 = {'booster':'gblinear', 'alpha':0,
'eta':0.1, 'max_depth':2, 'subsample':1, 'colsample_bytree':1, 'min_child_weight':1,
'gamma':0, 'silent':1, 'objective':'reg:linear', 'eval_metric':'rmse'}
xgb2 = XGBregressor(params2 )
dataset = DatasetRetriever(
 image_ids=np.array([path.split('/')[-1][:-4] for path in glob(f'{DATA_ROOT_PATH}/*.jpg')]),
transforms=get_test_transforms() 
)
 def collate_fn(batch):
return tuple(zip(*batch))
data_loader = DataLoader(
 dataset,
batch_size=1,
shuffle=False,
num_workers=4,
drop_last=False,
collate_fn=collate_fn
)<choose_model_class>
def load_net(checkpoint_path):
config = get_efficientdet_config('tf_efficientdet_d5')
 net = EfficientDet(config, pretrained_backbone=False)
 config.num_classes = 1
config.image_size=512
net.class_net = HeadNet(config, num_outputs=config.num_classes, norm_kwargs=dict(eps=.001, momentum=.01))
checkpoint = torch.load(checkpoint_path)
 net.load_state_dict(checkpoint['model_state_dict'])
 del checkpoint
gc.collect()
net = DetBenchEval(net, config)
 net.eval() ;
return net.cuda()
net = load_net(effdet_weights )<categorify>
RF = RandomForestRegressor(n_estimators=500, max_features=0.2)
 ETR = ExtraTreesRegressor(n_estimators=500, max_features=0.3, max_depth=None)
 Ada = AdaBoostRegressor(DecisionTreeRegressor(max_depth=15),n_estimators=200)
 GBR = GradientBoostingRegressor(n_estimators=200,max_depth=5,max_features=0.5)
 LR =LinearRegression()
params_lgb = {'objective':'regression','metric':'rmse',
'learning_rate':0.05,'max_depth':-1,'sub_feature':0.7,'sub_row':1,
'num_leaves':15,'min_data':30,'max_bin':20,
'bagging_fraction':0.9,'bagging_freq':40,'verbosity':0}
lgb1 = LGBregressor(params_lgb)
 E = Ensemble(xgb2, [xgb1,lgb1,RF,ETR,Ada,GBR] )
class BaseWheatTTA:

 image_size = 512
def augment(self, image):
raise NotImplementedError
def batch_augment(self, images):
raise NotImplementedError
def deaugment_boxes(self, boxes):
raise NotImplementedError
class TTAHorizontalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(1)
 def batch_augment(self, images):
return images.flip(2)
 def deaugment_boxes(self, boxes):
boxes[:, [1,3]] = self.image_size - boxes[:, [3,1]]
return boxes
class TTAVerticalFlip(BaseWheatTTA):

 def augment(self, image):
return image.flip(2)
 def batch_augment(self, images):
return images.flip(3)
 def deaugment_boxes(self, boxes):
boxes[:, [0,2]] = self.image_size - boxes[:, [2,0]]
return boxes
class TTARotate90(BaseWheatTTA):

 def augment(self, image):
return torch.rot90(image, 1,(1, 2))
def batch_augment(self, images):
return torch.rot90(images, 1,(2, 3))
def deaugment_boxes(self, boxes):
res_boxes = boxes.copy()
res_boxes[:, [0,2]] = self.image_size - boxes[:, [1,3]]
res_boxes[:, [1,3]] = boxes[:, [2,0]]
return res_boxes
class TTACompose(BaseWheatTTA):

 def __init__(self, transforms):
self.transforms = transforms
def augment(self, image):
for transform in self.transforms:
image = transform.augment(image)
 return image
def batch_augment(self, images):
for transform in self.transforms:
images = transform.batch_augment(images)
 return images
def prepare_boxes(self, boxes):
result_boxes = boxes.copy()
result_boxes[:,0] = np.min(boxes[:, [0,2]], axis=1)
 result_boxes[:,2] = np.max(boxes[:, [0,2]], axis=1)
 result_boxes[:,1] = np.min(boxes[:, [1,3]], axis=1)
 result_boxes[:,3] = np.max(boxes[:, [1,3]], axis=1)
 return result_boxes
def deaugment_boxes(self, boxes):
for transform in self.transforms[::-1]:
boxes = transform.deaugment_boxes(boxes)
 return self.prepare_boxes(boxes )<concatenate>
prediction = E.fit_predict(trainDf, testDf )
<predict_on_test><EOS> 
output = pd.read_csv('.. /input/test.csv')
 output = output[['id']]
output['price_doc'] = prediction
output.to_csv(r'Submission_Stack.csv',index=False)