Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<init_hyperparams>
%matplotlib inline
print(os.listdir(".. /input"))
pd.set_option('display.max_columns', None)
 pd.options.display.float_format = '{:.2f}'.format
class Trainer:
def __init__(self, model, data_module, experiment_id, optimizer=None, scheduler=None, device='cuda'):
self.model = model
self.data_module = data_module
self.optimizer = optimizer
self.scheduler = scheduler
self.device = device
self.fp16 = False
self.step_scheduler_after = None
self.n_epochs = None
self.metrics = {}
self.metrics['train'] = []
self.metrics['valid'] = []
self.current_epoch = 0
self.current_batch = 0
self.scaler = None
self._best_score = -np.inf
self._delta = None
self._current_score = None
self._counter = 0
self._patience = None
self.experiment_id=experiment_id
def configure_trainer(self):
if self.optimizer is None:
self.configure_optimizers()
if self.scheduler is None:
self.configure_schedulers()
if next(self.model.parameters() ).device != self.device:
self.model.to(self.device)
 if self.fp16:
self.scaler = torch.cuda.amp.GradScaler()
def configure_optimizers(self):
self.optimizer = torch.optim.Adam(self.model.parameters() , lr=self.lr)
 def configure_schedulers(self, **kwargs):
self.scheduler = None
def set_params(self, **kwargs):
for parameter, value in kwargs.items() :
setattr(self, parameter, value)
 def compute_metrics(self, batch_outputs_collection, batch_targets_collection):
y_pred = np.concatenate(batch_outputs_collection, axis=0)
 y_true = np.concatenate(batch_targets_collection, axis=0)
 assert y_pred.shape == y_true.shape, "shape mismatch"
scores = []
for i in range(y_true.shape[1]):
try:
score = roc_auc_score(y_true[:,i], y_pred[:,i])
 scores.append(score)
 except ValueError:
raise("Behaviour Not expected: FIXME")
 avg_score = np.mean(scores)
 return avg_score
def model_forward_pass(self, data):

 for key, value in data.items() :
data[key] = value.to(self.device)
 if self.fp16:
with torch.cuda.amp.autocast() :
output, loss = self.model(**data)
 else:
output, loss = self.model(**data)
 return output, loss
def train_one_batch(self, data):
self.optimizer.zero_grad()
output, loss = self.model_forward_pass(data)
 with torch.set_grad_enabled(True):
if self.fp16:
with torch.cuda.amp.autocast() :
self.scaler.scale(loss ).backward()
self.scaler.step(self.optimizer)
 self.scaler.update()
else:
loss.backward()
self.optimizer.step()
if self.scheduler:
if self.step_scheduler_after == "batch":
if self.step_scheduler_metric is None:
self.scheduler.step()
else:
pass
return output, loss
def train_one_epoch(self, dataloader):
self.model.train()
all_outputs = []
all_targets = []
all_losses = []
tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))
for batch_id, data in tk0:
batch_outputs, batch_loss= self.train_one_batch(data)
 all_outputs.append(batch_outputs.detach().cpu().numpy())
 all_targets.append(data['targets'].detach().cpu().numpy())
 all_losses.append(batch_loss.detach().cpu().item())
 tk0.set_postfix(loss=np.array(all_losses ).mean() , stage="train", epoch=self.current_epoch)
 tk0.close()
avg_auc = self.compute_metrics(all_outputs, all_targets)
 avg_loss = np.array(all_losses ).mean()
self.metrics['train'].append({'epoch': self.current_epoch,
'avg_loss': avg_loss, 'auc_score': avg_auc})
 print(self.metrics['train'][self.current_epoch -1])
 def validate_one_batch(self, data):
output, loss = self.model_forward_pass(data)
 return output, loss
def validate_one_epoch(self, dataloader):
self.model.eval()
all_outputs = []
all_targets = []
all_losses = []
tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))
for batch_id, data in tk0:
batch_outputs, batch_loss= self.validate_one_batch(data)
 all_outputs.append(batch_outputs.detach().cpu().numpy())
 all_targets.append(data['targets'].detach().cpu().numpy())
 all_losses.append(batch_loss.detach().cpu().item())
 tk0.set_postfix(loss=np.array(all_losses ).mean() , stage="validate", epoch=self.current_epoch)
 tk0.close()
avg_auc = self.compute_metrics(all_outputs, all_targets)
 avg_loss = np.array(all_losses ).mean()
self.metrics['valid'].append({'epoch': self.current_epoch,
'avg_loss': avg_loss, 'auc_score': avg_auc})
 print(self.metrics['valid'][-1])
 def early_stoping(self):

 self._current_score = self.metrics['valid'][-1]['auc_score']
if(self._current_score - self._best_score)> self._delta:
self._best_score = self._current_score
self._counter = 0
self.save_checkpoint()
print("early stopping counter reset to 0")
 else:
self._counter += 1
print(f"early stopping counter {self._counter} out of {self._patience}")
 if self._counter == self._patience:
return True
return False
def save_checkpoint(self):

 if not os.path.isdir(path_checkpoints_dir):
os.mkdir(path_checkpoints_dir)
 model_path = os.path.join(path_checkpoints_dir, f"{self.experiment_id}.pth")
 print(f"saved the model at {model_path}")
 model_state_dict = self.model.state_dict()
if self.optimizer is not None:
opt_state_dict = self.optimizer.state_dict()
else:
opt_state_dict = None
if self.scheduler is not None:
sch_state_dict = self.scheduler.state_dict()
else:
sch_state_dict = None
model_dict = {}
model_dict["state_dict"] = model_state_dict
model_dict["optimizer"] = opt_state_dict
model_dict["scheduler"] = sch_state_dict
model_dict["epoch"] = self.current_epoch
model_dict["fp16"] = self.fp16
model_dict['lr'] = self.lr
model_dict['metrics'] = self.metrics
model_dict['best_score'] = self._best_score
model_dict['patience'] = self._patience
model_dict['delta'] = self._delta
model_dict['train_batch_size'] = self.train_batch_size
model_dict['validation_batch_size'] = self.validation_batch_size
torch.save(model_dict, model_path)
 def load(self, model_path, device=None):

 if device:
self.device = device
checkpoint = torch.load(model_path)
 if self.model:
self.model.load_state_dict(checkpoint['state_dict'])
 self.model.to(self.device)
 if self.optimizer:
self.optimizer.load_state_dict(checkpoint['optimizer'])
 if self.scheduler:
self.scheduler.load_state_dict(checkpoint['scheduler'])
 self.fp16 = True
def predict(self, test_batch_size=64, device='cuda', load=False, model_path=None, dataloader_num_workers=4, save_prediction=True):

 self.model.eval()
self.device = device
self.test_batch_size = test_batch_size
if load:
if model_path:
self.load(model_path, device=self.device)
 else:
model_path = os.path.join(path_checkpoints_dir, f"{self.experiment_id}.pth")
 print(f"loaded model={model_path}")
 self.load(model_path, device=self.device)
 if self.model is None:
raise Exception("model cannot be None.Load or train the model before inference")
 dataloader = self.data_module.get_test_dataloader(batch_size=self.test_batch_size, shuffle=False, num_workers=dataloader_num_workers)
 all_outputs = []
tk0 = tqdm(enumerate(dataloader, 1), total=len(dataloader))
for batch_id, data in tk0:
for key, value in data.items() :
data[key] = value.to(self.device)
 batch_outputs, batch_loss= self.validate_one_batch(data)
 all_outputs.append(batch_outputs.detach().cpu().numpy())
 predictions = np.concatenate(all_outputs, axis=0)
 if save_prediction:
submission = pd.read_csv(path_sample_submission_file)
 assert submission.shape[0] == predictions.shape[0], "unexpected behavior.code fix required"
submission.iloc[:, 1:] = predictions
if not os.path.isdir(path_submissions_dir):
os.mkdir(path_submissions_dir)
 submission.to_csv(os.path.join(path_submissions_dir, f"{self.experiment_id}.csv"), index=False)
 tk0.close()
return predictions
def fit(self,
n_epochs=100,
lr=1e-3,
step_scheduler_after='epoch',
device='cuda',
fp16=False,
train_batch_size = 64,
validation_batch_size=64,
dataloader_shuffle=True,
dataloader_num_workers=4,
tensorboard_writer = None,
es_delta=1e-4,
es_patience=3, 
):

 self.n_epochs = n_epochs
self.step_scheduler_after = step_scheduler_after
self.device = device
self.fp16 = fp16
self.lr = lr
self._delta = es_delta
self._patience = es_patience
self.train_batch_size = train_batch_size
self.validation_batch_size = validation_batch_size
self.configure_trainer()
for i in range(1, self.n_epochs+1):
self.current_epoch = i
train_dataloader = self.data_module.get_train_dataloader(
 batch_size=train_batch_size,
shuffle=dataloader_shuffle,
num_workers=dataloader_num_workers,
pin_memory=True)
 self.train_one_epoch(train_dataloader)
 validation_dataloader = self.data_module.get_valid_dataloader(
 batch_size=validation_batch_size,
shuffle=dataloader_shuffle,
num_workers=dataloader_num_workers,
pin_memory=True 
)
 self.validate_one_epoch(validation_dataloader)
 es_flag = self.early_stoping()
if es_flag:
print(f"early stopping at epoch={i} out of {n_epochs}")
 break<choose_model_class>
trdf = pd.read_csv('.. /input/train.csv' )
def ensemble_models(model_paths, output_file, **kwargs):

 model = EfficientNetModel(pretrained=False)
 data_module = DataModule(None, None, test_dataset)
 preds_list = []
num_models = len(model_paths)
 print(f"number of models to ensemble={num_models}")
 for mpath in model_paths:
print(mpath)
 trainer = Trainer(model, data_module, None)
 prediction = trainer.predict(load=True, model_path=mpath, save_prediction=False, **kwargs)
 preds_list.append(prediction)
 mean_prediction = np.stack(preds_list, axis=-1 ).mean(axis=-1)
 print(f"mean prediction array shape={mean_prediction.shape}")
 submission = pd.read_csv(path_sample_submission_file)
 assert submission.shape[0] == mean_prediction.shape[0], "unexpected behavior.code fix required"
submission.iloc[:, 1:] = mean_prediction
if not os.path.isdir(path_submissions_dir):
os.mkdir(path_submissions_dir)
 submission.to_csv(os.path.join(path_submissions_dir, f"{output_file}.csv"), index=False )<load_pretrained>
trdf = trdf.drop(columns = ['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'] )
model_paths = os.listdir(path_trained_models,)
 model_paths = [os.path.join(path_trained_models, mpath)for mpath in model_paths]
ensemble_models(model_paths, "submission", test_batch_size=16)
 print("done" )<define_variables>
trdf['LotFrontage'].fillna(0, inplace = True)
 Glist = ['GarageCond', 'GarageType', 'GarageFinish', 'GarageQual','GarageYrBlt']
for item in Glist:
trdf[item].fillna(0, inplace = True)
 Blist = ['BsmtExposure','BsmtQual','BsmtExposure','BsmtCond','BsmtFinType1','BsmtFinType2']
for item in Blist:
trdf[item].fillna(0, inplace = True)
 trdf['MasVnrType'].fillna('None', inplace = True)
 trdf['MasVnrArea'].fillna(0, inplace = True)
 trdf['Electrical'].fillna('SBrkr',inplace = True )
OUTPUT_DIR = './'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TEST_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test'<define_variables>
trdf[trdf['TotalBsmtSF']>trdf['GrLivArea']]
class CFG:
debug=False
num_workers=4
model_name='resnet200d_320'
size=512
batch_size=128
seed=416
target_size=11
target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',
'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',
'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',
'Swan Ganz Catheter Present']
n_fold=5
trn_fold=[0]<compute_test_metric>
trdf[trdf['TotalBsmtSF']>5900]
def get_score(y_true, y_pred):
scores = []
for i in range(y_true.shape[1]):
score = roc_auc_score(y_true[:,i], y_pred[:,i])
 scores.append(score)
 avg_score = np.mean(scores)
 return avg_score, scores
def get_result(result_df):
preds = result_df[[f'pred_{c}' for c in CFG.target_cols]].values
labels = result_df[CFG.target_cols].values
score, scores = get_score(labels, preds)
 LOGGER.info(f'Score: {score:<.4f} Scores: {np.round(scores, decimals=4)}')
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def init_logger(log_file=OUTPUT_DIR+'inference.log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
LOGGER = init_logger()
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed )<load_from_csv>
trdf[trdf['GrLivArea']>4000]
oof_df = pd.read_csv('.. /input/ranzcr-exp12-step3-fold0/oof_df.csv')
 for fold in CFG.trn_fold:
fold_oof_df = oof_df[oof_df['fold']==fold].reset_index(drop=True)
 LOGGER.info(f"========== fold: {fold} result ==========")
 get_result(fold_oof_df )<load_from_csv>
trdf.drop(index = [523,1298], inplace = True )
if CFG.debug:
test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv', nrows=10)
 else:
test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 print(test.shape)
 test.head()<create_dataframe>
trdf = trdf.replace({"MSSubClass" : {20 : "SC20", 30 : "SC30", 40 : "SC40", 45 : "SC45",
50 : "SC50", 60 : "SC60", 70 : "SC70", 75 : "SC75",
80 : "SC80", 85 : "SC85", 90 : "SC90", 120 : "SC120",
150 : "SC150", 160 : "SC160", 180 : "SC180", 190 : "SC190"},
"MoSold" : {1 : "Jan", 2 : "Feb", 3 : "Mar", 4 : "Apr", 5 : "May", 6 : "Jun",
7 : "Jul", 8 : "Aug", 9 : "Sep", 10 : "Oct", 11 : "Nov", 12 : "Dec"}} )
train_dataset = TestDataset(test, transform=get_transforms(data='valid'))
for i in range(1):
image = train_dataset[i]
plt.imshow(image[0])
 plt.show()
plt.imshow(image[0].flip(-1))
plt.show()<load_pretrained>
trdf = trdf.replace({"Alley" : {"Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"No" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"No" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}}
)
class CustomResNet200D(nn.Module):
def __init__(self, model_name='resnet200d_320', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=False)
 if pretrained:
pretrained_path = '.. /input/resnet200d-pretrained-weight/resnet200d_ra2-bdba9bf9.pth'
self.model.load_state_dict(torch.load(pretrained_path))
print(f'load {model_name} pretrained model')
 n_features = self.model.fc.in_features
self.model.global_pool = nn.Identity()
self.model.fc = nn.Identity()
self.pooling = nn.AdaptiveAvgPool2d(1)
 self.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
bs = x.size(0)
 features = self.model(x)
 pooled_features = self.pooling(features ).view(bs, -1)
 output = self.fc(pooled_features)
 return output<categorify>
trdf.drop(columns = ['GarageArea','1stFlrSF', 'GarageYrBlt', 'TotRmsAbvGrd','YearRemodAdd', 'BsmtCond',
'BsmtFinType1','BsmtFinType2', 'GarageCond' ], inplace = True )
def inference(models, test_loader, device):
tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for model in models:
with torch.no_grad() :
y_preds1 = model(images)
 y_preds2 = model(images.flip(-1))
y_preds =(y_preds1.sigmoid().to('cpu' ).numpy() + y_preds2.sigmoid().to('cpu' ).numpy())/ 2
avg_preds.append(y_preds)
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<choose_model_class>
testdf = pd.read_csv('.. /input/test.csv' )
%%time
model = CustomResNet200D(CFG.model_name, pretrained=False)
 model_path = '.. /input/ranzcr-exp12-step3-fold0/resnet200d_320_fold0_best_loss.pth'
model.load_state_dict(torch.load(model_path)['model'])
 model.eval()
models = [model.to(device)]<create_dataframe>
( testdf.isna().sum() /(testdf.count() +testdf.isna().sum())).sort_values(ascending = False )
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
 predictions = inference(models, test_loader, device )<save_to_csv>
df = testdf.drop(columns = ['Alley', 'FireplaceQu', 'PoolQC', 'Fence', 'MiscFeature'] )
test[CFG.target_cols] = predictions
test[['StudyInstanceUID'] + CFG.target_cols].to_csv(OUTPUT_DIR+'submission.csv', index=False)
 test.head()<import_modules>
df['LotFrontage'].fillna(0, inplace = True)
 Glist = ['GarageCond', 'GarageType', 'GarageFinish', 'GarageQual','GarageYrBlt']
for item in Glist:
df[item].fillna(0, inplace = True)
 Blist = ['BsmtExposure','BsmtQual','BsmtExposure','BsmtCond','BsmtFinType1','BsmtFinType2']
for item in Blist:
df[item].fillna(0, inplace = True)
 df['MasVnrType'].fillna('None', inplace = True)
 df['MasVnrArea'].fillna(0, inplace = True)
 df['Electrical'].fillna('SBrkr',inplace = True )
import os
import random
import numpy as np
import pandas as pd
import cv2
import matplotlib.pyplot as plt
import torch
import torchvision
from torchvision import transforms
import tensorflow as tf
import albumentations as A<define_variables>
df = df.replace({"MSSubClass" : {20 : "SC20", 30 : "SC30", 40 : "SC40", 45 : "SC45",
50 : "SC50", 60 : "SC60", 70 : "SC70", 75 : "SC75",
80 : "SC80", 85 : "SC85", 90 : "SC90", 120 : "SC120",
150 : "SC150", 160 : "SC160", 180 : "SC180", 190 : "SC190"},
"MoSold" : {1 : "Jan", 2 : "Feb", 3 : "Mar", 4 : "Apr", 5 : "May", 6 : "Jun",
7 : "Jul", 8 : "Aug", 9 : "Sep", 10 : "Oct", 11 : "Nov", 12 : "Dec"}} )
DIR = ".. /input/ranzcr-clip-catheter-line-classification/"<load_from_csv>
df = df.replace({"Alley" : {"Grvl" : 1, "Pave" : 2},
"BsmtCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"BsmtExposure" : {"No" : 0, "Mn" : 1, "Av": 2, "Gd" : 3},
"BsmtFinType1" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtFinType2" : {"No" : 0, "Unf" : 1, "LwQ": 2, "Rec" : 3, "BLQ" : 4,
"ALQ" : 5, "GLQ" : 6},
"BsmtQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
"ExterCond" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"ExterQual" : {"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
"FireplaceQu" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"Functional" : {"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5,
"Min2" : 6, "Min1" : 7, "Typ" : 8},
"GarageCond" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"GarageQual" : {"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"HeatingQC" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"KitchenQual" : {"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
"LandSlope" : {"Sev" : 1, "Mod" : 2, "Gtl" : 3},
"LotShape" : {"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
"PavedDrive" : {"N" : 0, "P" : 1, "Y" : 2},
"PoolQC" : {"No" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
"Street" : {"Grvl" : 1, "Pave" : 2},
"Utilities" : {"ELO" : 1, "NoSeWa" : 2, "NoSewr" : 3, "AllPub" : 4}}
)
df_annotations = pd.read_csv(os.path.join(DIR, "train_annotations.csv"))<load_pretrained>
df.drop(columns = ['GarageArea','1stFlrSF', 'GarageYrBlt', 'TotRmsAbvGrd','YearRemodAdd', 'BsmtCond',
'BsmtFinType1','BsmtFinType2', 'GarageCond' ], inplace = True )
row = df_annotations.iloc[8]
image_path = os.path.join(DIR, "train", row["StudyInstanceUID"] + ".jpg")
 chosen_image = cv2.imread(image_path )<train_on_grid>
dn =(df.isna().sum() >0 )
def NeedleAugmentation(image, n_needles=2, dark_needles=False, p=0.5, needle_folder='.. /input/xray-needle-augmentation'):
aug_prob = random.random()
if aug_prob < p:
height, width, _ = image.shape
needle_images = [im for im in os.listdir(needle_folder)if 'png' in im]
for _ in range(1, n_needles):
needle = cv2.cvtColor(cv2.imread(os.path.join(needle_folder, random.choice(needle_images))), cv2.COLOR_BGR2RGB)
 needle = cv2.flip(needle, random.choice([-1, 0, 1]))
needle = cv2.rotate(needle, random.choice([0, 1, 2]))
h_height, h_width, _ = needle.shape
roi_ho = random.randint(0, abs(image.shape[0] - needle.shape[0]))
roi_wo = random.randint(0, abs(image.shape[1] - needle.shape[1]))
roi = image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width]
img2gray = cv2.cvtColor(needle, cv2.COLOR_BGR2GRAY)
 ret, mask = cv2.threshold(img2gray, 10, 255, cv2.THRESH_BINARY)
 mask_inv = cv2.bitwise_not(mask)
 img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 if dark_needles:
img_bg = cv2.bitwise_and(roi, roi, mask=mask_inv)
 needle_fg = cv2.bitwise_and(img_bg, img_bg, mask=mask)
 else:
needle_fg = cv2.bitwise_and(needle, needle, mask=mask)
 dst = cv2.add(img_bg, needle_fg, dtype=cv2.CV_64F)
 image[roi_ho:roi_ho + h_height, roi_wo:roi_wo + h_width] = dst
return image<load_pretrained>
for feature in dn[dn].index:
df[feature].fillna(df[feature].mode() [0], inplace = True )
chosen_image = cv2.imread(image_path)
 aug_image = NeedleAugmentation(chosen_image, n_needles=3, dark_needles=False, p=1.0)
 plt.imshow(aug_image )<load_pretrained>
df.isna().sum() >0
chosen_image = cv2.imread(image_path)
 aug_image = NeedleAugmentation(chosen_image, n_needles=3, dark_needles=True, p=1.0)
 plt.imshow(aug_image )<set_options>
TestId=df['Id']
total_features=pd.concat(( trdf.drop(['Id','SalePrice'], axis=1), df.drop(['Id'], axis=1)))
 total_features=pd.get_dummies(total_features, drop_first=True)
 train_features=total_features[0:trdf.shape[0]]
test_features=total_features[trdf.shape[0]:]
torch_trans_list = [transforms.CenterCrop(( 178, 178)) ,
transforms.Resize(128),
transforms.RandomRotation(45),
transforms.RandomAffine(35),
transforms.RandomCrop(128),
transforms.RandomHorizontalFlip(p=1),
transforms.RandomPerspective(p=1),
transforms.RandomVerticalFlip(p=1)]<define_variables>
train_features['GrLivArea'] = np.log(train_features['GrLivArea'])
 test_features['GrLivArea'] = np.log(test_features['GrLivArea'] )
INFER_MODE = 'TF'<install_modules>
X = train_features
y = np.log(trdf['SalePrice'] )
!pip install.. /input/timm-package/timm-0.1.26-py3-none-any.whl<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=101 )
if INFER_MODE == 'TORCH':
sys.path.append('.. /input/pytorch-image-models/pytorch-image-models-master')
 MODEL_DIR = '.. /input/ranzcr-pytorch-weights/'
OUTPUT_DIR = './'
if not os.path.exists(OUTPUT_DIR):
os.makedirs(OUTPUT_DIR)
 TEST_PATH = '.. /input/ranzcr-clip-catheter-line-classification/test'<define_variables>
ridge=Ridge(alpha=besta, copy_X=True, fit_intercept=True)
 ridge.fit(X_train, y_train)
 predictions=ridge.predict(X_test )
class CFG:
debug=False
num_workers=4
model_name='resnext50_32x4d'
size=600
batch_size=64
seed=42
target_size=11
target_cols=['ETT - Abnormal', 'ETT - Borderline', 'ETT - Normal',
'NGT - Abnormal', 'NGT - Borderline', 'NGT - Incompletely Imaged', 'NGT - Normal',
'CVC - Abnormal', 'CVC - Borderline', 'CVC - Normal',
'Swan Ganz Catheter Present']
n_fold=10
trn_fold=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]<import_modules>
explained_variance_score(y_test,predictions )
if INFER_MODE == 'TORCH':
Compose, OneOf, Normalize, Resize, RandomResizedCrop, RandomCrop, HorizontalFlip, VerticalFlip,
RandomBrightness, RandomContrast, RandomBrightnessContrast, Rotate, ShiftScaleRotate, Cutout,
IAAAdditiveGaussianNoise, Transpose 
)
 warnings.filterwarnings('ignore')
 device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 def get_score(y_true, y_pred):
scores = []
for i in range(y_true.shape[1]):
score = roc_auc_score(y_true[:,i], y_pred[:,i])
 scores.append(score)
 avg_score = np.mean(scores)
 return avg_score, scores
def get_result(result_df):
preds = result_df[[f'pred_{c}' for c in CFG.target_cols]].values
labels = result_df[CFG.target_cols].values
score, scores = get_score(labels, preds)
 LOGGER.info(f'Score: {score:<.4f} Scores: {np.round(scores, decimals=4)}')
 @contextmanager
def timer(name):
t0 = time.time()
LOGGER.info(f'[{name}] start')
 yield
LOGGER.info(f'[{name}] done in {time.time() - t0:.0f} s.')
 def init_logger(log_file=OUTPUT_DIR+'inference.log'):
logger = getLogger(__name__)
 logger.setLevel(INFO)
 handler1 = StreamHandler()
handler1.setFormatter(Formatter("%(message)s"))
handler2 = FileHandler(filename=log_file)
 handler2.setFormatter(Formatter("%(message)s"))
logger.addHandler(handler1)
 logger.addHandler(handler2)
 return logger
LOGGER = init_logger()
def seed_torch(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_torch(seed=CFG.seed)
 train = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/train.csv')
 folds = train.copy()
Fold = GroupKFold(n_splits=CFG.n_fold)
 groups = folds['PatientID'].values
for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[CFG.target_cols], groups)) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 display(folds.groupby('fold' ).size())
 test = pd.read_csv('.. /input/ranzcr-clip-catheter-line-classification/sample_submission.csv')
 if CFG.debug:
test = test.head()
class TestDataset(Dataset):
def __init__(self, df, transform=None):
self.df = df
self.file_names = df['StudyInstanceUID'].values
self.transform = transform
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
file_name = self.file_names[idx]
file_path = f'{TEST_PATH}/{file_name}.jpg'
image = cv2.imread(file_path)
 image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
 if self.transform:
augmented = self.transform(image=image)
 image = augmented['image']
return image
def get_transforms(*, data):
if data == 'train':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
])
 elif data == 'valid':
return Compose([
Resize(CFG.size, CFG.size),
Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225], 
),
ToTensorV2() ,
])
 class CustomResNext(nn.Module):
def __init__(self, model_name='resnext50_32x4d', pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG.target_size)
 def forward(self, x):
x = self.model(x)
 return x
def inference(model, states, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images)in tk0:
images = images.to(device)
 avg_preds = []
for state in states:
model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(images)
 avg_preds.append(y_preds.sigmoid().to('cpu' ).numpy())
 avg_preds = np.mean(avg_preds, axis=0)
 probs.append(avg_preds)
 probs = np.concatenate(probs)
 return probs<load_pretrained>
print('RMSE',np.sqrt(mean_squared_error(y_test, predictions)) )
if INFER_MODE == 'TORCH':
model = CustomResNext(CFG.model_name, pretrained=False)
 states = [torch.load(MODEL_DIR+f'needle_more_augs_10folds_pretrained_{CFG.model_name}_fold{fold}_best.pth')for fold in CFG.trn_fold]
test_dataset = TestDataset(test, transform=get_transforms(data='valid'))
test_loader = DataLoader(test_dataset, batch_size=CFG.batch_size, shuffle=False,
num_workers=CFG.num_workers, pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 test[CFG.target_cols] = predictions
test[['StudyInstanceUID'] + CFG.target_cols].to_csv(OUTPUT_DIR+'submission.csv', index=False )<install_modules>
lasso=Lasso(alpha=besta, copy_X=True, fit_intercept=True)
 lasso.fit(X_train, y_train)
 predictions=lasso.predict(X_test )
!pip install /kaggle/input/kerasapplications -q
!pip install /kaggle/input/efficientnet-keras-source-code/ -q --no-deps<set_options>
explained_variance_score(y_test,predictions )
if INFER_MODE == 'TF':
def auto_select_accelerator() :
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("Running on TPU:", tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print(f"Running on {strategy.num_replicas_in_sync} replicas")
 return strategy
def build_decoder(with_labels=True, target_size=(300, 300), ext='jpg'):
def decode(path):
file_bytes = tf.io.read_file(path)
 if ext == 'png':
img = tf.image.decode_png(file_bytes, channels=3)
 elif ext in ['jpg', 'jpeg']:
img = tf.image.decode_jpeg(file_bytes, channels=3)
 else:
raise ValueError("Image extension not supported")
 img = tf.cast(img, tf.float32)/ 255.0
img = tf.image.resize(img, target_size)
 return img
def decode_with_labels(path, label):
return decode(path), label
return decode_with_labels if with_labels else decode
def build_augmenter(with_labels=True):
def augment(img):
img = NeedleAugmentation(img, n_needles=2, dark_needles=False, p=0.5)
 img = tf.image.random_flip_left_right(img)
 img = tf.image.random_flip_up_down(img)
 return img
def augment_with_labels(img, label):
return augment(img), label
return augment_with_labels if with_labels else augment
def build_dataset(paths, labels=None, bsize=32, cache=True,
decode_fn=None, augment_fn=None,
augment=True, repeat=True, shuffle=1024,
cache_dir=""):
if cache_dir != "" and cache is True:
os.makedirs(cache_dir, exist_ok=True)
 if decode_fn is None:
decode_fn = build_decoder(labels is not None)
 if augment_fn is None:
augment_fn = build_augmenter(labels is not None)
 AUTO = tf.data.experimental.AUTOTUNE
slices = paths if labels is None else(paths, labels)
 dset = tf.data.Dataset.from_tensor_slices(slices)
 dset = dset.map(decode_fn, num_parallel_calls=AUTO)
 dset = dset.cache(cache_dir)if cache else dset
dset = dset.map(augment_fn, num_parallel_calls=AUTO)if augment else dset
dset = dset.repeat() if repeat else dset
dset = dset.shuffle(shuffle)if shuffle else dset
dset = dset.batch(bsize ).prefetch(AUTO)
 return dset
COMPETITION_NAME = "ranzcr-clip-catheter-line-classification"
strategy = auto_select_accelerator()
BATCH_SIZE = strategy.num_replicas_in_sync * 16
IMSIZE =(224, 240, 260, 300, 380, 456, 528, 600)
 load_dir = f"/kaggle/input/{COMPETITION_NAME}/"
sub_df = pd.read_csv(load_dir + 'sample_submission.csv')
 test_paths = load_dir + "test/" + sub_df['StudyInstanceUID'] + '.jpg'
label_cols = sub_df.columns[1:]
test_decoder = build_decoder(with_labels=False, target_size=(IMSIZE[7], IMSIZE[7]))
dtest = build_dataset(
 test_paths, bsize=BATCH_SIZE, repeat=False,
shuffle=False, augment=False, cache=False,
decode_fn=test_decoder 
)
 with strategy.scope() :
model = tf.keras.models.load_model(
 '.. /input/ranzcr-tpu-weights/efficientnetb7-tpu-needle-aug-resume-train.h5' 
)
 model.summary()
sub_df[label_cols] = model.predict(dtest, verbose=1)
 sub_df.to_csv('submission.csv', index=False )<define_variables>
print('RMSE',np.sqrt(mean_squared_error(y_test, predictions)) )
tez_path = '.. /input/tez-lib/'
effnet_path = '.. /input/efficientnet-pytorch/'
sys.path.append(tez_path)
 sys.path.append(effnet_path )<import_modules>
enet=ElasticNet(alpha=besta[0][0], l1_ratio=besta[0][1], normalize=False)
 enet.fit(X_train, y_train)
 predictions=enet.predict(X_test )
import os
import albumentations
import pandas as pd
import numpy as np
import tez
from tez.datasets import ImageDataset
import torch
import torch.nn as nn
from torch.nn import functional as F
from efficientnet_pytorch import EfficientNet<define_variables>
explained_variance_score(y_test,predictions )
INPUT_PATH = ".. /input/ranzcr-clip-catheter-line-classification/"
IMAGE_PATH = ".. /input/ranzcr-clip-catheter-line-classification/test/"
MODEL_PATH = ".. /input/ranzcr-effnet5/"
IMAGE_SIZE = 512<load_from_csv>
print('RMSE',np.sqrt(mean_squared_error(y_test, predictions)) )
df = pd.read_csv(os.path.join(INPUT_PATH, "sample_submission.csv"))<feature_engineering>
testpredictions=lasso.predict(test_features)
 testpredictions
class RanzcrModel(tez.Model):
def __init__(self):
super().__init__()
self.effnet = EfficientNet.from_name("efficientnet-b5")
 self.effnet._conv_stem.in_channels = 1
weight = self.effnet._conv_stem.weight.mean(1, keepdim=True)
 self.effnet._conv_stem.weight = torch.nn.Parameter(weight)
 self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(2048, 11)
 def forward(self, image, targets=None):
batch_size, _, _, _ = image.shape
x = self.effnet.extract_features(image)
 x = F.adaptive_avg_pool2d(x, 1 ).reshape(batch_size, -1)
 outputs = self.out(self.dropout(x))
return outputs, None, {}<normalization>
Test_price=np.exp(list(testpredictions))
Test_price
<define_variables><EOS> 
submission=pd.DataFrame()
submission['Id']=TestId
submission['SalePrice']=Test_price
submission.to_csv('submission.csv', index=False )