Kaggle competition: APTOS 2019 Blindness Detection
<SOS> metric: QuadraticWeightedKappa Kaggle data source: aptos-2019-blindness-detection<compute_test_metric>
def is_interactive() :
return 'runtime' in get_ipython().config.IPKernelApp.connection_file
print('Interactive?', is_interactive())
 debug = False
use_amp = True
score = cv_rmse(rfr)
 print("rfr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rfr'] =(score.mean() , score.std() )<compute_test_metric>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(21 )
score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['svr'] =(score.mean() , score.std() )<compute_test_metric>
torch.__version__, ignite.__version__
score = cv_rmse(ridge)
 print("ridge: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['ridge'] =(score.mean() , score.std() )<compute_test_metric>
%%time
if use_amp:
try:
except ImportError:
!pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext".. /input/nvidia-apex/repository/*
score = cv_rmse(rf)
 print("rf: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['rf'] =(score.mean() , score.std() )<compute_test_metric>
class Swish(nn.Module):
def forward(self, x):
return x * torch.sigmoid(x)
 class Flatten(nn.Module):
def forward(self, x):
return x.reshape(x.shape[0], -1 )
score = cv_rmse(gbr)
 print("gbr: {:.4f}({:.4f})".format(score.mean() , score.std()))
scores['gbr'] =(score.mean() , score.std() )<train_model>
class SqueezeExcitation(nn.Module):
def __init__(self, inplanes, se_planes):
super(SqueezeExcitation, self ).__init__()
self.reduce_expand = nn.Sequential(
 nn.Conv2d(inplanes, se_planes,
kernel_size=1, stride=1, padding=0, bias=True),
Swish() ,
nn.Conv2d(se_planes, inplanes,
kernel_size=1, stride=1, padding=0, bias=True),
nn.Sigmoid() 
)
 def forward(self, x):
x_se = torch.mean(x, dim=(-2, -1), keepdim=True)
 x_se = self.reduce_expand(x_se)
 return x_se * x
print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(train_labels))<train_model>
class MBConv(nn.Module):
def __init__(self, inplanes, planes, kernel_size, stride,
expand_rate=1.0, se_rate=0.25,
drop_connect_rate=0.2):
super(MBConv, self ).__init__()
expand_planes = int(inplanes * expand_rate)
 se_planes = max(1, int(inplanes * se_rate))
self.expansion_conv = None
if expand_rate > 1.0:
self.expansion_conv = nn.Sequential(
 nn.Conv2d(inplanes, expand_planes,
kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),
Swish() 
)
 inplanes = expand_planes
self.depthwise_conv = nn.Sequential(
 nn.Conv2d(inplanes, expand_planes,
kernel_size=kernel_size, stride=stride,
padding=kernel_size // 2, groups=expand_planes,
bias=False),
nn.BatchNorm2d(expand_planes, momentum=0.01, eps=1e-3),
Swish() 
)
 self.squeeze_excitation = SqueezeExcitation(expand_planes, se_planes)
 self.project_conv = nn.Sequential(
 nn.Conv2d(expand_planes, planes,
kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(planes, momentum=0.01, eps=1e-3), 
)
 self.with_skip = stride == 1
self.drop_connect_rate = torch.tensor(drop_connect_rate, requires_grad=False)
 def _drop_connect(self, x):
keep_prob = 1.0 - self.drop_connect_rate
drop_mask = torch.rand(x.shape[0], 1, 1, 1)+ keep_prob
drop_mask = drop_mask.type_as(x)
 drop_mask.floor_()
return drop_mask * x / keep_prob
def forward(self, x):
z = x
if self.expansion_conv is not None:
x = self.expansion_conv(x)
 x = self.depthwise_conv(x)
 x = self.squeeze_excitation(x)
 x = self.project_conv(x)
 if x.shape == z.shape and self.with_skip:
if self.training and self.drop_connect_rate is not None:
self._drop_connect(x)
 x += z
return x
print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, train_labels )<train_model>
def init_weights(module):
if isinstance(module, nn.Conv2d):
nn.init.kaiming_normal_(module.weight, a=0, mode='fan_out')
 elif isinstance(module, nn.Linear):
init_range = 1.0 / math.sqrt(module.weight.shape[1])
 nn.init.uniform_(module.weight, a=-init_range, b=init_range)
 class EfficientNet(nn.Module):
def _setup_repeats(self, num_repeats):
return int(math.ceil(self.depth_coefficient * num_repeats))
def _setup_channels(self, num_channels):
num_channels *= self.width_coefficient
new_num_channels = math.floor(num_channels / self.divisor + 0.5)* self.divisor
new_num_channels = max(self.divisor, new_num_channels)
 if new_num_channels < 0.9 * num_channels:
new_num_channels += self.divisor
return new_num_channels
def __init__(self, num_classes,
width_coefficient=1.0,
depth_coefficient=1.0,
se_rate=0.25,
dropout_rate=0.2,
drop_connect_rate=0.2):
super(EfficientNet, self ).__init__()
self.width_coefficient = width_coefficient
self.depth_coefficient = depth_coefficient
self.divisor = 8
list_channels = [32, 16, 24, 40, 80, 112, 192, 320, 1280]
list_channels = [self._setup_channels(c)for c in list_channels]
list_num_repeats = [1, 2, 2, 3, 3, 4, 1]
list_num_repeats = [self._setup_repeats(r)for r in list_num_repeats]
expand_rates = [1, 6, 6, 6, 6, 6, 6]
strides = [1, 2, 2, 2, 1, 2, 1]
kernel_sizes = [3, 3, 5, 3, 5, 5, 3]
self.stem = nn.Sequential(
 nn.Conv2d(3, list_channels[0], kernel_size=3, stride=2, padding=1, bias=False),
nn.BatchNorm2d(list_channels[0], momentum=0.01, eps=1e-3),
Swish() 
)
 blocks = []
counter = 0
num_blocks = sum(list_num_repeats)
 for idx in range(7):
num_channels = list_channels[idx]
next_num_channels = list_channels[idx + 1]
num_repeats = list_num_repeats[idx]
expand_rate = expand_rates[idx]
kernel_size = kernel_sizes[idx]
stride = strides[idx]
drop_rate = drop_connect_rate * counter / num_blocks
name = "MBConv{}_{}".format(expand_rate, counter)
 blocks.append((
name,
MBConv(num_channels, next_num_channels,
kernel_size=kernel_size, stride=stride, expand_rate=expand_rate,
se_rate=se_rate, drop_connect_rate=drop_rate)
))
counter += 1
for i in range(1, num_repeats):
name = "MBConv{}_{}".format(expand_rate, counter)
 drop_rate = drop_connect_rate * counter / num_blocks
blocks.append((
name,
MBConv(next_num_channels, next_num_channels,
kernel_size=kernel_size, stride=1, expand_rate=expand_rate,
se_rate=se_rate, drop_connect_rate=drop_rate)
))
counter += 1
self.blocks = nn.Sequential(OrderedDict(blocks))
self.head = nn.Sequential(
 nn.Conv2d(list_channels[-2], list_channels[-1],
kernel_size=1, bias=False),
nn.BatchNorm2d(list_channels[-1], momentum=0.01, eps=1e-3),
Swish() ,
nn.AdaptiveAvgPool2d(1),
Flatten() ,
nn.Dropout(p=dropout_rate),
nn.Linear(list_channels[-1], num_classes)
)
 self.apply(init_weights)
 def forward(self, x):
f = self.stem(x)
 f = self.blocks(f)
 y = self.head(f)
 return y
print('xgboost')
 xgb_model_full_data = xgboost.fit(X, train_labels )<train_model>
model = EfficientNet(num_classes=1000,
width_coefficient=1.4, depth_coefficient=1.8,
dropout_rate=0.4)
 resolution = 380
img_stats = [[0.485, 0.456, 0.406], [0.229, 0.224, 0.225]]
print('Svr')
 svr_model_full_data = svr.fit(X, train_labels )<train_model>
def print_num_params(model, display_all_modules=False):
total_num_params = 0
for n, p in model.named_parameters() :
num_params = 1
for s in p.shape:
num_params *= s
if display_all_modules: print("{}: {}".format(n, num_params))
total_num_params += num_params
print("-" * 50)
 print("Total number of parameters: {:.2e}".format(total_num_params))
print_num_params(model )
print('Ridge')
 ridge_model_full_data = ridge.fit(X, train_labels )<train_model>
from torchvision.models.resnet import resnet18, resnet34, resnet50
print('RandomForest')
 rf_model_full_data = rf.fit(X, train_labels )<train_model>
print_num_params(resnet18(pretrained=False, num_classes=1000))
print_num_params(resnet34(pretrained=False, num_classes=1000))
print_num_params(resnet50(pretrained=False, num_classes=1000))
print('GradientBoosting')
 gbr_model_full_data = gbr.fit(X, train_labels )<predict_on_test>
model_state = torch.load("/kaggle/input/efficientnet-pytorch/efficientnet-b4-e116e8b3.pth")
 mapping = {
k: v for k, v in zip(model_state.keys() , model.state_dict().keys())
 }
mapped_model_state = OrderedDict([ 
(mapping[k], v)for k, v in model_state.items()
])
 model.load_state_dict(mapped_model_state, strict=False )
def blended_predictions(X):
return(( 0.1 * ridge_model_full_data.predict(X)) + \ 
(0.2 * svr_model_full_data.predict(X)) + \ 
(0.1 * gbr_model_full_data.predict(X)) + \ 
(0.1 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.05 * rf_model_full_data.predict(X)) + \ 
(0.35 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
with open("/kaggle/input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master/examples/simple/labels_map.txt", "r")as h:
labels = json.load(h)
 img = Image.open("/kaggle/input/efficientnet-pytorch/efficientnet-pytorch/EfficientNet-PyTorch-master/examples/simple/img.jpg")
 tfms = transforms.Compose([transforms.Resize([resolution]*2),
transforms.ToTensor() ,
transforms.Normalize(*img_stats),])
 x = tfms(img ).unsqueeze(0)
 print(x.shape)
 _ = plt.imshow(img, shape=[resolution]*2 )
blended_score = rmsle(train_labels, blended_predictions(X))
scores['blended'] =(blended_score, 0)
 print('RMSLE score on train data:')
 print(blended_score )<load_from_csv>
model.eval()
with torch.no_grad() :
y_pred = model(x)
 print('-----')
 for idx in torch.topk(y_pred, k=5)[1].squeeze(0 ).tolist() :
prob = torch.softmax(y_pred, dim=1)[0, idx].item()
print('{label:<75}({p:.2f}%)'.format(label=labels[str(idx)], p=prob*100))
submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.shape<predict_on_test>
from torchvision.transforms import *
from torch.utils.data import Subset
import torchvision.utils as vutils
import pandas as pd
from sklearn.utils import shuffle
submission.iloc[:,1] = np.floor(np.expm1(blended_predictions(X_test)) )<save_to_csv>
df_train = pd.read_csv('.. /input/aptos2019-blindness-detection/train.csv')
 df_test = pd.read_csv('.. /input/aptos2019-blindness-detection/test.csv')
 x = df_train['id_code']
y = df_train['diagnosis']
x, y = shuffle(x, y)
 _ = y.hist()
n_classes = int(y.max() +1)
 class_weights = len(y)/ df_train.groupby('diagnosis' ).count().values.ravel()
class_weights *= n_classes / class_weights.sum()
print('class_weights:', class_weights.tolist() )
q1 = submission['SalePrice'].quantile(0.0045)
 q2 = submission['SalePrice'].quantile(0.99)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x > q1 else x*0.77)
 submission['SalePrice'] = submission['SalePrice'].apply(lambda x: x if x < q2 else x*1.1)
 submission.to_csv("submission_regression_New_Improvement.csv", index=False )<save_to_csv>
train_x, valid_x, train_y, valid_y = train_test_split(x.values, y.values, test_size=0.10, stratify=y, random_state=42)
 test_x = df_test.id_code.values
if debug:
train_x, train_y = train_x[:128], train_y[:128]
valid_x, valid_y = valid_x[:64], valid_y[:64]
print(train_x.shape)
 print(train_y.shape)
 print(valid_x.shape)
 print(valid_y.shape)
 print(test_x.shape )
submission['SalePrice'] *= 1.001619
submission.to_csv("submission_regression2.csv", index=False )<split>
class ImageDataset(torch.utils.data.Dataset):
def __init__(self, root, path_list, targets=None, transform=None, extension='.png'):
super().__init__()
self.root = root
self.path_list = path_list
self.targets = targets
self.transform = transform
self.extension = extension
if targets is not None:
assert len(self.path_list)== len(self.targets)
 self.targets = torch.LongTensor(targets)
 def __getitem__(self, index):

 path = self.path_list[index]
sample = Image.open(os.path.join(self.root, path+self.extension))
if self.transform is not None:
sample = self.transform(sample)
 if self.targets is not None:
return sample, self.targets[index]
else:
return sample, torch.LongTensor([])
 def __len__(self):
return len(self.path_list )
scaler = StandardScaler()
X_train, X_test, y_train, y_test = train_test_split(X, train_labels, test_size=0.3, random_state=42)
 X_train = scaler.fit_transform(X_train)
 X_test = scaler.transform(X_test )<categorify>
train_transform = Compose([
Resize([resolution]*2, BICUBIC),
ColorJitter(brightness=0.05, contrast=0.05, saturation=0.01, hue=0),
RandomAffine(degrees=15, translate=(0.01, 0.01), scale=(1.0, 1.25), fillcolor=(0,0,0), resample=BICUBIC),
RandomHorizontalFlip() ,
ToTensor() ,
Normalize(*img_stats)
 ])
 test_transform = Compose([
Resize([resolution]*2, BICUBIC),
ToTensor() ,
Normalize(*img_stats)
 ])
 train_dataset = ImageDataset(root='.. /input/aptos2019-blindness-detection/train_images',
path_list=train_x, targets=train_y, transform=train_transform)
 train_eval_dataset = ImageDataset(root='.. /input/aptos2019-blindness-detection/train_images',
path_list=valid_x, targets=valid_y, transform=test_transform)
 test_dataset = ImageDataset(root='.. /input/aptos2019-blindness-detection/test_images',
path_list=test_x, transform=test_transform)
 print(len(train_dataset), len(test_dataset), len(train_eval_dataset))
pca = PCA(n_components=3)
 X_train = pca.fit_transform(X_train)
 X_test = pca.transform(X_test )<set_options>
train_batch_size = 32
eval_batch_size = 16
num_workers = os.cpu_count()
print('num_workers:', num_workers)
 train_loader = DataLoader(train_dataset, batch_size=train_batch_size, num_workers=num_workers,
shuffle=True, drop_last=True, pin_memory=True)
 test_loader = DataLoader(test_dataset, batch_size=eval_batch_size, num_workers=num_workers,
shuffle=False, drop_last=False, pin_memory=True)
 eval_train_loader = DataLoader(train_eval_dataset, batch_size=eval_batch_size, num_workers=num_workers,
shuffle=False, drop_last=False, pin_memory=True )
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 pd.set_option('display.max_columns', None)
 train.head(5)
 test.head(5)
 print("The train data size before dropping Id feature is : {} ".format(train.shape))
print("The test data size before dropping Id feature is : {} ".format(test.shape))
print("*" * 70)
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 print("The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))
ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
 ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 sns.distplot(train['SalePrice'] , fit=norm, color = 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
train["SalePrice"] = np.log1p(train["SalePrice"])
 sns.distplot(train['SalePrice'] , fit=norm, color= 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
house_data = pd.concat(( train, test)).reset_index(drop=True)
 house_data.drop(['SalePrice'], axis=1, inplace=True)
 print("house_data size is : {}".format(house_data.shape))
data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head(20)
 f, ax = plt.subplots(figsize=(20, 12))
plt.xticks(rotation='60')
 sns.barplot(x=data_null.index, y=data_null, color = 'b')
 plt.xlabel('Features', fontsize=15)
 plt.ylabel('Percent of missing values', fontsize=15)
 plt.title('Percent missing data by feature', fontsize=15)
 corrmat = train.corr()
plt.subplots(figsize=(35,35))
sns.heatmap(corrmat, vmax=0.9, square=True, annot=True)
 house_data["PoolQC"] = house_data["PoolQC"].fillna("None")
 house_data["MiscFeature"] = house_data["MiscFeature"].fillna("None")
 house_data["Alley"] = house_data["Alley"].fillna("None")
 house_data["Fence"] = house_data["Fence"].fillna("None")
 house_data["FireplaceQu"] = house_data["FireplaceQu"].fillna("None")
 house_data["LotFrontage"] = house_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
house_data[col] = house_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
house_data[col] = house_data[col].fillna('None')
 house_data["MasVnrType"] = house_data["MasVnrType"].fillna("None")
 house_data["MasVnrArea"] = house_data["MasVnrArea"].fillna(0)
 house_data['MSZoning'] = house_data['MSZoning'].fillna(house_data['MSZoning'].mode() [0])
 house_data = house_data.drop(['Utilities'], axis=1)
 house_data["Functional"] = house_data["Functional"].fillna("Typ")
 house_data['Electrical'] = house_data['Electrical'].fillna(house_data['Electrical'].mode() [0])
 house_data['KitchenQual'] = house_data['KitchenQual'].fillna(house_data['KitchenQual'].mode() [0])
 house_data['Exterior1st'] = house_data['Exterior1st'].fillna(house_data['Exterior1st'].mode() [0])
 house_data['Exterior2nd'] = house_data['Exterior2nd'].fillna(house_data['Exterior2nd'].mode() [0])
 house_data['SaleType'] = house_data['SaleType'].fillna(house_data['SaleType'].mode() [0])
 house_data['MSSubClass'] = house_data['MSSubClass'].fillna("None")
 data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head()
house_data['MSSubClass'] = house_data['MSSubClass'].apply(str)
 house_data['OverallCond'] = house_data['OverallCond'].astype(str)
 house_data['YrSold'] = house_data['YrSold'].astype(str)
 house_data['MoSold'] = house_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(house_data[c].values))
house_data[c] = lbl.transform(list(house_data[c].values))
print('Shape all_data: {}'.format(house_data.shape))
house_data['TotalSF'] = house_data['TotalBsmtSF'] + house_data['1stFlrSF'] + house_data['2ndFlrSF']
numeric_feats = house_data.dtypes[house_data.dtypes != "object"].index
skewed_feats = house_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
house_data[feat] = boxcox1p(house_data[feat], lam)
 house_data = pd.get_dummies(house_data)
 print(house_data.shape)
 train = house_data[:ntrain]
test = house_data[ntrain:]
n_folds = 8
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
 averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False)
<set_options>
model.eval()
with torch.no_grad() :
y_pred = model.cuda()(batch[0][:1].cuda())
 print('-----')
 for idx in torch.topk(y_pred, k=5)[1].squeeze(0 ).tolist() :
prob = torch.softmax(y_pred, dim=1)[0, idx].item()
print('{label:<75}({p:.2f}%)'.format(label=labels[str(idx)], p=prob*100))
%matplotlib inline
color = sns.color_palette()
sns.set_style('darkgrid')
 def ignore_warn(*args, **kwargs):
pass
warnings.warn = ignore_warn
pd.set_option('display.float_format', lambda x: '{:.3f}'.format(x))
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv')
 pd.set_option('display.max_columns', None)
 train.head(5)
 test.head(5)
 print("The train data size before dropping Id feature is : {} ".format(train.shape))
print("The test data size before dropping Id feature is : {} ".format(test.shape))
print("*" * 70)
 train_ID = train['Id']
test_ID = test['Id']
train.drop("Id", axis = 1, inplace = True)
 test.drop("Id", axis = 1, inplace = True)
 print("The train data size after dropping Id feature is : {} ".format(train.shape))
print("The test data size after dropping Id feature is : {} ".format(test.shape))
ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 train = train.drop(train[(train['GrLivArea']>4000)&(train['SalePrice']<300000)].index)
 ax = sns.scatterplot(x = train['GrLivArea'], y = train['SalePrice'], hue = train['OverallCond'],)
 sns.jointplot(x='GrLivArea', y='SalePrice', data= train)
 sns.distplot(train['SalePrice'] , fit=norm, color = 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
train["SalePrice"] = np.log1p(train["SalePrice"])
 sns.distplot(train['SalePrice'] , fit=norm, color= 'b'); 
(mu, sigma)= norm.fit(train['SalePrice'])
 print('
mu = {:.2f} and sigma = {:.2f}
'.format(mu, sigma))
plt.legend(['Normal dist.( $\mu=$ {:.2f} and $\sigma=$ {:.2f})'.format(mu, sigma)],
loc='best')
 plt.ylabel('Frequency')
 plt.title('SalePrice distribution')
 fig = plt.figure()
res = stats.probplot(train['SalePrice'], plot=plt)
 plt.show()
ntrain = train.shape[0]
ntest = test.shape[0]
y_train = train.SalePrice.values
house_data = pd.concat(( train, test)).reset_index(drop=True)
 house_data.drop(['SalePrice'], axis=1, inplace=True)
 print("house_data size is : {}".format(house_data.shape))
data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)[:30]
missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head(20)
 f, ax = plt.subplots(figsize=(20, 12))
plt.xticks(rotation='60')
 sns.barplot(x=data_null.index, y=data_null, color = 'b')
 plt.xlabel('Features', fontsize=15)
 plt.ylabel('Percent of missing values', fontsize=15)
 plt.title('Percent missing data by feature', fontsize=15)
 corrmat = train.corr()
plt.subplots(figsize=(35,35))
sns.heatmap(corrmat, vmax=0.9, square=True, annot=True)
 house_data["PoolQC"] = house_data["PoolQC"].fillna("None")
 house_data["MiscFeature"] = house_data["MiscFeature"].fillna("None")
 house_data["Alley"] = house_data["Alley"].fillna("None")
 house_data["Fence"] = house_data["Fence"].fillna("None")
 house_data["FireplaceQu"] = house_data["FireplaceQu"].fillna("None")
 house_data["LotFrontage"] = house_data.groupby("Neighborhood")["LotFrontage"].transform(
 lambda x: x.fillna(x.median()))
for col in('GarageType', 'GarageFinish', 'GarageQual', 'GarageCond'):
house_data[col] = house_data[col].fillna('None')
 for col in('GarageYrBlt', 'GarageArea', 'GarageCars'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF','TotalBsmtSF', 'BsmtFullBath', 'BsmtHalfBath'):
house_data[col] = house_data[col].fillna(0)
 for col in('BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2'):
house_data[col] = house_data[col].fillna('None')
 house_data["MasVnrType"] = house_data["MasVnrType"].fillna("None")
 house_data["MasVnrArea"] = house_data["MasVnrArea"].fillna(0)
 house_data['MSZoning'] = house_data['MSZoning'].fillna(house_data['MSZoning'].mode() [0])
 house_data = house_data.drop(['Utilities'], axis=1)
 house_data["Functional"] = house_data["Functional"].fillna("Typ")
 house_data['Electrical'] = house_data['Electrical'].fillna(house_data['Electrical'].mode() [0])
 house_data['KitchenQual'] = house_data['KitchenQual'].fillna(house_data['KitchenQual'].mode() [0])
 house_data['Exterior1st'] = house_data['Exterior1st'].fillna(house_data['Exterior1st'].mode() [0])
 house_data['Exterior2nd'] = house_data['Exterior2nd'].fillna(house_data['Exterior2nd'].mode() [0])
 house_data['SaleType'] = house_data['SaleType'].fillna(house_data['SaleType'].mode() [0])
 house_data['MSSubClass'] = house_data['MSSubClass'].fillna("None")
 data_null =(house_data.isnull().sum() / len(house_data)) * 100
data_null = data_null.drop(data_null[data_null == 0].index ).sort_values(ascending=False)
 missing_data = pd.DataFrame({'Missing Ratio' :data_null})
 missing_data.head()
house_data['MSSubClass'] = house_data['MSSubClass'].apply(str)
 house_data['OverallCond'] = house_data['OverallCond'].astype(str)
 house_data['YrSold'] = house_data['YrSold'].astype(str)
 house_data['MoSold'] = house_data['MoSold'].astype(str)
 cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(house_data[c].values))
house_data[c] = lbl.transform(list(house_data[c].values))
print('Shape all_data: {}'.format(house_data.shape))
house_data['TotalSF'] = house_data['TotalBsmtSF'] + house_data['1stFlrSF'] + house_data['2ndFlrSF']
numeric_feats = house_data.dtypes[house_data.dtypes != "object"].index
skewed_feats = house_data[numeric_feats].apply(lambda x: skew(x.dropna())).sort_values(ascending=False)
 print("
Skew in numerical features:
")
 skewness = pd.DataFrame({'Skew' :skewed_feats})
 skewness.head(10)
 skewness = skewness[abs(skewness)> 0.75]
print("There are {} skewed numerical features to Box Cox transform".format(skewness.shape[0]))
skewed_features = skewness.index
lam = 0.15
for feat in skewed_features:
house_data[feat] = boxcox1p(house_data[feat], lam)
 house_data = pd.get_dummies(house_data)
 print(house_data.shape)
 train = house_data[:ntrain]
test = house_data[ntrain:]
n_folds = 8
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse= np.sqrt(-cross_val_score(model, train.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse)
 lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(lasso)
 print("
Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class AveragingModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, models):
self.models = models
def fit(self, X, y):
self.models_ = [clone(x)for x in self.models]
for model in self.models_:
model.fit(X, y)
 return self
def predict(self, X):
predictions = np.column_stack([
model.predict(X)for model in self.models_
])
 return np.mean(predictions, axis=1)
 averaged_models = AveragingModels(models =(ENet, GBoost, KRR, lasso))
score = rmsle_cv(averaged_models)
 print(" Averaged base models score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 stacked_averaged_models = StackingAveragedModels(base_models =(ENet, GBoost, KRR),
meta_model = lasso)
 score = rmsle_cv(stacked_averaged_models)
 print("Stacking Averaged models score: {:.4f}({:.4f})".format(score.mean() , score.std()))
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
stacked_averaged_models.fit(train.values, y_train)
 stacked_train_pred = stacked_averaged_models.predict(train.values)
 stacked_pred = np.expm1(stacked_averaged_models.predict(test.values))
print(rmsle(y_train, stacked_train_pred))
model_xgb.fit(train, y_train)
 xgb_train_pred = model_xgb.predict(train)
 xgb_pred = np.expm1(model_xgb.predict(test))
print(rmsle(y_train, xgb_train_pred))
model_lgb.fit(train, y_train)
 lgb_train_pred = model_lgb.predict(train)
 lgb_pred = np.expm1(model_lgb.predict(test.values))
print(rmsle(y_train, lgb_train_pred))

 print('RMSLE score on train data:')
 print(rmsle(y_train,stacked_train_pred*0.70 +
xgb_train_pred*0.15 + lgb_train_pred*0.15))
ensemble = stacked_pred*0.70 + xgb_pred*0.15 + lgb_pred*0.15
sub = pd.DataFrame()
sub['Id'] = test_ID
sub['SalePrice'] = ensemble
sub.to_csv('submission.csv',index=False )<load_from_csv>
del batch
torch.cuda.empty_cache()
gc.collect()
train = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv', index_col='Id')
 test = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv', index_col='Id')
 train.head()<train_model>
model.head[6] = nn.Linear(in_features, n_classes+1)
 classes =('No DR', 'Mild', 'Moderate', 'Severe', 'Proliferative DR' )
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
 print('==Testing model R^2(higher is better)==')
 linr = LinearRegression() ; cv = cross_val_score(linr, X_train, y_train, cv=5, scoring='r2'); print("LinReg 5-fold CV score: {:.2f}".format(np.mean(cv)))
 elas = ElasticNet(alpha=0.01); cv = cross_val_score(elas, X_train, y_train, cv=5); print("ElasticNet 5-fold CV score: {:.3f}".format(np.mean(cv)))
 ridge = Ridge() ; cv = cross_val_score(ridge, X_train, y_train, cv=5); print("Ridge 5-fold CV score: {:.3f}".format(np.mean(cv)))
 br = BayesianRidge() ; cv = cross_val_score(br, X_train, y_train, cv=5); print("Bayesian Ridge 5-fold CV score: {:.3f}".format(np.mean(cv)))
 lasso = Lasso(alpha=0.001); cv = cross_val_score(lasso, X_train, y_train, cv=5); print("Lasso 5-fold CV score: {:.3f}".format(np.mean(cv))); lasso_coef = lasso.fit(X_train, y_train ).coef_
knn = KNeighborsRegressor() ; cv = cross_val_score(knn, X_train, y_train, cv=5); print("knn 5-fold CV score: {:.3f}".format(np.mean(cv)))
 svr = SVR() ; cv = cross_val_score(svr, X_train, y_train, cv=5); print("SVMReg 5-fold CV score: {:.3f}".format(np.mean(cv)))
 dtree = DecisionTreeRegressor() ; cv = cross_val_score(dtree, X_train, y_train, cv=5); print("Dec.tree 5-fold CV score: {:.3f}".format(np.mean(cv)))
 rftree = RandomForestRegressor() ; cv = cross_val_score(rftree, X_train, y_train, cv=5); print("RF 5-fold CV score: {:.3f}".format(np.mean(cv)))
 ada = AdaBoostRegressor() ; cv = cross_val_score(ada, X_train, y_train, cv=5); print("Adaboosted 5-fold CV score: {:.3f}".format(np.mean(cv)))
 grad = GradientBoostingRegressor() ; cv = cross_val_score(grad, X_train, y_train, cv=5); print("Grad.boosted 5-fold CV score: {:.3f}".format(np.mean(cv)))
 estimators = [('LinReg',linr),('Ridge',ridge),('Lasso',lasso),('BayesianRidge', br),('SVMREg', svr),('RandomForest',rftree),('Gradboosted',grad)]
voting = VotingRegressor(estimators=estimators, n_jobs=-1); cv = cross_val_score(voting, X_train, y_train, cv=5); print("...all combined?! {:.3f}".format(np.mean(cv)))
 _ = plt.figure(figsize=(30,10)) ; _ = plt.plot(range(len(train.columns)) ,lasso_coef[0:80]); _ = plt.xticks(range(len(train.columns)) , train.columns, rotation=60); _ = plt.title('Most important predictors')
 print('
==Testing some of the basic models on data(lower RMSE indicated better fit)==');
grad.fit(X_train, y_train); y_pred = grad.predict(X_test); rmse_grad = np.sqrt(mean_squared_error(y_test, y_pred))
print('RMSE of grad.boosted regressor on test data: {:.3f}'.format(rmse_grad))
voting.fit(X_train, y_train); y_pred = voting.predict(X_test); rmse_vot = np.sqrt(mean_squared_error(y_test, y_pred))
print('RMSE of ensemble voting on test data: {:.3f}'.format(rmse_vot))
xg_reg = xgb.XGBRegressor(objective='reg:squarederror', n_estimators=10); xg_reg.fit(X_train, y_train); pred = xg_reg.predict(X_test); print('RMSE of XGBoost regressor: {:.3f}'.format(np.sqrt(mean_squared_error(y_test,pred))))
DM_train = xgb.DMatrix(data=X_train, label=y_train); DM_test = xgb.DMatrix(data=X_test, label=y_test); params = {'booster':'gblinear','objective':'reg:squarederror'}
cv_results = xgb.cv(dtrain=DM_train, params=params, nfold=5, num_boost_round=50, early_stopping_rounds=10, metrics="rmse", as_pandas=True)
 ',cv_results)
 xg_reg = xgb.train(params=params, dtrain=DM_train, num_boost_round=10); preds = xg_reg.predict(DM_test); print('RMSE of XGBoosted base learner: {:.3f}'.format(np.sqrt(mean_squared_error(y_test,preds)))) 
<save_to_csv>
assert torch.cuda.is_available()
assert torch.backends.cudnn.enabled, "NVIDIA/Apex:Amp requires cudnn backend to be enabled."
torch.backends.cudnn.benchmark = True
device = "cuda"
grad.fit(X, y)
 predict = np.expm1(grad.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 sub.head() 
<train_on_grid>
model = model.to(device )
params = {'alpha': [0.0001,0.001,0.01,0.1,1,5,10]}; lasso = Lasso()
lasso_mse = GridSearchCV(estimator=lasso, param_grid=params, scoring='neg_mean_squared_error', cv=4, n_jobs=-1)
 lasso_mse.fit(X,y); print("Best Lasso paramaters found:",lasso_mse.best_params_);print("Lowest RMSE found:", np.sqrt(np.abs(lasso_mse.best_score_)))
 params = {'alpha_1': [0.0000001,0.000001,0.00001,0.0001],
'alpha_2': [0.0000001,0.000001,0.00001,0.0001],
'lambda_1': [0.0000001,0.000001,0.00001,0.0001],
'lambda_2': [0.0000001,0.000001,0.00001,0.0001]}; br = BayesianRidge()
br_mse = GridSearchCV(estimator=br, param_grid=params, scoring='neg_mean_squared_error', cv=4, n_jobs=-1)
 br_mse.fit(X,y); print("Best Bayesian Ridge paramaters found:",br_mse.best_params_);print("Lowest RMSE found:", np.sqrt(np.abs(br_mse.best_score_)))
 gbm_param_grid = {'learning_rate':[0.001,0.01,0.1,0.2,0.3,0.5],
'max_depth': np.arange(3,12,1),
'n_estimators': [50,100,200,300],
'subsample': [0.1,0.2,0.3,0.5,0.75],
'colsample_bytree': np.arange(0.1,0.8,0.1)}
gbm = xgb.XGBRegressor(objective ='reg:squarederror')
 random_mse = RandomizedSearchCV(param_distributions=gbm_param_grid, n_iter=10, scoring='neg_mean_squared_error', cv=5, estimator=gbm, verbose=1)
 random_mse.fit(X,y); print("Best XGB paramaters found:",random_mse.best_params_);print("Lowest RMSE found:", np.sqrt(np.abs(random_mse.best_score_)))
<predict_on_test>
class FocalLoss(nn.Module):
def __init__(self, alpha=None, gamma=2., reduction='mean'):
super().__init__()
self.alpha = torch.tensor(alpha)
 self.gamma = gamma
self.reduction = reduction
def forward(self, inputs, targets):
self.alpha = self.alpha.type(inputs.type() , non_blocking=True)
 CE_loss = F.cross_entropy(inputs, targets, reduction='none')
 pt = torch.exp(-CE_loss)
 F_loss = self.alpha[targets] *(1-pt)**self.gamma * CE_loss
if self.reduction == 'sum':
return F_loss.sum()
elif self.reduction == 'mean':
return F_loss.mean()
return F_loss
predict = np.expm1(br_mse.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 print(sub.head())
 br = BayesianRidge(alpha_1=0.0001, alpha_2=1e-07, lambda_1=1e-07, lambda_2=0.0001)
 br.fit(X, y)
 predict = np.expm1(br.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 print(sub.head())
 xgb = xgb.XGBRegressor(learning_rate=0.1, max_depth=3, n_estimators=300, subsample=0.75)
 xgb.fit(X, y)
 predict = np.expm1(xgb.predict(test))
sub = pd.DataFrame({'Id':test.index, 'SalePrice':predict})
 print(sub.head())
 sub.to_csv('./submission.csv', index = False )<set_options>
activation = lambda y: y
def cont_kappa(input, targets, activation=None):

 n = len(targets)
 y = targets.float().unsqueeze(0)
 pred = input.float().squeeze(-1 ).unsqueeze(0)
 if activation is not None:
pred = activation(pred)
 wo =(pred - y)**2
we =(pred - y.t())**2
return 1 -(n * wo.sum() / we.sum())
warnings.filterwarnings("ignore" )<load_from_csv>
kappa_loss = lambda pred, y: 1 - cont_kappa(pred, y )
training_folder = ".. /input/vinbigdata-chest-xray-abnormalities-detection/train/"
df = pd.read_csv(".. /input/vinbigdata-chest-xray-abnormalities-detection/train.csv")
 df = df.query("class_id<14")
 df = df.query("rad_id=='R9'" )<count_duplicates>
y = torch.from_numpy(np.random.randint(5, size=6))
preds = torch.ones_like(y.float())* 2
print(y.tolist())
 print(preds.tolist())
 print(activation(preds ).tolist())
 cont_kappa(preds, y), kappa_loss(preds, y), F.mse_loss(preds, y.float() )
df["several_issues"] = df.duplicated(subset=['image_id'])
 df["box_size"] = [(row.y_max-row.y_min)*(row.x_max-row.x_min)for idx, row in df.iterrows() ]<groupby>
class MultiTaskLoss(FocalLoss):
def __init__(self, alpha=None, gamma=2.0, second_loss=F.mse_loss, second_mult=0.1):
super().__init__(alpha, gamma)
 self.second_loss = second_loss
self.second_mult = second_mult
def forward(self, inputs, targets):
loss = super().forward(inputs[...,:-1], targets)
 loss += self.second_mult * self.second_loss(inputs[...,-1], targets.float())
 return loss
df.groupby("class_id")["box_size"].mean()<groupby>
criterion = MultiTaskLoss(gamma=2., alpha=class_weights, second_loss=kappa_loss, second_mult=0.5)
 lr = 1e-2
optimizer = optim.SGD([
{
"params": chain(model.stem.parameters() , model.blocks.parameters()),
"lr": lr * 0.1,
},
{
"params": model.head[:6].parameters() ,
"lr": lr * 0.2,
},
{
"params": model.head[6].parameters() ,
"lr": lr
}],
momentum=0.99, weight_decay=1e-4, nesterov=True )
df.groupby("class_id" ).image_id.count()<define_variables>
if use_amp:
model, optimizer = amp.initialize(model, optimizer, opt_level="O2", num_losses=1 )
selected_classes = [0,3,5,7,10]
category_list = ["Aortic enlargement", "Cardiomegaly", "ILD", "Lung Opacity", "Pleural effusion"]
filtered_df = df.query("class_id in @selected_classes" )<feature_engineering>
def update_fn(engine, batch):
x = convert_tensor(batch[0], device=device, non_blocking=True)
 y = convert_tensor(batch[1], device=device, non_blocking=True)
 model.train()
y_pred = model(x)
 loss = criterion(y_pred, y)
 optimizer.zero_grad()
if use_amp:
with amp.scale_loss(loss, optimizer, loss_id=0)as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
optimizer.step()
return {
"batchloss": loss.item() ,
}
selected_classes_dict = {"0":0,"3":1,"5":2,"7":3,"10":4}
filtered_df["reclass_id"] = [selected_classes_dict[str(row.class_id)] for idx, row in filtered_df.iterrows() ]<categorify>
torch.cuda.empty_cache()
gc.collect()
try:
batch = next(iter(train_loader))
res = update_fn(engine=None, batch=batch)
 print(res)
 finally:
print('max_memory_allocated:', torch.cuda.max_memory_allocated())
 del batch
torch.cuda.empty_cache()
_ = gc.collect()
def get_mask(img_dimensions, x_min, y_min, x_max, y_max):
img_height, img_width = img_dimensions
img_mask = np.full(( img_height,img_width),0)
 img_mask[y_min:y_max,x_min:x_max] = 255
return img_mask.astype(np.float32)
 def rle_encoding(x):
dots = np.where(x.T.flatten() == 255)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b + 1, 0))
run_lengths[-1] += 1
prev = b
return ' '.join([str(x)for x in run_lengths] )<prepare_x_and_y>
from ignite.engine import Engine, Events, create_supervised_evaluator
from ignite.metrics import RunningAverage, Accuracy, Precision, Recall, Loss, TopKCategoricalAccuracy
from ignite.contrib.handlers import TensorboardLogger
from ignite.contrib.handlers.tensorboard_logger import OutputHandler, OptimizerParamsHandler
def read_xray(path, voi_lut = True, fix_monochrome = True):
dicom = pydicom.read_file(path)
 if voi_lut:
data = apply_voi_lut(dicom.pixel_array, dicom)
 else:
data = dicom.pixel_array
if fix_monochrome and dicom.PhotometricInterpretation == "MONOCHROME1":
data = np.amax(data)- data
data = data - np.min(data)
 data = data / np.max(data)
 data =(data * 255 ).astype(np.uint8)
 return data<count_values>
def qw_kappa(pred, y):
return cohen_kappa_score(torch.argmax(pred[...,:-1], dim=1 ).cpu().numpy() ,
y.cpu().numpy() ,
weights='quadratic')
 def cl_accuracy(pred, y):
return accuracy_score(torch.argmax(pred[...,:-1], dim=1 ).cpu().numpy() ,
y.cpu().numpy())
 trainer = Engine(update_fn)
 metrics = {
'Loss': Loss(criterion),
'Accuracy': Loss(cl_accuracy),
'ClKappa': Loss(qw_kappa),
'RgKappa': Loss(lambda pred, y: cont_kappa(pred[...,-1], y)) ,
}
evaluator = create_supervised_evaluator(model, metrics=metrics, device=device, non_blocking=True)
 class Metrics(RunningAverage):
def __init__(self, evaluator, output_transform=None, interactive=None):
super().__init__(alpha=0.9, output_transform=output_transform)
 self._evaluator = evaluator
self.interactive = interactive
self.validation_history = {}
self.loss_history = []
def attach(self, engine, name):
super().attach(engine, name)
 engine.add_event_handler(Events.EPOCH_COMPLETED, self.run_evaluation)
 def compute(self):
loss = super().compute()
self.loss_history.append(loss)
 return loss
def run_evaluation(self, engine=None):
self._evaluator.run(eval_train_loader)
 if self.interactive:
print(evaluator.state.metrics)
 for k,v in self._evaluator.state.metrics.items() :
if k not in self.validation_history.keys() :
self.validation_history[k] = [v]
else:
self.validation_history[k].append(v)
 history = Metrics(evaluator, output_transform=lambda out: out['batchloss'], interactive=is_interactive())
 history.attach(trainer, "batchloss" )
filtered_df.groupby("class_id" ).image_id.count()<create_dataframe>
log_path = "./log"
tb_logger = TensorboardLogger(log_dir=log_path)
 tb_logger.attach(trainer,
log_handler=OutputHandler('training', ['batchloss', ]),
event_name=Events.ITERATION_COMPLETED )
balanced_filtered_df = pd.DataFrame()
samples_per_class = 500
for class_name in filtered_df.class_name.unique() :
balanced_filtered_df = balanced_filtered_df.append(filtered_df.query("class_name==@class_name")[:samples_per_class],
ignore_index=True )<create_dataframe>
weight_decay = 1e-4
cycle_mult = 2
sim_epochs = 31
epoch_size = len(train_loader)
 lr_sched_params = {'param_name':'lr', 'cycle_size':epoch_size, 'cycle_mult':cycle_mult,
'start_value_mult':0.8, 'end_value_mult':0.25}
mom_sched_params = {'param_name':'momentum', 'cycle_size':epoch_size, 'cycle_mult':cycle_mult,
'start_value':0.85, 'end_value':0.96}
wd_sched_params = {'param_name':'weight_decay', 'cycle_size':epoch_size*sim_epochs, 'cycle_mult':cycle_mult,
'start_value':weight_decay/10, 'end_value':weight_decay*10}
lr = 0.85
body_sched_params = {**lr_sched_params, 'start_value':lr/10, 'end_value':lr/100}
body_sched = CosineAnnealingScheduler(optimizer.param_groups[0], **body_sched_params)
 feat_sched_params = {**lr_sched_params, 'start_value':lr/5, 'end_value':lr/50}
feat_sched = CosineAnnealingScheduler(optimizer.param_groups[1], **feat_sched_params)
 head_sched_params = {**lr_sched_params, 'start_value':lr, 'end_value':lr/20}
head_sched = CosineAnnealingScheduler(optimizer.param_groups[2], **feat_sched_params)
 mom_sched = CosineAnnealingScheduler(optimizer, **mom_sched_params)
 wd_sched = CosineAnnealingScheduler(optimizer, **wd_sched_params)
 schedulers = [body_sched, feat_sched, head_sched, mom_sched, wd_sched]
names = ["lr(body)", "lr(feat)", "lr(head)", "momentum", "wd"]
lr_values0 = np.array(body_sched.simulate_values(num_events=epoch_size*sim_epochs, **body_sched_params))
lr_values1 = np.array(feat_sched.simulate_values(num_events=epoch_size*sim_epochs, **feat_sched_params))
lr_values2 = np.array(head_sched.simulate_values(num_events=epoch_size*sim_epochs, **head_sched_params))
wd_values = np.array(wd_sched.simulate_values(num_events=epoch_size*sim_epochs, **wd_sched_params))
mom_values = np.array(mom_sched.simulate_values(num_events=epoch_size*sim_epochs, **mom_sched_params))
fig = plt.figure(figsize=(16, 4))
ax = plt.subplot()
plt.title(f"Cosine annealing with start_value_mult={lr_sched_params['start_value_mult']}")
 ax.plot(lr_values0[:, 0], lr_values0[:, 1], label=names[0])
 ax.plot(lr_values1[:, 0], lr_values1[:, 1], label=names[1])
 ax.plot(lr_values2[:, 0], lr_values2[:, 1], label=names[2])
 ax.plot(wd_values[:, 0], wd_values[:, 1], label=names[4])
 ax.set_yscale('log')
 ax.set_xlabel('Batches processed')
 ax.set_ylabel("learning rate")
 ax.legend(frameon=False, loc='upper right')
 ax2 = ax.twinx()
ax2.plot(mom_values[:, 0], mom_values[:, 1], label=names[3])
 ax2.set_ylabel("momentum")
 ax2.legend(frameon=False, loc='lower right')
 _ = ax.plot()
samples_df = pd.DataFrame(diagnostic_per_image)
 samples_df["Height"] = image_size
samples_df["Width"] = image_size<define_variables>
scheduler = ParamGroupScheduler(schedulers=schedulers, names=names)
 trainer.add_event_handler(Events.ITERATION_STARTED, scheduler )
DATA_DIR = Path('.. /working/')
 ROOT_DIR = ".. /.. /working"
NUM_CATS = len(selected_classes)
 IMAGE_SIZE = 512
os.chdir('.. /working/maskrcnn-tf2-keras')
 sys.path.append(ROOT_DIR+'/maskrcnn-tf2-keras')
<define_variables>
tb_logger.attach(trainer,
log_handler=OptimizerParamsHandler(optimizer, "lr"),
event_name=Events.EPOCH_STARTED )
COCO_WEIGHTS_PATH = '.. /.. /input/coco-weights/mask_rcnn_coco.h5'
class DiagnosticConfig(Config):
NAME = "Diagnostic"
NUM_CLASSES = NUM_CATS + 1
GPU_COUNT = 1
IMAGES_PER_GPU = 2
BACKBONE = 'resnet50'
IMAGE_MIN_DIM = IMAGE_SIZE
IMAGE_MAX_DIM = IMAGE_SIZE
IMAGE_RESIZE_MODE = 'none'
POST_NMS_ROIS_TRAINING = 250
POST_NMS_ROIS_INFERENCE = 150
MAX_GROUNDTRUTH_INSTANCES = 5
BACKBONE_STRIDES = [4, 8, 16, 32, 64]
BACKBONESHAPE =(8, 16, 24, 32, 48)
 RPN_ANCHOR_SCALES =(8,16,24,32,48)
 ROI_POSITIVE_RATIO = 0.33
DETECTION_MAX_INSTANCES = 300
DETECTION_MIN_CONFIDENCE = 0.7
STEPS_PER_EPOCH = int(len(samples_df)*0.9/IMAGES_PER_GPU)
 VALIDATION_STEPS = len(samples_df)-int(len(samples_df)*0.9/IMAGES_PER_GPU)
 config = DiagnosticConfig()
config.display()<import_modules>
def setup_logger(logger):
handler = logging.StreamHandler()
formatter = logging.Formatter("%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
 handler.setFormatter(formatter)
 logger.addHandler(handler)
 logger.setLevel(logging.INFO )
class DiagnosticDataset(utils.Dataset):
def __init__(self, df):
super().__init__(self)
 for i, name in enumerate(category_list):
self.add_class("diagnostic", i+1, name)
 for i, row in df.iterrows() :
self.add_image("diagnostic",
image_id=row.name,
path=".. /"+resized_folder+str(row.image_id)+".jpg",
labels=row['CategoryId'],
annotations=row['EncodedPixels'],
height=row['Height'], width=row['Width'])
 def image_reference(self, image_id):
info = self.image_info[image_id]
return info['path'], [category_list[int(x)] for x in info['labels']]
def load_image(self, image_id):
return cv2.imread(self.image_info[image_id]['path'])
 def load_mask(self, image_id):
info = self.image_info[image_id]
mask = np.zeros(( IMAGE_SIZE, IMAGE_SIZE, len(info['annotations'])) , dtype=np.uint8)
 labels = []
for m,(annotation, label)in enumerate(zip(info['annotations'], info['labels'])) :
sub_mask = np.full(info['height']*info['width'], 0, dtype=np.uint8)
 annotation = [int(x)for x in annotation.split(' ')]
for i, start_pixel in enumerate(annotation[::2]):
sub_mask[start_pixel: start_pixel+annotation[2*i+1]] = 1
sub_mask = sub_mask.reshape(( info['height'], info['width']), order='F')
 sub_mask = cv2.resize(sub_mask,(IMAGE_SIZE, IMAGE_SIZE), interpolation=cv2.INTER_NEAREST)
 mask[:, :, m] = sub_mask
labels.append(int(label)+1)
 return mask, np.array(labels )<create_dataframe>
trainer.add_event_handler(Events.ITERATION_COMPLETED, TerminateOnNan())
 def default_score_fn(engine):
score = engine.state.metrics['ClKappa']
return score
best_model_handler = ModelCheckpoint(dirname=log_path,
filename_prefix="best",
n_saved=10,
score_name="ClKappa",
score_function=default_score_fn,
require_empty=False)
 evaluator.add_event_handler(Events.COMPLETED, best_model_handler, {'model': model, })
 @trainer.on(Events.EPOCH_COMPLETED)
 @evaluator.on(Events.COMPLETED)
 def empty_cuda_cache(engine):
torch.cuda.empty_cache()
gc.collect()
training_percentage = 0.9
training_set_size = int(training_percentage*len(samples_df))
validation_set_size = int(( 1-training_percentage)*len(samples_df))
train_dataset = DiagnosticDataset(samples_df[:training_set_size])
 train_dataset.prepare()
valid_dataset = DiagnosticDataset(samples_df[training_set_size:training_set_size+validation_set_size])
 valid_dataset.prepare()
for i in range(10):
image_id = random.choice(train_dataset.image_ids)
 image = train_dataset.load_image(image_id)
 mask, class_ids = train_dataset.load_mask(image_id)
 visualize.display_top_masks(image, mask, class_ids, train_dataset.class_names, limit=5 )<load_pretrained>
@trainer.on(Events.EPOCH_STARTED)
 def tweak_bn_momenta(engine):
epoch = engine.state.epoch
if epoch <= 6:
momentum = 0.06 / epoch
pbar.log_message(f"setting bn momentum to {momentum}")
 for module in model.modules() :
if isinstance(module, nn.modules.batchnorm._BatchNorm):
module.momentum = momentum
LR = 1e-4
EPOCHS = [1,18]
model = modellib.MaskRCNN(mode='training', config=config, model_dir="")
 model.load_weights(COCO_WEIGHTS_PATH, by_name=True, exclude=['mrcnn_class_logits', 'mrcnn_bbox_fc', 'mrcnn_bbox', 'mrcnn_mask'] )<train_model>
num_epochs = 7 if debug else sim_epochs
state = trainer.run(train_loader, max_epochs=num_epochs )
%%time
model.train(train_dataset, valid_dataset,
learning_rate=LR,
epochs=EPOCHS[0],
layers='heads')
 history = model.keras_model.history.history<train_model>
evaluator.state.metrics
%%time
model.train(train_dataset, valid_dataset,
learning_rate=LR/10,
epochs=EPOCHS[1],
layers='all')
 new_history = model.keras_model.history.history
for k in new_history: history[k] = history[k] + new_history[k]<train_model>
!ls {log_path}
checkpoints = next(os.walk(log_path)) [2]
checkpoints = sorted(filter(lambda f: f.endswith(".pth"), checkpoints))
scores = [c.split('=')[-1][:-4] for c in checkpoints]
best_epoch = np.argmax(scores)
 print(best_epoch, scores)
 if not checkpoints:
print('No weight files in {}'.format(log_path))
else:
model_path = f'efficientNet_{scores[best_epoch]}.pth'
!cp {os.path.join(log_path, checkpoints[best_epoch])} {model_path}
print(model_path)
 !rm {log_path}/*.pth
best_epoch = np.argmin(history["val_loss"][1:])+ 1
print("Best epoch: ", best_epoch)
 print("Valid loss: ", history["val_loss"][1:][best_epoch-1] )<set_options>
best_model = model
best_model.load_state_dict(torch.load(model_path))
best_model = best_model.cuda().eval()
class InferenceConfig(DiagnosticConfig):
GPU_COUNT = 1
IMAGES_PER_GPU = 1
IMAGE_MIN_DIM = IMAGE_SIZE
IMAGE_MAX_DIM = IMAGE_SIZE
IMAGE_RESIZE_MODE = 'none'
DETECTION_MIN_CONFIDENCE = 0.8
DETECTION_NMS_THRESHOLD = 0.5
inference_config = InferenceConfig()
model = modellib.MaskRCNN(mode='inference',
config=inference_config,
model_dir="" )<load_pretrained>
with torch.no_grad() :
y_pred = best_model(batch[0][:16].cuda())
 print('Regressor activations:')
 print(activation(y_pred[:16,-1] ).reshape([2,8] ).cpu() , '
')
 print('Predictions for first item:')
 for idx in torch.topk(y_pred[0,:-1], k=n_classes)[1].squeeze(0 ).tolist() :
prob = torch.softmax(y_pred, dim=-1)[0, idx].item()
print('{label:<75}({p:.2f}%)'.format(label=classes[idx], p=prob*100))
glob_list = glob.glob(f'diagnostic*/mask_rcnn_diagnostic_{best_epoch:04d}.h5')
 model_path = glob_list[0] if glob_list else ''
model.load_weights(model_path, by_name=True )<feature_engineering>
del batch
def refine_masks(masks, rois):
areas = np.sum(masks.reshape(-1, masks.shape[-1]), axis=0)
 mask_index = np.argsort(areas)
 union_mask = np.zeros(masks.shape[:-1], dtype=bool)
 for m in mask_index:
masks[:, :, m] = np.logical_and(masks[:, :, m], np.logical_not(union_mask))
union_mask = np.logical_or(masks[:, :, m], union_mask)
 for m in range(masks.shape[-1]):
mask_pos = np.where(masks[:, :, m]==True)
 if np.any(mask_pos):
y1, x1 = np.min(mask_pos, axis=1)
 y2, x2 = np.max(mask_pos, axis=1)
 rois[m, :] = [y1, x1, y2, x2]
return masks, rois
def decode_rle(rle, height, width):
s = rle.split()
starts, lengths = [np.asarray(x, dtype=int)for x in(s[0:][::2], s[1:][::2])]
starts -= 1
ends = starts + lengths
img = np.zeros(height*width, dtype=np.uint8)
 for lo, hi in zip(starts, ends):
img[lo:hi] = 1
return img.reshape(( height, width)).T
def annotations_to_mask(annotations, height, width):
if isinstance(annotations, list):
mask = np.zeros(( height, width, len(annotations)))
 for i, rle_code in enumerate(annotations):
mask[:, :, i] = decode_rle(rle_code, height, width)
 else:
error_message = "{} is expected to be a list or str but received {}".format(annotation, type(annotation))
raise TypeError(error_message)
 return mask
def find_anomalies(dicom_image, display=False):
image_dimensions = dicom_image.shape
resized_img = cv2.resize(dicom_image,(image_size,image_size), interpolation = cv2.INTER_AREA)
 saved_filename = resized_test_folder+"temp_image.jpg"
cv2.imwrite(saved_filename, resized_img)
 img = cv2.imread(saved_filename)
 result = model.detect([img])
 r = result[0]
if r['masks'].size > 0:
masks = np.zeros(( img.shape[0], img.shape[1], r['masks'].shape[-1]), dtype=np.uint8)
 for m in range(r['masks'].shape[-1]):
masks[:, :, m] = cv2.resize(r['masks'][:, :, m].astype('uint8'), 
(img.shape[1], img.shape[0]), interpolation=cv2.INTER_NEAREST)
 y_scale = image_dimensions[0]/IMAGE_SIZE
x_scale = image_dimensions[1]/IMAGE_SIZE
rois =(r['rois'] * [y_scale, x_scale, y_scale, x_scale] ).astype(int)
 masks, rois = refine_masks(masks, rois)
 else:
masks, rois = r['masks'], r['rois']
if display:
visualize.display_instances(img, rois, masks, r['class_ids'],
['bg']+category_list, r['scores'],
title="prediction", figsize=(12, 12))
return rois, r['class_ids'], r['scores']<randomize_order>
use_regressor = False
def inference_update_with_tta(engine, batch, use_regressor=use_regressor):
global preds, targets
best_model.eval()
with torch.no_grad() :
x, y = batch
x = x.cuda()
if use_regressor:
y_pred1 = best_model(x)[...,-1]
y_pred2 = best_model(x.flip(dims=(-1,)))[...,-1]
curr_pred =(activation(y_pred1)+ activation(y_pred2)) * 0.5
preds += curr_pred.cpu().squeeze().tolist()
else:
y_pred1 = best_model(x)[...,:-1]
y_pred2 = best_model(x.flip(dims=(-1,)))[...,:-1]
curr_pred = F.softmax(y_pred1, dim=-1)+ F.softmax(y_pred2, dim=-1)
 preds += curr_pred.argmax(dim=-1 ).cpu().squeeze().tolist()
targets += y.cpu().squeeze().tolist()
return y_pred1, y
inferencer = Engine(inference_update_with_tta)
 ProgressBar(desc="Inference" ).attach(inferencer )
test_folder = ".. /.. /input/vinbigdata-chest-xray-abnormalities-detection/test/"
test_file_list = os.listdir(test_folder)[:5]
for test_file in test_file_list:
dicom_image = read_xray(test_folder+test_file)
 find_anomalies(dicom_image, display=True )<create_dataframe>
preds, targets = [], []
result_state = inferencer.run(eval_train_loader, max_epochs=1)
 print('valid accuracy:',(np.array(preds)== np.array(targets)).mean() )
submission_df = pd.DataFrame(results )<save_to_csv>
class KappaOptimizer(nn.Module):
def __init__(self):
super().__init__()
self.coef = [0.5, 1.5, 2.5, 3.5]
self.func = self.quad_kappa
def predict(self, preds):
return self._predict(self.coef, preds)
 @classmethod
def _predict(cls, coef, preds):
if type(preds ).__name__ == 'Tensor':
y_hat = preds.clone().view(-1)
 else:
y_hat = torch.FloatTensor(preds ).view(-1)
 for i,pred in enumerate(y_hat):
if pred < coef[0]: y_hat[i] = 0
elif pred < coef[1]: y_hat[i] = 1
elif pred < coef[2]: y_hat[i] = 2
elif pred < coef[3]: y_hat[i] = 3
else: y_hat[i] = 4
return y_hat.int()
def quad_kappa(self, preds, y):
return self._quad_kappa(self.coef, preds, y)
 @classmethod
def _quad_kappa(cls, coef, preds, y):
y_hat = cls._predict(coef, preds)
 return cohen_kappa_score(y, y_hat, weights='quadratic')
 def fit(self, preds, y):

 print('Early score:', self.quad_kappa(preds, y))
neg_kappa = lambda coef: -self._quad_kappa(coef, preds, y)
 opt_res = sp.optimize.minimize(neg_kappa, x0=self.coef, method='nelder-mead',
options={'maxiter':100, 'fatol':1e-20, 'xatol':1e-20})
 print(opt_res)
 self.coef = opt_res.x
print('New score:', self.quad_kappa(preds, y))
def forward(self, preds, y):

 return torch.tensor(self.quad_kappa(preds, y))
if use_regressor:
kappa_opt = KappaOptimizer()
kappa_opt.fit(preds, targets)
 opt_preds = kappa_opt.predict(preds ).tolist()
_ = pd.DataFrame(preds ).hist()
_ = pd.DataFrame(opt_preds ).hist()
submission_df.to_csv('.. /submission.csv', index=False )<drop_column>
preds, targets = [], []
result_state = inferencer.run(test_loader, max_epochs=1 )
!rm -rf.. /.. /working/resized_train/
!rm -rf.. /.. /working/resized_test/<import_modules>
if use_regressor:
preds = kappa_opt.predict(preds ).tolist()
import numpy as np
import pandas as pd<load_from_csv>
submission = pd.DataFrame({'id_code': pd.read_csv('.. /input/aptos2019-blindness-detection/sample_submission.csv' ).id_code.values,
'diagnosis': np.squeeze(preds ).astype(np.int32)})
 submission.hist()
submission.head()
sub_image = pd.read_csv('/kaggle/input/melanoma-submissions2/submissionImage.csv')
 sub_tabular = pd.read_csv('/kaggle/input/melanoma-submissions2/submissionTabular.csv')
 sub_tabular78 = pd.read_csv('/kaggle/input/tabular-prediction/submission_tabular_78.csv')
 sub_multiple = pd.read_csv('/kaggle/input/melanoma-submissions2/submission_multiple_data_source.csv')
 sub_public_merge = pd.read_csv('/kaggle/input/submission-9/submission_935.csv')
 sub = pd.read_csv('/kaggle/input/siim-isic-melanoma-classification/sample_submission.csv')
 sub_score946 = pd.read_csv('.. /input/submission-9/submission_946.csv')
 sub_score940 = pd.read_csv('.. /input/submission-9/submission_940.csv')
 sub_mean = pd.read_csv('/kaggle/input/siim-isic-multiple-model-training-stacking-923/submission_mean.csv' )<define_variables>
submission.to_csv('submission.csv', index=False )
sub.target = sub_score946.target *0.98 + sub_score940.target *0.0 + sub_tabular78.target *0.02<save_to_csv>
submission.to_csv('submission.csv', index=False )