Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<count_values>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
leak_df.meter.value_counts()<count_duplicates>
!git clone https://github.com/adambielski/GrouPy.git
sys.path.insert(0,'./GrouPy/')
print(leak_df.duplicated().sum() )<feature_engineering>
class BasicBlock(nn.Module):
expansion = 1
def __init__(self, in_planes, planes, stride=1):
super(BasicBlock, self ).__init__()
self.conv1 = P4MConvP4M(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
 self.bn1 = nn.BatchNorm3d(planes)
 self.conv2 = P4MConvP4M(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
 self.bn2 = nn.BatchNorm3d(planes)
 self.shortcut = nn.Sequential()
if stride != 1 or in_planes != self.expansion*planes:
self.shortcut = nn.Sequential(
 P4MConvP4M(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
nn.BatchNorm3d(self.expansion*planes)
)
 def forward(self, x):
out = F.relu(self.bn1(self.conv1(x)))
 out = self.bn2(self.conv2(out))
out += self.shortcut(x)
 out = F.relu(out)
 return out
class Bottleneck(nn.Module):
expansion = 4
def __init__(self, in_planes, planes, stride=1):
super(Bottleneck, self ).__init__()
self.conv1 = P4MConvP4M(in_planes, planes, kernel_size=1, bias=False)
 self.bn1 = nn.BatchNorm3d(planes)
 self.conv2 = P4MConvP4M(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
 self.bn2 = nn.BatchNorm3d(planes)
 self.conv3 = P4MConvP4M(planes, self.expansion*planes, kernel_size=1, bias=False)
 self.bn3 = nn.BatchNorm3d(self.expansion*planes)
 self.shortcut = nn.Sequential()
if stride != 1 or in_planes != self.expansion*planes:
self.shortcut = nn.Sequential(
 P4MConvP4M(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
nn.BatchNorm3d(self.expansion*planes)
)
 def forward(self, x):
out = F.relu(self.bn1(self.conv1(x)))
 out = F.relu(self.bn2(self.conv2(out)))
 out = self.bn3(self.conv3(out))
out += self.shortcut(x)
 out = F.relu(out)
 return out
class ResNet(nn.Module):
def __init__(self, block, num_blocks, num_classes=10):
super(ResNet, self ).__init__()
self.in_planes = 23
self.conv1 = P4MConvZ2(3, 23, kernel_size=3, stride=1, padding=1, bias=False)
 self.bn1 = nn.BatchNorm3d(23)
 self.layer1 = self._make_layer(block, 23, num_blocks[0], stride=1)
 self.layer2 = self._make_layer(block, 45, num_blocks[1], stride=2)
 self.layer3 = self._make_layer(block, 91, num_blocks[2], stride=2)
 self.layer4 = self._make_layer(block, 181, num_blocks[3], stride=2)
 self.linear = nn.Linear(181*8*block.expansion, num_classes)
 def _make_layer(self, block, planes, num_blocks, stride):
strides = [stride] + [1]*(num_blocks-1)
 layers = []
for stride in strides:
layers.append(block(self.in_planes, planes, stride))
self.in_planes = planes * block.expansion
return nn.Sequential(*layers)
 def forward(self, x):
out = F.relu(self.bn1(self.conv1(x)))
 out = self.layer1(out)
 out = self.layer2(out)
 out = self.layer3(out)
 out = self.layer4(out)
 outs = out.size()
print(out.size() ,out.size(0))
out = out.view(outs[0], outs[1]*outs[2], outs[3], outs[4])
 print(out.size() ,out.size(0))
out = F.avg_pool2d(out, 4)
 print(out.size() ,out.size(0))
out = out.view(out.size(0), -1)
 print(out.size())
 out = self.linear(out)
 print(out.size())
 return out
def ResNet18(pretrained=False):
return ResNet(BasicBlock, [2,2,2,2])
 def ResNet34(pretrained=False):
return ResNet(BasicBlock, [3,4,6,3])
 def ResNet50(pretrained=False):
return ResNet(Bottleneck, [3,4,6,3])
 def ResNet101(pretrained=False):
return ResNet(Bottleneck, [3,4,23,3])
 def ResNet152(pretrained=False):
return ResNet(Bottleneck, [3,8,36,3] )
print(len(leak_df)/ len(train_df))<set_options>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=32
num_workers=None
sz=96
del train_df
gc.collect()<load_from_csv>
df_trn=pd.read_csv(train_lbl )
sample_submission1 = pd.read_csv('.. /input/ashrae-kfold-lightgbm-without-leak-1-08/submission.csv', index_col=0)
 sample_submission2 = pd.read_csv('.. /input/ashrae-half-and-half/submission.csv', index_col=0)
 sample_submission3 = pd.read_csv('.. /input/ashrae-highway-kernel-route4/submission.csv', index_col=0 )<feature_engineering>
tfms = get_transforms(do_flip=False, flip_vert=False, max_rotate=.0, max_zoom=.3,
max_lighting=0.15, max_warp=0.10 )
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
test_df.loc[test_df.pred3<0, 'pred3'] = 0
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )<merge>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train',bs=bs, ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder);
stats=data.batch_stats()
data.normalize(stats )
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3', 'row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<feature_engineering>
data.show_batch(rows=3, figsize=(12,9))
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )<filter>
def auc_score(y_pred,y_true):
return roc_auc_score(to_np(y_true),to_np(F.sigmoid(y_pred)) [:,1] )
leak_df[leak_df.pred1_l1p.isnull() ]<compute_train_metric>
class AdaptiveConcatPool3d(nn.Module):
"Layer that concats `AdaptiveAvgPool3d` and `AdaptiveMaxPool3d`."
def __init__(self, sz:Optional[int]=None):
"Output will be 2*sz or 2 if sz is None"
super().__init__()
sz = sz or 1
self.ap,self.mp = nn.AdaptiveAvgPool3d(sz), nn.AdaptiveMaxPool3d(sz)
 def forward(self, x): return torch.cat([self.mp(x), self.ap(x)], 1 )
leak_df['mean_pred'] = np.mean(leak_df[['pred1', 'pred2', 'pred3']].values, axis=1)
 leak_df['mean_pred_l1p'] = np.log1p(leak_df.mean_pred)
 leak_score = np.sqrt(mean_squared_error(leak_df.mean_pred_l1p, leak_df.meter_reading_l1p))
sns.distplot(leak_df.mean_pred_l1p)
 sns.distplot(leak_df.meter_reading_l1p)
 print('mean score=', leak_score )<compute_test_metric>
def create_head_custom(nf:int, nc:int, lin_ftrs:Optional[Collection[int]]=None, ps:Floats=0.5, bn_final:bool=False):
"Model head that takes `nf` features, runs through `lin_ftrs`, and about `nc` classes."
lin_ftrs = [nf, 512, nc] if lin_ftrs is None else [nf] + lin_ftrs + [nc]
ps = listify(ps)
 if len(ps)==1: ps = [ps[0]/2] *(len(lin_ftrs)-2)+ ps
actns = [nn.ReLU(inplace=True)] *(len(lin_ftrs)-2)+ [None]
layers = [AdaptiveConcatPool3d() , Flatten() ]
for ni,no,p,actn in zip(lin_ftrs[:-1],lin_ftrs[1:],ps,actns):
layers += bn_drop_lin(ni,no,True,p,actn)
 if bn_final: layers.append(nn.BatchNorm1d(lin_ftrs[-1], momentum=0.01))
return nn.Sequential(*layers)
N = 10
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p * leak_df['pred1'].values +(1.-p)* leak_df ['pred3'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
learn = create_cnn(
 data,
ResNet18,
path='.',
metrics=[fbeta],
ps=0.5,
pretrained=False
)
best_weight = np.argmin(scores)* 1./N
print(scores.min() , best_weight )<compute_test_metric>
learn.model[1]=create_head_custom(362,2)
 learn.model.to(learn.data.device)
 print(learn.summary())
scores = np.zeros(N,)
 for i in range(N):
p = i * 1./N
v = p *(best_weight * leak_df['pred1'].values +(1.-best_weight)* leak_df ['pred3'].values)+(1.-p)* leak_df ['pred2'].values
vl1p = np.log1p(v)
 scores[i] = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))<find_best_params>
learn.lr_find(num_it=500,end_lr=1000)
 learn.recorder.plot()
best_weight2 = np.argmin(scores)* 1./N
print(scores.min() , best_weight2)
<prepare_x_and_y>
learn.fit_one_cycle(2,slice(5e-5,1e-4))
X_train = np.array([leak_df['pred1'].values,leak_df['pred2'].values, leak_df['pred3'].values] ).T
y_train = leak_df.meter_reading_l1p<find_best_params>
learn.save('gCNN_1cycle' )
weights = tf.Variable([[0.3],[0.3],[0.3]])
 steps = 1000
lr = 0.1
opt = tf.optimizers.SGD(lr)
 @tf.function()
def train_step(opt):
with tf.GradientTape() as tape:
y = tf.matmul(X_train, weights)[:,0]
loss = tf.reduce_mean(( tf.math.log1p(y)- y_train)** 2)
 grads = tape.gradient(loss, weights)
 opt.apply_gradients([(grads, weights)])
 return loss
prev_loss = 9999
for i in range(steps):
loss = train_step(opt)
 if loss > prev_loss:
lr /= 2
opt.lr = lr
prev_loss = loss
print(f'step: {i} {loss.numpy() }')
<compute_test_metric>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
np.sqrt(loss )<feature_engineering>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
ws = weights.numpy()
w1 = ws[0,0]
w2 = ws[1,0]
w3 = ws[2,0]
print("The weights are: w1=" + str(w1)+ ", w2=" + str(w2)+ ", w3=" + str(w3))
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred2 + w3 * test_df.pred3
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<feature_engineering>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<save_to_csv>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<set_options>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage().sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df
path_data = "/kaggle/input/ashrae-energy-prediction/"
save_grid = "grid_search.csv"
path_train = path_data + "train.csv"
path_test = path_data + "test.csv"
path_building = path_data + "building_metadata.csv"
path_weather_train = path_data + "weather_train.csv"
path_weather_test = path_data + "weather_test.csv"
plt.style.use("seaborn")
 sns.set(font_scale=1)
 df_train = pd.read_csv(path_train)
 building = pd.read_csv(path_building)
 le = LabelEncoder()
building.primary_use = le.fit_transform(building.primary_use)
 weather_train = pd.read_csv(path_weather_train)
 df_train = reduce_mem_usage(df_train)
 building = reduce_mem_usage(building)
 weather_train = reduce_mem_usage(weather_train)
 def prepare_data(X, building_data, weather_data, test=False):

 X = X.merge(building_data, on="building_id", how="left")
 X = X.merge(weather_data, on=["site_id", "timestamp"], how="left")
 X.timestamp = pd.to_datetime(X.timestamp, format="%Y-%m-%d %H:%M:%S")
 X.square_feet = np.log1p(X.square_feet)
 if not test:
X.sort_values("timestamp", inplace=True)
 X.reset_index(drop=True, inplace=True)
 gc.collect()
holidays = ["2016-01-01", "2016-01-18", "2016-02-15", "2016-05-30", "2016-07-04",
"2016-09-05", "2016-10-10", "2016-11-11", "2016-11-24", "2016-12-26",
"2017-01-01", "2017-01-16", "2017-02-20", "2017-05-29", "2017-07-04",
"2017-09-04", "2017-10-09", "2017-11-10", "2017-11-23", "2017-12-25",
"2018-01-01", "2018-01-15", "2018-02-19", "2018-05-28", "2018-07-04",
"2018-09-03", "2018-10-08", "2018-11-12", "2018-11-22", "2018-12-25",
"2019-01-01"]
X["hour"] = X.timestamp.dt.hour
X["weekday"] = X.timestamp.dt.weekday
X["is_holiday"] =(X.timestamp.dt.date.astype("str" ).isin(holidays)).astype(int)
 drop_features = [ "sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count","cloud_coverage", "precip_depth_1_hr"]
X.drop(drop_features, axis=1, inplace=True)
 if test:
row_ids = X.row_id
X.drop("row_id", axis=1, inplace=True)
 return X, row_ids
else:
y = np.log1p(X.meter_reading)
 X.drop("meter_reading", axis=1, inplace=True)
 return X, y
X_train, y_train = prepare_data(df_train, building, weather_train)
 del df_train, weather_train
gc.collect()
y_train = y_train[~(( X_train.building_id <= 104)&(X_train.meter==0)&(X_train.timestamp<='2016-05-20')) ]
X_train = X_train[~(( X_train.building_id <= 104)&(X_train.meter==0)&(X_train.timestamp<='2016-05-20')) ]
y_train = y_train[X_train.building_id != 1099]
X_train = X_train[X_train.building_id != 1099]
X_train.drop("timestamp", axis=1, inplace=True)
 X_train.drop("building_id", axis=1, inplace=True)
 df_test = pd.read_csv(path_test)
 weather_test = pd.read_csv(path_weather_test)
 df_test = reduce_mem_usage(df_test)
 weather_test = reduce_mem_usage(weather_test)
 X_test, row_ids = prepare_data(df_test, building, weather_test, test=True)
 X_test.drop("timestamp", axis=1, inplace=True)
 X_test.drop("building_id", axis=1, inplace=True)
 nsplits = 3
folds = KFold(n_splits=nsplits)
 categorical_features = ["site_id", "meter", "hour", "weekday","primary_use"]
params = {
"objective": "regression",
"boosting": "gbdt",
"max_depth": -1,
"num_leaves": 1000,
"learning_rate": 0.1,
"feature_fraction": 0.95,
"reg_lambda": 2,
"max_bin": 550,
"metric": "rmse"
}
pred = np.zeros(len(X_test)) 
<split>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
df_fimp = pd.DataFrame()
for tr_idx,va_idx in folds.split(X_train,y_train):
X_half_1 = X_train.iloc[tr_idx]
y_half_1 = y_train.iloc[tr_idx]
X_half_2 = X_train.iloc[va_idx]
y_half_2 = y_train.iloc[va_idx]
d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)
 d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)
 print("Building model with first half and validating on second half:")
 model_half = lgb.train(params, train_set=d_half_1, num_boost_round=300, valid_sets=[d_half_1,d_half_2], verbose_eval=100, early_stopping_rounds=50)
 pred += np.expm1(model_half.predict(X_test, num_iteration=model_half.best_iteration)) / nsplits
df_fimp_1 = pd.DataFrame()
df_fimp_1["feature"] = X_train.columns.values
df_fimp_1["importance"] = model_half.feature_importance()
df_fimp = pd.concat([df_fimp, df_fimp_1], axis=0)
 del model_half
gc.collect()
plt.figure(figsize=(14, 7))
sns.barplot(x="importance", y="feature", data=df_fimp.sort_values(by="importance", ascending=False))
plt.title("LightGBM Feature Importance")
 plt.tight_layout()
pred = pred*0.90
submission = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(pred, 0, a_max=None)})
 submission.to_csv("submission.csv", index=False )<filter>
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_TTA_{pred_score}.csv' )