Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<compute_test_metric>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
def func_to_opt(scores):
score = []
for x in scores:
v = x[0] * leak_df['pred1'].values + x[1] * leak_df['pred2'].values + \
x[2] * leak_df['pred3'].values + x[3] * leak_df['pred4'].values
val =(v > 0 ).astype(int)*v
vl1p = np.log1p(val)
 curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
score.append(curr_score)
 return np.array(score )<train_model>
from torchvision.models import *
%%time
ga_holder = GAOptimizer(function=lambda l: func_to_opt(l))
result = ga_holder.fit(40)
 print(result )<concatenate>
model_path='.'
path='.. /input/histopathologic-cancer-detection/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
final = result[0].flatten() /result[0].flatten().sum()
print(final )<find_best_params>
test_fnames=[str(file)for file in Path(test_folder ).iterdir() ]
func_to_opt(np.array([final]))<feature_engineering>
df_trn=pd.read_csv(train_lbl )
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
w1 = final[0]
w2 = final[1]
w3 = final[2]
w4 = final[3]
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred2 + w3 * test_df.pred3 +\
w4 * test_df.pred4
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<feature_engineering>
df_WSI=pd.read_csv('.. /input/histopathologiccancerwsi/patch_id_wsi.csv' )
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<count_missing_values>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=0.0, max_zoom=.15,
max_lighting=0.1, max_warp=0.15 )
sample_submission.isna().sum()<save_to_csv>
df_notinWSI=df_trn.set_index('id' ).drop(df_WSI.id )
sample_submission.to_csv('submission.csv', index=False, float_format='%.5f' )<set_options>
valWSI=df_WSI.groupby(by='wsi')['id'].count().sample(frac=0.23 ).index
py.init_notebook_mode(connected=True)
<set_options>
trnWSI=[i[0] for i in df_WSI.groupby(by='wsi')['id'] if i[0] not in valWSI]
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype('category')
 end_mem = df.memory_usage().sum() / 1024**2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 *(start_mem - end_mem)/ start_mem))
return df<load_from_disk>
val_idx=np.hstack([df_WSI.groupby(by='wsi')['id'].indices[WSI] for WSI in valWSI] )
%%time
root = Path('.. /input/ashrae-feather-format-for-fast-loading')
 train_df = pd.read_feather(root/'train.feather')
 test_df = pd.read_feather(root/'test.feather')
 building_meta_df = pd.read_feather(root/'building_metadata.feather' )<feature_engineering>
val_idx=np.append(df_notinWSI.index.values,df_WSI.id[val_idx])
leak_df = pd.read_feather('.. /input/ashrae-leak-data-station/leak.feather')
 leak_df.fillna(0, inplace=True)
 leak_df = leak_df[(leak_df.timestamp.dt.year > 2016)&(leak_df.timestamp.dt.year < 2019)]
leak_df.loc[leak_df.meter_reading < 0, 'meter_reading'] = 0
leak_df = leak_df[leak_df.building_id!=245]
lead_site_id_0 = leak_df[leak_df['meter']==0]<load_from_csv>
trn_idx=np.hstack([df_WSI.groupby(by='wsi')['id'].indices[WSI] for WSI in trnWSI])
 trn_idx=df_WSI.id[trn_idx]
sample_submission3 = pd.read_csv(".. /input/ashrae-final-submission/fe2_lgbm_tz.csv", index_col=0)
 sample_submission2 = pd.read_csv(".. /input/ashrae-final-submission/submission.csv", index_col=0)
 sample_submission1 = pd.read_csv(".. /input/ashrae-final-submission/submission_final_day.csv", index_col=0 )<feature_engineering>
val_idx=df_trn.reset_index().set_index('id' ).loc[val_idx,'index'].values
trn_idx=df_trn.reset_index().set_index('id' ).loc[trn_idx,'index'].values
test_df['pred1'] = sample_submission1.meter_reading
test_df['pred2'] = sample_submission2.meter_reading
test_df['pred3'] = sample_submission3.meter_reading
del sample_submission1, sample_submission2, sample_submission3
gc.collect()
test_df = reduce_mem_usage(test_df)
 leak_df = reduce_mem_usage(leak_df )<merge>
np.random.shuffle(val_idx)
 np.random.shuffle(trn_idx )
leak_df = leak_df.merge(test_df[['building_id', 'meter', 'timestamp', 'pred1', 'pred2', 'pred3','row_id']], left_on = ['building_id', 'meter', 'timestamp'], right_on = ['building_id', 'meter', 'timestamp'], how = "left")
 leak_df = leak_df.merge(building_meta_df[['building_id', 'site_id']], on='building_id', how='left' )<feature_engineering>
src =(ImageList.from_df(df_trn, path=path, suffix='.tif',folder='train')
.split_by_idxs(trn_idx,val_idx)
.label_from_df(label_delim=' '))
src.add_test(test_fnames);
leak_df['pred1_l1p'] = np.log1p(leak_df.pred1)
 leak_df['pred2_l1p'] = np.log1p(leak_df.pred2)
 leak_df['pred3_l1p'] = np.log1p(leak_df.pred3)
 leak_df['meter_reading_l1p'] = np.log1p(leak_df.meter_reading )<compute_test_metric>
data=ImageDataBunch.create_from_ll(src, ds_tfms=tfms, size=sz,bs=bs)
 stats=data.batch_stats()
data.normalize(stats);
v = 0.1 * leak_df['pred1'].values + 0.6 * leak_df['pred2'].values + 0.2 * leak_df['pred3'].values
vl1p = np.log1p(v)
 curr_score = np.sqrt(mean_squared_error(vl1p, leak_df.meter_reading_l1p))
print(curr_score )<feature_engineering>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true[:,1],torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
class FocalLoss(nn.Module):
def __init__(self, alpha=1, gamma=2, logits=False, reduce=True):
super(FocalLoss, self ).__init__()
self.alpha = alpha
self.gamma = gamma
self.logits = logits
self.reduce = reduce
def forward(self, inputs, targets):
if self.logits:
BCE_loss = F.binary_cross_entropy_with_logits(inputs, targets, reduce=False)
 else:
BCE_loss = F.binary_cross_entropy(inputs, targets, reduce=False)
 pt = torch.exp(-BCE_loss)
 F_loss = self.alpha *(1-pt)**self.gamma * BCE_loss
if self.reduce:
return torch.mean(F_loss)
 else:
return F_loss
sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))
w1 = 0.1
w2 = 0.6
w3 = 0.2
print("The weights are: w1=" + str(w1)+ ", w2=" + str(w2)+ ", w3=" + str(w3))
sample_submission['meter_reading'] = w1 * test_df.pred1 + w2 * test_df.pred2 + w3 * test_df.pred3
sample_submission.loc[sample_submission.meter_reading < 0, 'meter_reading'] = 0<feature_engineering>
learn = create_cnn(
 data,
densenet169,
path='.',
metrics=[auc_score],
ps=0.5
)
leak_df = leak_df[['meter_reading', 'row_id']].set_index('row_id' ).dropna()
sample_submission.loc[leak_df.index, 'meter_reading'] = leak_df['meter_reading']<save_to_csv>
x,y=learn.get_preds()
sample_submission.to_csv('submission.csv', index=False, float_format='%.4f' )<define_variables>
auc_score(x,y )
DATA_PATH = ".. /input/ashrae-energy-prediction/"<load_from_csv>
learn.fit_one_cycle(1,1e-2)
 learn.recorder.plot()
learn.recorder.plot_losses()
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 train_df = train_df [ train_df['building_id'] != 1099 ]
train_df = train_df.query('not(building_id <= 104 & meter == 0 & timestamp <= "2016-05-20")')
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )<data_type_conversions>
learn.unfreeze()
learn.lr_find()
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp_formatted"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] =(df["timestamp_formatted"].dt.hour ).fillna(-1 ).astype(np.int8)
 df["weekend"] =(df["timestamp_formatted"].dt.weekday ).fillna(-1 ).astype(np.int8)
 df["day"] =(df["timestamp_formatted"].dt.day ).fillna(-1 ).astype(np.int8)
 df["month"] =(df["timestamp_formatted"].dt.month ).fillna(-1 ).astype(np.int8)
 df['square_feet_log'] = np.log1p(df['square_feet'])
 df['years'] =(df["year_built"] - 1900 ).fillna(-1 ).astype(np.int8)
 df['wind_compass'] = -1
df.loc[(df['wind_direction'] >= 315)|(df['wind_direction'] < 45), 'wind_compass'] = 0
df.loc[(df['wind_direction'] >= 45)&(df['wind_direction'] < 135), 'wind_compass'] = 1
df.loc[(df['wind_direction'] >= 135)&(df['wind_direction'] < 225), 'wind_compass'] = 2
df.loc[(df['wind_direction'] >= 225)&(df['wind_direction'] < 315), 'wind_compass'] = 3
df['wind_compass'] = df['wind_compass'].astype(np.int8)
 df['wind_scale'] = -1
df.loc[df['wind_speed'] < 0.5, 'wind_scale'] = 0
df.loc[(df['wind_speed'] >= 0.5)&(df['wind_speed'] < 1.5), 'wind_scale'] = 1
df.loc[(df['wind_speed'] >= 1.5)&(df['wind_speed'] < 3.5), 'wind_scale'] = 2
df.loc[(df['wind_speed'] >= 3.5)&(df['wind_speed'] < 5.5), 'wind_scale'] = 3
df.loc[(df['wind_speed'] >= 5.5)&(df['wind_speed'] < 8.5), 'wind_scale'] = 4
df.loc[df['wind_speed'] >= 8.5, 'wind_scale'] = 5
df['wind_scale'] = df['wind_scale'].astype(np.int8)
 drop = ["timestamp",'square_feet',"timestamp_formatted","year_built","sea_level_pressure", "wind_direction", "wind_speed"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage is: {:.2f} MB".format(end_mem))
return df<create_dataframe>
learn.fit_one_cycle(10,slice(1e-4,1e-3))
weather_df = fill_weather_dataset(weather_df )<categorify>
preds,y,losses = learn.get_preds(with_loss=True)
 interp = ClassificationInterpretation(learn, preds, y.long() , losses )
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )<merge>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()<feature_engineering>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
train_df = features_engineering(train_df )<prepare_x_and_y>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
target = np.log1p(train_df["meter_reading"])
 features = train_df.drop('meter_reading', axis = 1)
 del train_df, sample
gc.collect()<split>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend", "day", "month", 'wind_compass', 'wind_scale']
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 2,
"metric": "rmse",
}
kf = KFold(n_splits=3)
 models = []
for train_index,test_index in kf.split(features):
train_features = features.loc[train_index]
train_target = target.loc[train_index]
test_features = features.loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()<set_options>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
del features, target
gc.collect()<load_from_csv>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )<merge>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
<load_from_csv><EOS> 
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )