Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<merge>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()<feature_engineering>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
test_df = features_engineering(test_df )<predict_on_test>
df_trn=pd.read_csv(train_lbl )
results = np.zeros(len(test_df))
step = 50000
for model in models:
for i in tqdm(range(0, len(test_df), step)) :
results[i: min(i+step, len(test_df)) ] += np.expm1(model.predict(test_df.iloc[i: min(i+step, len(test_df)) , :], num_iteration=model.best_iteration))
del model
gc.collect()
results = results / len(models )<set_options>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)
del test_df, models
gc.collect()<save_to_csv>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
results_df = pd.DataFrame({"row_id": row_ids, "meter_reading": np.clip(results, 0, a_max=None)})
 del row_ids,results
gc.collect()
results_df.to_csv("submission.csv", index=False)
<define_variables>
data.show_batch(rows=5, figsize=(15,15))
DATA_PATH = ".. /input/ashrae-energy-prediction/"<load_from_csv>
from sklearn.metrics import roc_auc_score
train_df = pd.read_csv(DATA_PATH + 'train.csv')
 bad_rows = pd.read_csv('/kaggle/input/rows-to-drop/rows_to_drop.csv')
 train_df.drop(bad_rows.loc[:, '0'], inplace = True)
 train_df.reset_index(drop = True, inplace = True)
 building_df = pd.read_csv(DATA_PATH + 'building_metadata.csv')
 weather_df = pd.read_csv(DATA_PATH + 'weather_train.csv' )<data_type_conversions>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
def fill_weather_dataset(weather_df):
time_format = "%Y-%m-%d %H:%M:%S"
start_date = datetime.datetime.strptime(weather_df['timestamp'].min() ,time_format)
 end_date = datetime.datetime.strptime(weather_df['timestamp'].max() ,time_format)
 total_hours = int(((end_date - start_date ).total_seconds() + 3600)/ 3600)
 hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format)for x in range(total_hours)]
missing_hours = []
for site_id in range(16):
site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])
 new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])
 new_rows['site_id'] = site_id
weather_df = pd.concat([weather_df,new_rows])
 weather_df = weather_df.reset_index(drop=True)
 weather_df["datetime"] = pd.to_datetime(weather_df["timestamp"])
 weather_df["day"] = weather_df["datetime"].dt.day
weather_df["week"] = weather_df["datetime"].dt.week
weather_df["month"] = weather_df["datetime"].dt.month
weather_df = weather_df.set_index(['site_id','day','month'])
 air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean() ,columns=["air_temperature"])
 weather_df.update(air_temperature_filler,overwrite=False)
 cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()
cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=["cloud_coverage"])
 weather_df.update(cloud_coverage_filler,overwrite=False)
 due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean() ,columns=["dew_temperature"])
 weather_df.update(due_temperature_filler,overwrite=False)
 sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()
sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])
 weather_df.update(sea_level_filler,overwrite=False)
 wind_direction_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean() ,columns=['wind_direction'])
 weather_df.update(wind_direction_filler,overwrite=False)
 wind_speed_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean() ,columns=['wind_speed'])
 weather_df.update(wind_speed_filler,overwrite=False)
 precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()
precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])
 weather_df.update(precip_depth_filler,overwrite=False)
 weather_df = weather_df.reset_index()
weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)
 return weather_df
def reduce_mem_usage(df, use_float16=False):

 start_mem = df.memory_usage().sum() / 1024**2
print("Memory usage of dataframe is {:.2f} MB".format(start_mem))
for col in df.columns:
if is_datetime(df[col])or is_categorical_dtype(df[col]):
continue
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == "int":
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
if use_float16 and c_min > np.finfo(np.float16 ).min and c_max < np.finfo(np.float16 ).max:
df[col] = df[col].astype(np.float16)
 elif c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 else:
df[col] = df[col].astype("category")
 end_mem = df.memory_usage().sum() / 1024**2
print("Memory usage after optimization is: {:.2f} MB".format(end_mem))
print("Decreased by {:.1f}%".format(100 *(start_mem - end_mem)/ start_mem))
return df
def features_engineering(df):
df.sort_values("timestamp")
 df.reset_index(drop=True)
 df["timestamp"] = pd.to_datetime(df["timestamp"],format="%Y-%m-%d %H:%M:%S")
 df["hour"] = df["timestamp"].dt.hour
df["weekend"] = df["timestamp"].dt.weekday
df['square_feet'] = np.log1p(df['square_feet'])
 df['month'] = df['timestamp'].dt.month
df['month'].replace(( 1, 2, 3, 4), 1, inplace = True)
 df['month'].replace(( 5, 6, 7, 8), 2, inplace = True)
 df['month'].replace(( 9, 10, 11, 12), 3, inplace = True)
 drop = ["timestamp","sea_level_pressure", "wind_direction", "wind_speed","year_built","floor_count"]
df = df.drop(drop, axis=1)
 gc.collect()
le = LabelEncoder()
df["primary_use"] = le.fit_transform(df["primary_use"])
 return df<create_dataframe>
from torchvision.models import *
weather_df = fill_weather_dataset(weather_df )<categorify>
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.8
)
train_df = reduce_mem_usage(train_df,use_float16=True)
 building_df = reduce_mem_usage(building_df,use_float16=True)
 weather_df = reduce_mem_usage(weather_df,use_float16=True )<merge>
lr = 3e-03
train_df = train_df.merge(building_df, left_on='building_id',right_on='building_id',how='left')
 train_df = train_df.merge(weather_df,how='left',left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])
 del weather_df
gc.collect()<feature_engineering>
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()
train_df = features_engineering(train_df )<prepare_x_and_y>
learn.unfreeze()
learn.lr_find()
target = np.log1p(train_df["meter_reading"])
 train_df = train_df.drop('meter_reading', axis = 1)
 gc.collect()<split>
learn.fit_one_cycle(1,slice(1e-6,1e-5))
categorical_features = ["building_id", "site_id", "meter", "primary_use", "weekend"]
features = [col for col in train_df.columns if col not in ['month']]
params = {
"objective": "regression",
"boosting": "gbdt",
"num_leaves": 1580,
"learning_rate": 0.05,
"feature_fraction": 0.85,
"reg_lambda": 1,
"metric": "rmse",
}
kf = StratifiedKFold(n_splits=5, shuffle=False, random_state=2319)
 models = []
for train_index,test_index in kf.split(train_df, train_df['month']):
train_features = train_df[features].loc[train_index]
train_target = target.loc[train_index]
test_features = train_df[features].loc[test_index]
test_target = target.loc[test_index]
d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)
 d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)
 model = lgb.train(params, train_set=d_training, num_boost_round=3000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)
 models.append(model)
 del train_features, train_target, test_features, test_target, d_training, d_test
gc.collect()<set_options>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
del features, target
gc.collect()<load_from_csv>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
test_df = pd.read_csv(DATA_PATH + 'test.csv')
 row_ids = test_df["row_id"]
test_df.drop("row_id", axis=1, inplace=True)
 test_df = reduce_mem_usage(test_df )<merge>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
test_df = test_df.merge(building_df,left_on='building_id',right_on='building_id',how='left')
 del building_df
gc.collect()<load_from_csv>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
weather_df = pd.read_csv(DATA_PATH + 'weather_test.csv')
 weather_df = fill_weather_dataset(weather_df)
 weather_df = reduce_mem_usage(weather_df )<merge>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
test_df = test_df.merge(weather_df,how='left',on=['timestamp','site_id'])
 del weather_df
gc.collect()<feature_engineering>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
test_df = features_engineering(test_df )<predict_on_test>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
<set_options><EOS> 
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )