Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<train_model>
import numpy as np
import pandas as pd
from fastai import *
from fastai.vision import *
from torchvision.models import *
import os
def fit_train(x,y,X_val,y_val,layer_size=64,mod = 'nn'):
if mod =='nn':
model = create_model_nn(x.shape[1],layer_size)
 history = model.fit(x, y, epochs=3, batch_size=128,validation_data=(X_val,y_val),verbose=1)
 print('Mean RMSE : ',np.sqrt(history.history['val_loss'] ).mean())
 elif mod =='xgboost':
model = create_model_xgboost(x,y,X_val,y_val)
 elif mod =='lgbm':
model = create_model_lgbm(x,y,X_val,y_val)
 else:
raise Exception('invalid model')
 return model
def calc_rmse(pred,y):
diff = pred - y
RMSE =(( diff ** 2 ).mean())**.5
print('RMSE : ',RMSE)
 def eval_set(model,x,y,mod='nn'):
if mod == 'nn':
p = model.predict(x,batch_size=64,verbose=1)
 pred = p[:,0]
elif mod == 'xgboost':
dx = xgboost.DMatrix(x)
 pred = model.predict(dx,ntree_limit=model.best_ntree_limit)
 elif mod == 'lgbm':
pred = model.predict(x, num_iteration=model.best_iteration)
 else:
raise Exception('invalid model')
 calc_rmse(pred,y )<train_model>
df = pd.read_csv('.. /input/train_labels.csv')
 df.head()
print('Training on training/val Dataset')
 model_training = fit_train(X_train,y_train,X_val,y_val)
 print('-'*40)
 print('Training on Sample Dataset')
 model_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500)
 print('-'*40)
 print('Training on Full training Dataset')
 model_full = create_model_nn(X.shape[1])
 model_full.fit(X, Y, epochs=3, batch_size=128,verbose=1)
 print('-'*40 )<train_model>
SZ = 96
BS = 64
NUM_WORKERS = 4
print('Training on training/val Dataset')
 model_training = fit_train(X_train,y_train,X_val,y_val,mod='xgboost')
 print('-'*40)
 print('Training on Sample Dataset')
 model_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500,mod='xgboost')
 print('-'*40)
 print('Training on Full training Dataset')
 model_full = create_model_xgboost(X, Y)
 print('Full Training set -- Using Sample training model')
 eval_set(model_sample,X_train,y_train,mod='xgboost' )<train_model>
data =(
 ImageItemList 
.from_csv(path, 'train_labels.csv', folder='train', suffix='.tif')
.random_split_by_pct() 
.label_from_df() 
.add_test(ImageItemList.from_folder('.. /input/test')) 
.transform(get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,max_lighting=0.05, max_warp=0.) , size=SZ, resize_method=ResizeMethod.SQUISH)
.databunch(bs=BS, num_workers=NUM_WORKERS, path = '.')
.normalize(imagenet_stats)
 )
print('Training on training/val Dataset')
 model_training = fit_train(X_train,y_train,X_val,y_val,mod='lgbm')
 print('-'*40)
 print('Training on Sample Dataset')
 model_sample = fit_train(X_train_sample,y_train_sample,X_val_sample,y_val_sample,layer_size=500,mod='lgbm')
 print('-'*40)
 print('Training on Full training Dataset')
 model_full = create_model_lgbm(X, Y)
 print('Full Training set -- Using Sample training model')
 eval_set(model_sample,X_train,y_train,mod='lgbm')
 print('-'*40 )<save_to_csv>
data.show_batch(3 )
df_test['predictions'] = model_full.predict(df_test.loc[:,df_test.columns[1:]],num_iteration=model_full.best_iteration)
 df_test.loc[df_test['predictions']< 0 ,'predictions']= 0
sub_df = pd.DataFrame({'fullVisitorId':df_test['fullVisitorId'] , 'PredictedLogRevenue': np.expm1(df_test['predictions'])})
 sub_df = sub_df.groupby("fullVisitorId")["PredictedLogRevenue"].sum().reset_index()
sub_df.columns = ["fullVisitorId", "PredictedLogRevenue"]
sub_df["PredictedLogRevenue"] = np.log1p(sub_df["PredictedLogRevenue"])
 sub_df.to_csv("baseline_lgb_submission.csv", index=False )<set_options>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
data_root = '.. /input/make-data-ready'
print(os.listdir(data_root))
pd.set_option('display.max_rows',200)
<categorify>
learn = create_cnn(data, densenet161, metrics=[auc_score], ps=0.5 )
def load_data(data='train',n=2):
df = pd.DataFrame()
for i in range(n):
if data=='train':
if i > 8 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'train_{i}.pkl'))
elif data=='test':
if i > 2 :
break
dfpart = pd.read_pickle(pjoin(data_root,f'test_{i}.pkl'))
df = pd.concat([df,dfpart])
 del dfpart
return df 
<load_from_csv>
learn.fit_one_cycle(1, 1e-3 )
df_train = load_data(n=9)
 df_test = load_data('test',n=4 )<concatenate>
learn.fit_one_cycle(4, slice(1e-6, 1e-4))
df = pd.concat([df_train, df_test] )<define_variables>
SZ = 96*2
BS = 64//2
NUM_WORKERS = 0
col_drop = ['Date_Year', 'Date_Month', 'Date_Week','Date_Hour','device_isMobile','device_deviceCategory',
'Date_Day', 'Date_Dayofweek', 'Date_Dayofyear', 'Date_Is_month_end',
'Date_Is_month_start', 'Date_Is_quarter_end', 'Date_Is_quarter_start',
'Date_Is_year_end', 'Date_Is_year_start','totals_visits',
'date','visitId','totals_totalTransactionRevenue','geoNetwork_city','geoNetwork_continent',
'geoNetwork_metro','geoNetwork_networkDomain',
'geoNetwork_region','geoNetwork_subContinent','trafficSource_adContent',
'trafficSource_adwordsClickInfo.adNetworkType','trafficSource_adwordsClickInfo.gclId',
'trafficSource_adwordsClickInfo.slot','trafficSource_campaign',
'trafficSource_keyword','trafficSource_referralPath','trafficSource_medium',
'customDimensions_value','customDimensions_index','trafficSource_source',
'totals_bounces','visitNumber','totals_newVisits']
df.drop(col_drop, axis=1, inplace=True )<feature_engineering>
data =(
 ImageItemList 
.from_csv(path, 'train_labels.csv', folder='train', suffix='.tif')
.random_split_by_pct() 
.label_from_df() 
.add_test(ImageItemList.from_folder('.. /input/test')) 
.transform(get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,max_lighting=0.05, max_warp=0.) , size=SZ, resize_method=ResizeMethod.SQUISH)
.databunch(bs=BS, num_workers=NUM_WORKERS, path = '.')
.normalize(imagenet_stats)
 )
country_drop=df.groupby('geoNetwork_country')['totals_transactions'].sum() [df.groupby('geoNetwork_country')['totals_transactions'].sum().sort_values() <4].index.tolist()
df.loc[df[df.geoNetwork_country.isin(country_drop)].index,'geoNetwork_country'] = 'NaN'
df.loc[df[~df.device_browser.isin(['Edge', 'Internet Explorer', 'Firefox', 'Safari', 'Chrome'])].index,'device_browser'] = 'NaN'
df.loc[df[~df.device_operatingSystem.isin(['Android', 'iOS', 'Linux', 'Chrome OS', 'Windows', 'Macintosh'])].index,'device_operatingSystem'] = 'NaN' 
<categorify>
learn.data = data
col_lb = ['channelGrouping','device_browser','device_operatingSystem', 'geoNetwork_country',
'trafficSource_adwordsClickInfo.isVideoAd','trafficSource_isTrueDirect']
for col in col_lb:
lb = LabelEncoder()
df[col]=lb.fit_transform(df[col] )<define_variables>
learn.fit_one_cycle(1, 1e-3/2 )
to_median = ['channelGrouping','device_browser','device_operatingSystem','geoNetwork_country','trafficSource_adwordsClickInfo.isVideoAd','trafficSource_isTrueDirect','trafficSource_adwordsClickInfo.page']
to_sum =['totals_hits','totals_pageviews','totals_timeOnSite','totals_transactionRevenue', 'totals_transactions']
to_mean =['totals_hits','totals_pageviews','totals_sessionQualityDim']
to_std = ['totals_hits','totals_pageviews']
to_time = 'visitStartTime'<data_type_conversions>
learn.fit_one_cycle(1, slice(1e-6/2, 1e-4/2))
target_period = pd.date_range(start='2016-08-01',end='2018-12-01', freq='2MS')
 train_period = target_period.to_series().shift(periods=-210, freq='d',axis= 0)
 time_to = train_period[train_period.index>np.datetime64('2016-08-01')].astype('int')//10**9
time_end = target_period.to_series().shift(periods=-45, freq='d',axis= 0)[4:]<groupby>
preds = learn.get_preds(ds_type=DatasetType.Test )
user_x = df.iloc[df_train.shape[0]:,:]
test_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),
user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last'] ).add_suffix('_time' ).sub(time_to.values[-1] ).abs() ,
user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min() ).rename('time_diff'),
user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),
user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),
user_x.groupby('fullVisitorId')[to_std].std(ddof=0 ).add_suffix('_std')],axis=1 ).reset_index()
test_ID= test_x.fullVisitorId
test_x = test_x.drop(['fullVisitorId'], axis=1,errors='ignore')
 test_x = test_x.astype('int' )<prepare_x_and_y>
df_sub = pd.read_csv('.. /input/sample_submission.csv' ).set_index('id')
 df_sub.head()
i=4
user_x = df[(df.visitStartTime>=(time_to.index.astype('int')//10**9)[i])&(df.visitStartTime<(time_end.index.astype('int')//10**9)[i])]
user_y = df[(df.visitStartTime>=time_to.values[i])&(df.visitStartTime<time_to.values[i+1])]
train_x = pd.concat([user_x.groupby('fullVisitorId')[to_median].median().add_suffix('_median'),
user_x.groupby('fullVisitorId')['visitStartTime'].agg(['first','last'] ).add_suffix('_time' ).sub(time_to.values[i] ).abs() ,
user_x.groupby('fullVisitorId')['visitStartTime'].apply(lambda x: x.max() -x.min() ).rename('time_diff'),
user_x.groupby('fullVisitorId')[to_sum].sum().add_suffix('_sum'),
user_x.groupby('fullVisitorId')[to_mean].mean().add_suffix('_mean'),
user_x.groupby('fullVisitorId')[to_std].std(ddof=0 ).add_suffix('_std')],axis=1 ).reset_index()
merged=train_x.merge(user_y.groupby('fullVisitorId')['totals_transactionRevenue'].sum().reset_index() ,\
how='left', on='fullVisitorId')
 val_y = merged.totals_transactionRevenue
val_y.fillna(0, inplace=True)
 val_x = merged.drop(['fullVisitorId','totals_transactionRevenue'], axis=1,errors='ignore')
 val_x = val_x.astype('int' )<import_modules>
clean_fname = np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned = clean_fname(data.test_ds.items)
 fname_cleaned = fname_cleaned.astype(str )
import lightgbm as lgb
from xgboost import XGBRegressor
from catboost import CatBoostRegressor
from sklearn.metrics import mean_squared_error<init_hyperparams>
df_sub.loc[fname_cleaned,'label'] = preds[0][:,1].tolist()
<prepare_x_and_y><EOS> 
df_sub.to_csv('sub.csv' )