Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<drop_column>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
new_df1 = new_df1.drop(labels=['date','device.isMobile'], axis=1)
 test_df1 = test_df1.drop(labels=['date','device.isMobile'], axis=1 )<drop_column>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
new_df1 = new_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId',
'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot',
'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],
axis=1)
 test_df1 = test_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId',
'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot',
'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],
axis=1)
 print(new_df1.shape)
 new_df1.head()<train_model>
df_trn=pd.read_csv(train_lbl )
new_df1.fillna('0',inplace=True)
 test_df1.fillna('0',inplace=True )<count_missing_values>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)
new_df1.isna().sum()<data_type_conversions>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
new_df1['trafficSource.adContent'].fillna('NoAdContent',inplace=True)
 new_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 test_df1['trafficSource.adContent'].fillna('NoAdContent',inplace=True)
 test_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 for columns in ['totals.sessionQualityDim','totals.timeOnSite',
'totals.transactions']:
new_df1[columns].fillna('0',inplace=True)
 new_df1[columns] = new_df1[columns].astype('int')
 test_df1[columns].fillna('0',inplace=True)
 test_df1[columns] = test_df1[columns].astype('int' )<categorify>
data.show_batch(rows=5, figsize=(12,9))
def convert_category_todummies(df,field):
dummy_df = pd.get_dummies(df[field])
 df = pd.concat([df,dummy_df],axis=1)
 df.drop(labels=[field],axis=1,inplace=True)
 return df<categorify>
from sklearn.metrics import roc_auc_score
def convert_category_tolevel(df,field):
df[field],index = pd.factorize(df[field])
 return df<drop_column>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
merged_df['totals.transactionRevenue'] = np.log1p(merged_df['totals.transactionRevenue'])
 new_df1 = merged_df.iloc[:train_size]
test_df1 = merged_df[train_size:]
test_df1 = test_df1.drop(labels=['totals.transactionRevenue'],axis=1)
 print(new_df1.shape, test_df1.shape )<drop_column>
from torchvision.models import *
del train_df, new_df, test_df,merged_df, transaction_df, nontransaction_df, plot_df
gc.collect()<groupby>
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.5
)
train_x = new_df1.groupby(by=['fullVisitorId'] ).mean()
del new_df1
gc.collect()<prepare_x_and_y>
lr = 1e-04
scaler1 = MinMaxScaler()
train_y = train_x['totals.transactionRevenue']
train_x = train_x.drop(labels=['totals.transactionRevenue'],axis=1)
 scaled_x = scaler1.fit_transform(train_x.values)
 train_x = pd.DataFrame(scaled_x, columns=train_x.columns )<split>
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()
X_train, X_val, y_train, y_val = train_test_split(train_x, train_y, test_size=0.15, random_state=1,)
 lgb_train_data = lgb.Dataset(X_train, label=y_train)
 lgb_val_data = lgb.Dataset(X_val, label=y_val)
 params = {
"objective" : "regression",
"metric" : "rmse",
"max_depth" : 10,
"boosting" : "gbdt",
"learning_rate" : 0.0025,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 6,
"bagging_seed" : 42,
"seed": 42}
lgbModel = lgb.train(params, lgb_train_data,
num_boost_round=5000,
valid_sets=[lgb_train_data, lgb_val_data],
early_stopping_rounds=100,
verbose_eval=500 )<normalization>
learn.unfreeze()
learn.lr_find()
test_x = test_df1.groupby(by=['fullVisitorId'] ).mean()
visitorid = test_x.index
scaled_test = scaler1.transform(test_x)
 test_x = pd.DataFrame(scaled_test,columns=test_x.columns )<predict_on_test>
learn.fit_one_cycle(10,slice(1e-4,1e-3))
y_pred_train = lgbModel.predict(X_train, num_iteration=model.best_iteration)
 y_pred_val = lgbModel.predict(X_val, num_iteration=model.best_iteration)
 y_pred_submit = lgbModel.predict(test_x, num_iteration=model.best_iteration)
 print(f"LGBM: RMSE val: {np.sqrt(mean_squared_error(y_val, y_pred_val)) } - RMSE train: {np.sqrt(mean_squared_error(y_train, y_pred_train)) }" )<train_model>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
<train_model>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
<save_to_csv>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
submission = pd.DataFrame({'fullVisitorId':visitorid,'PredictedLogRevenue':np.squeeze(y_pred_submit)})
 submission['fullVisitorId']= submission['fullVisitorId'].astype(str)
 submission['PredictedLogRevenue']=submission['PredictedLogRevenue'].apply(lambda x: 0 if x<0 else x)
 submission.to_csv('submission.csv',index=False )<load_from_csv>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
file_train = ".. /input/train_v2.csv"
file_test = ".. /input/test_v2.csv"
chunk_size = 10000
def load_data(file,chunk_size,nrows_load=None,test_data=False):
df_res = pd.DataFrame()
df_reader = pd.read_csv(file,
dtype={ 'date': str, 'fullVisitorId': str},
chunksize=10000)
 for cidx, df in enumerate(df_reader):
df.reset_index(drop=True, inplace=True)
 process_df(df,test_data)
 df_res = pd.concat([df_res,df ], axis=0 ).reset_index(drop=True)
 del df
gc.collect()
if cidx % 20 == 0:
print('{}: rows loaded: {}'.format(cidx, df_res.shape[0]))
if nrows_load:
if res.shape[0] >= nrows_load:
break
return df_res<data_type_conversions>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
def parse_json(x,s):
res = json.loads(x)
 try:
return res[s]
except:
return float('NaN')
 def process_df(df,test_data):
df['days'] = df['date'].str[-2:]
df['days'] = df['days'].astype(int)
 df['month'] = df['date'].str[-4:-2]
df['month'] = df['month'].astype(int)
 df['year'] = df['date'].str[:4]
df['year'] = df['year'].astype(int)
 process_dict = {
'totals':['transactionRevenue','newVisits','pageviews','hits'] ,
'trafficSource':['campaign','source','medium'] ,
'device':['browser'],
'geoNetwork': ['country','city','continent','region','subContinent']
}
for c,l in process_dict.items() :
for it in l:
df[it] = df[c].apply(lambda x : parse_json(x,it))
cols = ['country','campaign','source','medium','continent','city','region','socialEngagementType','browser'
,'channelGrouping','subContinent','date']
labelencoder_X=LabelEncoder()
for c in cols:
df.loc[:,c] = labelencoder_X.fit_transform(df.loc[:,c])
 df['transactionRevenue'].fillna(0,inplace=True)
 df['newVisits'].fillna(0,inplace=True)
 df['pageviews'].fillna(0,inplace=True)
 df['transactionRevenue'] = df['transactionRevenue'].astype('float32')
 df['newVisits']= df['newVisits'].astype('uint16')
 df['pageviews'] = df['pageviews'].astype('uint16')
 df['hits'] = df['hits'].astype('uint32')
 rm_col = ['subContinent',
'channelGrouping','date','continent','customDimensions','fullVisitorId']
if test_data:
rm_col = rm_col[:-1]
df.drop(list(process_dict.keys())+ rm_col, axis=1,inplace=True)
 df = load_data(file_train,chunk_size)
 df_test =load_data(file_test,chunk_size,test_data=True )<groupby>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
per = sum(df['transactionRevenue'] > 0)/ len(df['transactionRevenue'])
 print('Percentage of transactions greater than 0 : {} %'.format(per*100))<prepare_x_and_y>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
<train_model><EOS> 
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )