Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<feature_engineering>
warnings.filterwarnings("ignore")
 %load_ext autoreload
%autoreload
data['trafficSource.adwordsClickInfo.adNetworkType']=data['trafficSource.adwordsClickInfo.adNetworkType'].fillna('No Network' )<count_values>
model_path='.'
path='.. /input/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
ORG_SIZE=96
bs=64
num_workers=None
sz=96
del data['trafficSource.adwordsClickInfo.gclId']
data['trafficSource.adwordsClickInfo.adNetworkType'].value_counts()
data['trafficSource.adwordsClickInfo.page']=data['trafficSource.adwordsClickInfo.page'].fillna('99' )<create_dataframe>
df_trn=pd.read_csv(train_lbl )
key_word=pd.DataFrame(data.groupby(['trafficSource.keyword'])['totals.transactionRevenue'].mean() ).reset_index()
key_word=key_word.sort_values(by='totals.transactionRevenue',ascending=False)
<feature_engineering>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=.1,
max_lighting=0.05, max_warp=0.)
data['trafficSource.referralPath']=data['trafficSource.referralPath'].fillna('No Reference Path' )<feature_engineering>
data = ImageDataBunch.from_csv(path,csv_labels=train_lbl,folder='train', ds_tfms=tfms, size=sz, suffix='.tif',test=test_folder,bs=bs);
stats=data.batch_stats()
data.normalize(stats )
def process_device_b(data1):
i = str(data1)
 if '_' in i:
return 'other'
elif '.' in i:
return 'other'
elif len(i)>25:
return 'other'
else:
return i
data['device.browser']=data['device.browser'].apply(process_device_b )<string_transform>
data.show_batch(rows=5, figsize=(12,9))
def key_word(key):
low_key=str(key ).lower()
if 'goo' in low_key or 'google' in low_key or 'googl' in low_key or 'stor' in low_key:
return 'google'
if 'merchandi' in low_key:
return 'merchandise'
if 'youtube' in low_key or 'yo' in low_key or 'yuotub' in low_key or 'yuo tub' in low_key or 'tub' in low_key or 'yutob' in low_key:
return 'youtube'
if bool(re.search(r'\d', low_key)) :
return 'other'
if 'tshirt' in low_key or 't shirt' in low_key or 't-shirt' in low_key:
return 'tshirt'
if 'shirts' in low_key or 'shirt' in low_key:
return 'shirts'
if 'lamp' in low_key or 'lamps' in low_key:
return 'lamps'
if 'apparel' in low_key:
return 'apparel'
if len(low_key)>28:
return 'other'
if 'note' in low_key:
return 'notebook'
if 'android' in low_key:
return 'android'
else:
return low_key
data['trafficSource.keyword']=data['trafficSource.keyword'].apply(key_word)
<feature_engineering>
from sklearn.metrics import roc_auc_score
def traffic_feature(source_value):
if 'google' in source_value:
return 'google'
elif 'yahoo' in source_value:
return 'yahoo'
elif 'facebook' in source_value:
return 'facebook'
elif 'youtube' in source_value:
return 'youtube'
elif 'pinterest' in source_value:
return 'pinterest'
elif 'baidu' in source_value:
return 'baidu'
elif 'reddit' in source_value:
return 'reddit'
elif 'wow' in source_value:
return 'wow'
elif len(source_value)>20:
return 'other'
elif 'bing' in source_value:
return 'bing'
else:
return source_value
data['trafficSource.source']=data['trafficSource.source'].apply(traffic_feature)
<groupby>
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
data.groupby(['geoNetwork.networkDomain','channelGrouping'])['channelGrouping'].count()<define_variables>
from torchvision.models import *
drop_cols = ['trafficSource.referralPath', 'trafficSource.adContent', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.adwordsClickInfo.page',
'trafficSource.adwordsClickInfo.adNetworkType','visitStartTime','device.isMobile']<feature_engineering>
learn = create_cnn(
 data,
densenet201,
path='.',
metrics=[auc_score],
ps=0.5
)
for i, t in data.dtypes.iteritems() :
if t == object:
data[i] = pd.factorize(data[i])[0]<prepare_x_and_y>
lr = 1e-04
train_x=train_df.iloc[:,:-1]
train_y=train_df.iloc[:,-1]
test_x=test_df.iloc[:,:-1]
test_y=test_df.iloc[:,-1] 
<init_hyperparams>
learn.fit_one_cycle(1,lr)
 learn.recorder.plot()
learn.recorder.plot_losses()
param = { 'objective':'regression',
'max_depth': -1,
'learning_rate':0.005,
"min_child_samples":40,
"boosting":"gbdt",
"feature_fraction":0.8,
"bagging_freq":1,
"bagging_fraction":0.8 ,
"bagging_seed": 3,
"metric": 'rmse',
"lambda_l1": 1,
'lambda_l2': 1,
"verbosity": -1}<feature_engineering>
learn.unfreeze()
learn.lr_find()
train_y = train_y.apply(lambda x: np.log(x)if x > 0 else x )<split>
learn.fit_one_cycle(10,slice(1e-4,1e-3))
folds = KFold(n_splits=5, shuffle=True, random_state=15)
 oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
start = time.time()
features = list(train_df.columns)
 feature_importance_df = pd.DataFrame()
for fold_,(trn_idx, val_idx)in enumerate(folds.split(train_x.values, train_y.values)) :
trn_data = lgb.Dataset(train_x.iloc[trn_idx], label=train_y.iloc[trn_idx])
 val_data = lgb.Dataset(train_x.iloc[val_idx], label=train_y.iloc[val_idx])
 num_round = 10000
clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=500 ,early_stopping_rounds = 500)
 oof[val_idx] = clf.predict(train_x.iloc[val_idx].values, num_iteration=clf.best_iteration)
 fold_importance_df = pd.Series()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = clf.feature_importance()
fold_importance_df["fold"] = fold_ + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 predictions += clf.predict(test_x.values, num_iteration=clf.best_iteration)/ folds.n_splits<compute_test_metric>
preds,y=learn.get_preds()
pred_score=auc_score(preds,y)
 pred_score
print("CV score: {:<8.5f}".format(mean_squared_error(oof, train_y)**0.5))<save_to_csv>
preds,y=learn.TTA()
pred_score_tta=auc_score(preds,y)
 pred_score_tta
submission = pd.DataFrame()
submission['fullVisitorId'] = test['fullVisitorId']
submission['PredictedLogRevenue'] = predictions
submit_file = submission[['fullVisitorId', 'PredictedLogRevenue']].groupby('fullVisitorId' ).sum().reset_index()
submit_file.to_csv('submit_group.csv',index=False)
<load_from_csv>
preds_test,y_test=learn.get_preds(ds_type=DatasetType.Test )
def load_df(csv_path, nrows=None):
USE_COLUMNS = [
'channelGrouping', 'date', 'device', 'fullVisitorId', 'geoNetwork',
'socialEngagementType', 'totals', 'trafficSource', 'visitId',
'visitNumber', 'visitStartTime', 'customDimensions'
]
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
df = pd.read_csv(csv_path,
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
nrows=nrows, usecols=USE_COLUMNS)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}.{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 return df<load_from_csv>
preds_test_tta,y_test_tta=learn.TTA(ds_type=DatasetType.Test )
%%time
train_df = load_df(".. /input/train_v2.csv")
 test_df = load_df(".. /input/test_v2.csv" )<drop_column>
sub=pd.read_csv(f'{path}/sample_submission.csv' ).set_index('id')
 sub.head()
train_df = train_df.drop(labels=['trafficSource.campaignCode'],axis=1)
 train_df = train_df.drop(labels=['customDimensions','totals.totalTransactionRevenue'],axis=1)
 test_df = test_df.drop(labels=['customDimensions','totals.transactionRevenue','totals.totalTransactionRevenue'],axis=1 )<data_type_conversions>
clean_fname=np.vectorize(lambda fname: str(fname ).split('/')[-1].split('.')[0])
 fname_cleaned=clean_fname(data.test_ds.items)
 fname_cleaned=fname_cleaned.astype(str )
train_df['date']= pd.to_datetime(train_df['date'],format='%Y%m%d')
 test_df['date']= pd.to_datetime(test_df['date'],format='%Y%m%d')
 train_df.head()<data_type_conversions>
sub.loc[fname_cleaned,'label']=to_np(preds_test[:,1])
 sub.to_csv(f'submission_{pred_score}.csv' )
<count_unique_values><EOS> 
sub.loc[fname_cleaned,'label']=to_np(preds_test_tta[:,1])
 sub.to_csv(f'submission_{pred_score_tta}.csv' )