Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<count_missing_values>
np.random.seed(42)
 %matplotlib inline
new_df.isna().sum()<drop_column>
model_path='.'
path='/kaggle/input/histopathologic-cancer-detection/'
train_folder=f'{path}train'
test_folder=f'{path}test'
train_lbl=f'{path}train_labels.csv'
bs=64
num_workers=None
sz=96
new_df1 = new_df.drop(labels=['geoNetwork.region','geoNetwork.networkDomain','geoNetwork.metro','geoNetwork.continent','geoNetwork.country','visitId','visitStartTime'],axis=1)
 test_df1 = test_df.drop(labels=['geoNetwork.region','geoNetwork.networkDomain','geoNetwork.metro','geoNetwork.continent','geoNetwork.country','visitId','visitStartTime'],axis=1)
<drop_column>
print(torch.cuda.is_available())
 print(torch.backends.cudnn.enabled )
new_df1 = new_df1.drop(labels=['date','device.isMobile'], axis=1)
 test_df1 = test_df1.drop(labels=['date','device.isMobile'], axis=1 )<drop_column>
df_train = pd.read_csv(train_lbl)
 print(f'Number of labels {len(df_train)}' )
new_df1 = new_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],axis=1)
 test_df1 = test_df1.drop(labels=['trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId', 'trafficSource.adwordsClickInfo.page', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.medium', 'trafficSource.medium', 'trafficSource.referralPath','trafficSource.source'],axis=1)
 print(new_df1.shape)
 new_df1.head()<train_model>
df_train['label'].value_counts(normalize=True )
new_df1.fillna('0',inplace=True)
 test_df1.fillna('0',inplace=True )<count_missing_values>
tfms = get_transforms(do_flip=True, flip_vert=True, max_rotate=.0, max_zoom=1.1,max_lighting=0.05, max_warp=0.)
new_df1.isna().sum()<data_type_conversions>
data = ImageDataBunch.from_csv(path,folder='train',valid_pct=0.3,csv_labels=train_lbl,ds_tfms=tfms, size=90, suffix='.tif',test=test_folder,bs=64 )
new_df1['trafficSource.adContent'].fillna('Noadcontent',inplace=True)
 new_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 test_df1['trafficSource.adContent'].fillna('Noadcontent',inplace=True)
 test_df1['trafficSource.keyword'].fillna('NA',inplace=True)
 for columns in ['totals.sessionQualityDim','totals.timeOnSite','totals.transactions']:
new_df1[columns].fillna('0',inplace=True)
 new_df1[columns] = new_df1[columns].astype('int')
 test_df1[columns].fillna('0',inplace=True)
 test_df1[columns] = test_df1[columns].astype('int' )<categorify>
stats=data.batch_stats()
data.normalize(stats)
def convert_category_todummies(df,field):
dummy_df = pd.get_dummies(df[field])
 df = pd.concat([df,dummy_df],axis=1)
 df.drop(labels=[field],axis=1,inplace=True)
 return df<categorify>
data.show_batch(rows=3, figsize=(8,5))
def convert_category_tolevel(df,field):
df[field],index = pd.factorize(df[field])
 return df<prepare_x_and_y>
model_dir = "/kaggle/working/tmp/models/"
os.makedirs('/kaggle/working/tmp/models/' )
scaler1 = MinMaxScaler()
merged_df['totals.transactionRevenue'] = np.log1p(merged_df['totals.transactionRevenue'])
 new_df1 = merged_df.iloc[:train_size]
test_df1 = merged_df[train_size:]
test_df1 = test_df1.drop(labels=['totals.transactionRevenue'],axis=1)
 print(new_df1.shape, test_df1.shape )<drop_column>
dir(fastai.vision.models )
del train_df, new_df, test_df,merged_df, transaction_df, nontransaction_df, plot_df
gc.collect()<groupby>
learner_resnet50 = cnn_learner(data=data, base_arch=models.resnet50,model_dir=model_dir, metrics=[accuracy,error_rate], ps=0.5 )
train_x = new_df1.groupby(by=['fullVisitorId'] ).mean()
del new_df1
gc.collect() 
<normalization>
lr_find(learner_resnet50 )
train_y = train_x['totals.transactionRevenue']
train_x = train_x.drop(labels=['totals.transactionRevenue'],axis=1)
 scaled_x = scaler1.fit_transform(train_x.values)
 train_x = pd.DataFrame(scaled_x, columns=train_x.columns )<split>
defaults.device = torch.device('cuda' )
X_train, X_val, y_train, y_val = train_test_split(train_x, train_y, test_size=0.15, random_state=1)
 lgb_train_data = lgb.Dataset(X_train, label=y_train)
 lgb_val_data = lgb.Dataset(X_val, label=y_val)
 params = {
"objective" : "regression",
"metric" : "rmse",
"max_depth" : 10,
"boosting" : "gbdt",
"learning_rate" : 0.0025,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.7,
"bagging_frequency" : 6,
"bagging_seed" : 42,
"seed": 42}
model = lgb.train(params, lgb_train_data,
num_boost_round=5000,
valid_sets=[lgb_train_data, lgb_val_data],
early_stopping_rounds=100,
verbose_eval=500 )<normalization>
learner_resnet50.fit_one_cycle(1, 1e-02 )
test_x = test_df1.groupby(by=['fullVisitorId'] ).mean()
visitorid = test_x.index
scaled_test = scaler1.transform(test_x)
 test_x = pd.DataFrame(scaled_test,columns=test_x.columns )<predict_on_test>
learner_resnet50.recorder.plot_lr(show_moms=True )
y_pred_train = model.predict(X_train, num_iteration=model.best_iteration)
 y_pred_val = model.predict(X_val, num_iteration=model.best_iteration)
 y_pred_submit = model.predict(test_x, num_iteration=model.best_iteration)
 print(f"LGBM: RMSE val: {np.sqrt(mean_squared_error(y_val, y_pred_val)) } - RMSE train: {np.sqrt(mean_squared_error(y_train, y_pred_train)) }" )<data_type_conversions>
learner_resnet50.show_results(alpha=1 )
<compute_test_metric>
learner_resnet50.save('stage-1',return_path=True )
<train_model>
learner_resnet50.unfreeze()
inputs = Input(shape=(1,70))
x = Bidirectional(LSTM(200,recurrent_dropout=0.2, kernel_initializer='lecun_normal', return_sequences=True))(inputs)
 x = Bidirectional(LSTM(120,recurrent_dropout=0.2, kernel_initializer='lecun_normal'))(x)
 x = Dense(50, activation='sigmoid' )(x)
 x = Dropout(0.1 )(x)
 x = Dense(20,activation='elu' )(x)
 output = Dense(1,activation='linear' )(x)
 model2 = Model(inputs=inputs, outputs=output)
 model2.compile(loss='mse', optimizer='adam')
 model2.fit(X_train, y_train, epochs=4, batch_size=64, validation_data=(X_val, y_val), verbose=1, shuffle=False )<compute_test_metric>
lr_find(learner_resnet50)
 learner_resnet50.recorder.plot()
test_x = test_x.values
test_x = test_x.reshape(( test_x.shape[0],1,test_x.shape[1]))
y_pred_train = model2.predict(X_train)
 y_pred_val = model2.predict(X_val)
 y_pred_submit = model2.predict(test_x)
 print(f"LSTM: RMSE val: {np.sqrt(mean_squared_error(y_val, y_pred_val)) } - RMSE train: {np.sqrt(mean_squared_error(y_train, y_pred_train)) }" )<save_to_csv>
learner_resnet50.fit_one_cycle(1,slice(1e-06,1e-05),pct_start=0.8 )
submission = pd.DataFrame({'fullVisitorId':visitorid,'PredictedLogRevenue':np.squeeze(y_pred_submit)})
 submission['fullVisitorId']= submission['fullVisitorId'].astype(str)
 submission['PredictedLogRevenue']=submission['PredictedLogRevenue'].apply(lambda x: 0 if x<0 else x)
 submission.to_csv('submission2.csv',index=False )<set_options>
learner_resnet50.save('stage-2',return_path=True )
%matplotlib inline
py.init_notebook_mode(connected=True)
 gc.enable()
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input")) 
<set_options>
interp = ClassificationInterpretation.from_learner(learner_resnet50 )
json_data=["device","geoNetwork","totals","trafficSource"]
gc.enable()
features = ['channelGrouping', 'date', 'fullVisitorId', 'visitId',\
'visitNumber', 'visitStartTime', 'device_browser',\
'device_deviceCategory', 'device_isMobile', 'device_operatingSystem',\
'geoNetwork_city', 'geoNetwork_continent', 'geoNetwork_country',\
'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region',\
'geoNetwork_subContinent', 'totals_bounces', 'totals_hits',\
'totals_newVisits', 'totals_pageviews', 'totals_transactionRevenue',\
'trafficSource_adContent', 'trafficSource_campaign',\
'trafficSource_isTrueDirect', 'trafficSource_keyword',\
'trafficSource_medium', 'trafficSource_referralPath',\
'trafficSource_source']
def load_df(csv_path='.. /input/train_v2.csv'):
JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']
ans = pd.DataFrame()
dfs = pd.read_csv(csv_path, sep=',',
converters={column: json.loads for column in JSON_COLUMNS},
dtype={'fullVisitorId': 'str'},
chunksize = 100000)
 for df in dfs:
df.reset_index(drop = True,inplace = True)
 for column in JSON_COLUMNS:
column_as_df = json_normalize(df[column])
 column_as_df.columns = [f"{column}_{subcolumn}" for subcolumn in column_as_df.columns]
df = df.drop(column, axis=1 ).merge(column_as_df, right_index=True, left_index=True)
 print(f"Loaded {os.path.basename(csv_path)}.Shape: {df.shape}")
 use_df = df[features]
del df
gc.collect()
ans = pd.concat([ans, use_df], axis = 0 ).reset_index(drop = True)
 print(ans.shape)
 return ans
train = load_df()
train.shape<load_from_csv>
losses,idxs = interp.top_losses()
len(data.valid_ds)==len(losses)==len(idxs )
test=load_df(csv_path='.. /input/test_v2.csv' )<prepare_x_and_y>
pred_val ,y_val = learner_resnet50.get_preds()
def auc_score(y_pred,y_true,tens=True):
score=roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score=tensor(score)
 else:
score=score
return score
pred_score=auc_score(pred_val ,y_val)
 pred_score
train["totals_transactionRevenue"].fillna(0, inplace=True)
 train_y = train["totals_transactionRevenue"].values<data_type_conversions>
pred_score_acc=accuracy(pred_val ,y_val)
 pred_score_acc
num_cols = ["totals_hits", "totals_pageviews", "visitNumber", "visitStartTime", 'totals_bounces', 'totals_newVisits']
for col in num_cols:
train[col] = train[col].astype(float)
 test[col] = test[col].astype(float)
 excluded_features = [
'date', 'fullVisitorId', 'sessionId', 'totals.transactionRevenue',
'visitId', 'visitStartTime', 'nb_sessions', 'max_visits','visit_time','totals_visits'
]
categorical_features = [
_f for _f in train.columns
if(_f not in excluded_features)&(train[_f].dtype == 'object')
 ]
categorical_features<feature_engineering>
fpr, tpr, thresholds = roc_curve(y_val.numpy() , pred_val.numpy() [:,1], pos_label=1)
 pred_score_auc = auc(fpr, tpr)
 print(f'ROC area: {pred_score_auc}' )
train_val=np.log1p(train_y )<feature_engineering>
learner_resnet50.export('/kaggle/working/tmp/models/export.pkl' )
for f in categorical_features:
train[f], indexer = pd.factorize(train[f])
 test[f] = indexer.get_indexer(test[f])
 train.head()<prepare_x_and_y>
loaded_learner = load_learner(Path(model_dir))
loaded_learner.data.classes
train["totals_transactionRevenue"].fillna(0, inplace=True)
 train_y = train["totals_transactionRevenue"]
train_val=np.log1p(train_y )<split>
pred_class,pred_idx,pred_probs = loaded_learner.predict(img)
 print(pred_class, pred_idx,pred_probs )
train_x,test_x,val_x,val_y=train_test_split(train,train_val,test_size=0.2, random_state=42)
<split>
pred_class,pred_idx,pred_probs = loaded_learner.predict(img)
 print(pred_class, pred_idx,pred_probs )
trn_x=train_x[categorical_features+num_cols]
tst_x=test_x[categorical_features+num_cols] 
<train_model>
img = open_image(Path('.. /input/test-image/test_img.tif'))
pred_class,pred_idx,pred_probs = loaded_learner.predict(img)
 img.show()
targets = ['Non-Cancerous','Cancerous']
print("Tissue cell is identified as" , targets[pred_idx] , "with probability of", float(pred_probs[pred_idx]*100))
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 30,
"min_child_samples" : 100,
"learning_rate" : 0.1,
"bagging_fraction" : 0.7,
"feature_fraction" : 0.5,
"bagging_frequency" : 5,
"bagging_seed" : 2018,
"verbosity" : -1
}
lgtrain = lgb.Dataset(train_X, label=train_y)
 lgval = lgb.Dataset(val_X, label=val_y)
 model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)
 return pred_test_y, model, pred_val_y
pred_test, model, pred_val = run_lgb(trn_x, val_x,tst_x, val_y, test_X )<feature_engineering>
loaded_learner_val = load_learner(Path(model_dir),test=ImageList.from_folder(Path(test_folder)) )
pred_val[pred_val<0] = 0
val_pred_df = pd.DataFrame({"fullVisitorId":test_x["fullVisitorId"].values})
 val_pred_df["transactionRevenue"] = test_x["totals_transactionRevenue"].values
val_pred_df["PredictedRevenue"] = np.expm1(pred_val)
 val_pred_df = val_pred_df.groupby("fullVisitorId")["transactionRevenue", "PredictedRevenue"].sum().reset_index()
print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df["transactionRevenue"].values), np.log1p(val_pred_df["PredictedRevenue"].values))))<save_to_csv>
pred_test ,y_test = loaded_learner_val.get_preds(ds_type=DatasetType.Test )
submission = pd.DataFrame({"fullVisitorId":test["fullVisitorId"].values})
 pred_test[pred_test<0] = 0
submission["PredictedLogRevenue"] = np.expm1(pred_test)
 submission = submission.groupby("fullVisitorId")["PredictedLogRevenue"].sum().reset_index()
submission.columns = ["fullVisitorId", "PredictedLogRevenue"]
submission["PredictedLogRevenue"] = np.log1p(submission["PredictedLogRevenue"])
 submission.to_csv("predicted_values.csv", index=False )<import_modules>
sub=pd.read_csv('.. /input/histopathologic-cancer-detection/sample_submission.csv' ).set_index('id' )
import numpy as np
import pandas as pd
import os 
<load_from_csv>
clean_names = np.vectorize(lambda imgname: str(imgname ).split('/')[-1][:-4])
 cleaned_names = clean_names(data.test_ds.items ).astype(str )
train = pd.read_csv('/kaggle/input/liverpool-ion-switching/train.csv')
 test = pd.read_csv('/kaggle/input/liverpool-ion-switching/test.csv' )<set_options>
sub.loc[cleaned_names,'label']=pred_test.numpy() [:,1]
sub.to_csv(f'/kaggle/working/submission_{int(pred_score_auc*100)}auc.csv' )
<define_variables><EOS> 
predicted_prob_test = pd.read_csv('./submission_98auc.csv')
 predicted_prob_test.head(10 )