Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<feature_engineering>
import numpy as np
import pandas as pd
from fastai import *
from fastai.vision import *
from torchvision.models import *
import os
import path
test.loc[test['id'] == 3033,'budget'] = 250
test.loc[test['id'] == 3051,'budget'] = 50
test.loc[test['id'] == 3084,'budget'] = 337
test.loc[test['id'] == 3224,'budget'] = 4
test.loc[test['id'] == 3594,'budget'] = 25
test.loc[test['id'] == 3619,'budget'] = 500
test.loc[test['id'] == 3831,'budget'] = 3
test.loc[test['id'] == 3935,'budget'] = 500
test.loc[test['id'] == 4049,'budget'] = 995946
test.loc[test['id'] == 4424,'budget'] = 3
test.loc[test['id'] == 4460,'budget'] = 8
test.loc[test['id'] == 4555,'budget'] = 1200000
test.loc[test['id'] == 4624,'budget'] = 30
test.loc[test['id'] == 4645,'budget'] = 500
test.loc[test['id'] == 4709,'budget'] = 450
test.loc[test['id'] == 4839,'budget'] = 7
test.loc[test['id'] == 3125,'budget'] = 25
test.loc[test['id'] == 3142,'budget'] = 1
test.loc[test['id'] == 3201,'budget'] = 450
test.loc[test['id'] == 3222,'budget'] = 6
test.loc[test['id'] == 3545,'budget'] = 38
test.loc[test['id'] == 3670,'budget'] = 18
test.loc[test['id'] == 3792,'budget'] = 19
test.loc[test['id'] == 3881,'budget'] = 7
test.loc[test['id'] == 3969,'budget'] = 400
test.loc[test['id'] == 4196,'budget'] = 6
test.loc[test['id'] == 4221,'budget'] = 11
test.loc[test['id'] == 4222,'budget'] = 500
test.loc[test['id'] == 4285,'budget'] = 11
test.loc[test['id'] == 4319,'budget'] = 1
test.loc[test['id'] == 4639,'budget'] = 10
test.loc[test['id'] == 4719,'budget'] = 45
test.loc[test['id'] == 4822,'budget'] = 22
test.loc[test['id'] == 4829,'budget'] = 20
test.loc[test['id'] == 4969,'budget'] = 20
test.loc[test['id'] == 5021,'budget'] = 40
test.loc[test['id'] == 5035,'budget'] = 1
test.loc[test['id'] == 5063,'budget'] = 14
test.loc[test['id'] == 5119,'budget'] = 2
test.loc[test['id'] == 5214,'budget'] = 30
test.loc[test['id'] == 5221,'budget'] = 50
test.loc[test['id'] == 4903,'budget'] = 15
test.loc[test['id'] == 4983,'budget'] = 3
test.loc[test['id'] == 5102,'budget'] = 28
test.loc[test['id'] == 5217,'budget'] = 75
test.loc[test['id'] == 5224,'budget'] = 3
test.loc[test['id'] == 5469,'budget'] = 20
test.loc[test['id'] == 5840,'budget'] = 1
test.loc[test['id'] == 5960,'budget'] = 30
test.loc[test['id'] == 6506,'budget'] = 11
test.loc[test['id'] == 6553,'budget'] = 280
test.loc[test['id'] == 6561,'budget'] = 7
test.loc[test['id'] == 6582,'budget'] = 218
test.loc[test['id'] == 6638,'budget'] = 5
test.loc[test['id'] == 6749,'budget'] = 8
test.loc[test['id'] == 6759,'budget'] = 50
test.loc[test['id'] == 6856,'budget'] = 10
test.loc[test['id'] == 6858,'budget'] = 100
test.loc[test['id'] == 6876,'budget'] = 250
test.loc[test['id'] == 6972,'budget'] = 1
test.loc[test['id'] == 7079,'budget'] = 8000000
test.loc[test['id'] == 7150,'budget'] = 118
test.loc[test['id'] == 6506,'budget'] = 118
test.loc[test['id'] == 7225,'budget'] = 6
test.loc[test['id'] == 7231,'budget'] = 85
test.loc[test['id'] == 5222,'budget'] = 5
test.loc[test['id'] == 5322,'budget'] = 90
test.loc[test['id'] == 5350,'budget'] = 70
test.loc[test['id'] == 5378,'budget'] = 10
test.loc[test['id'] == 5545,'budget'] = 80
test.loc[test['id'] == 5810,'budget'] = 8
test.loc[test['id'] == 5926,'budget'] = 300
test.loc[test['id'] == 5927,'budget'] = 4
test.loc[test['id'] == 5986,'budget'] = 1
test.loc[test['id'] == 6053,'budget'] = 20
test.loc[test['id'] == 6104,'budget'] = 1
test.loc[test['id'] == 6130,'budget'] = 30
test.loc[test['id'] == 6301,'budget'] = 150
test.loc[test['id'] == 6276,'budget'] = 100
test.loc[test['id'] == 6473,'budget'] = 100
test.loc[test['id'] == 6842,'budget'] = 30<categorify>
imgs = os.listdir('.. /input/train')[:5]
arch = resnet50
bs = 64
sz = 48
path = Path('.. /input')
 open_image('.. /input/train/'+imgs[1] )
le = LabelEncoder()<categorify>
df = pd.read_csv(path/'train_labels.csv')
 df.head()
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else ast.literal_eval(x))
return df
dfx = text_to_dict(train)
 for col in dict_columns:
train[col]=dfx[col]<categorify>
tfms = get_transforms()
src =(ImageItemList.from_csv(path, 'train_labels.csv', folder = 'train', suffix = '.tif')
.random_split_by_pct() 
.label_from_df() 
.add_test_folder('test'))
dfx = text_to_dict(test)
 for col in dict_columns:
test[col]=dfx[col]<feature_engineering>
np.random.seed(50)
 data =(src.transform(tfms, size = sz)
.databunch().normalize(imagenet_stats))
def date(x):
x=str(x)
 year=x.split('/')[2]
if int(year)<19:
return x[:-2]+'20'+year
else:
return x[:-2]+'19'+year
train['release_date'] =train['release_date'].fillna('1/1/90' ).apply(lambda x: date(x))
test['release_date'] =test['release_date'].fillna('1/1/90' ).apply(lambda x: date(x))
train['release_date'] = train['release_date'].apply(lambda x: datetime.strptime(x,'%m/%d/%Y'))
test['release_date'] = test['release_date'].apply(lambda x: datetime.strptime(x,'%m/%d/%Y'))
train['release_day'] = train['release_date'].apply(lambda x:x.weekday())
 train['release_month'] = train['release_date'].apply(lambda x:x.month)
 train['release_year'] = train['release_date'].apply(lambda x:x.year)
 test['release_day'] = test['release_date'].apply(lambda x:x.weekday())
 test['release_month'] = test['release_date'].apply(lambda x:x.month)
 test['release_year'] = test['release_date'].apply(lambda x:x.year)
 day=train['release_day'].value_counts().sort_index()
sns.barplot(day.index,day)
 plt.gca().set_xticklabels(["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"],rotation='45')
 plt.ylabel('No of releases' )<feature_engineering>
data.show_batch(rows = 3, figsize=(11,12))
def data_preprocessing(df):
df['belongs_to_collection']= df['belongs_to_collection'].apply(lambda x: 1 if x != {} else 0)
 df['has_homepage'] = 1
df.loc[ pd.isnull(df['homepage']),'has_homepage'] = 0
df['num_genres'] = df['genres'].apply(lambda x: len(x)if x != {} else 0)
 df['budget'] = np.log1p(df['budget'])
 df['isTaglineNA'] = df['tagline'].apply(lambda x: 0 if pd.isnull(x)else 1)
 df['original_language'] = le.fit_transform(df['original_language'])
 df['isOriginalLanguageEng'] = 0
df.loc[ df['original_language'].astype(str)== "en" ,"isOriginalLanguageEng"] = 1
df['ismovie_released']=1
df.loc[(df['status']!='Released'),'ismovie_released']=0
df['no_spoken_languages']=df['spoken_languages'].apply(lambda x: len(x))
df['original_title_letter_count'] = df['original_title'].str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['crew'].apply(lambda x : len(x))
df['genders_0_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 0]))
df['genders_1_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 1]))
df['genders_2_crew'] = df['crew'].apply(lambda x: sum([1 for i in x if i['gender'] == 2]))
cols_to_normalize=['runtime','popularity','budget','_budget_runtime_ratio','_budget_year_ratio','_budget_popularity_ratio','_releaseYear_popularity_ratio',
'_releaseYear_popularity_ratio2','_num_Keywords','_num_cast','no_spoken_languages','original_title_letter_count','original_title_word_count',
'title_word_count','overview_word_count','tagline_word_count','production_countries_count','production_companies_count','cast_count','crew_count',
'genders_0_crew','genders_1_crew','genders_2_crew']
for col in cols_to_normalize:
if(col in df):
print(col)
 x_array=[]
x_array=np.array(df[col].fillna(0))
X_norm=normalize([x_array])[0]
df[col]=X_norm
df = df.drop(['homepage','tagline','poster_path','overview','genres',
'imdb_id','original_title','cast','Keywords','title',
'production_companies','production_countries',
'release_date','spoken_languages','crew','status'],axis=1)
 df.fillna(-1,inplace=True)
 return df<feature_engineering>
def auc_score(y_pred,y_true,tens=True):
score = roc_auc_score(y_true,torch.sigmoid(y_pred)[:,1])
 if tens:
score = tensor(score)
 return score
<feature_engineering>
learn = create_cnn(data, arch, metrics = [accuracy, auc_score], model_dir = '/tmp/models/' )
test['revenue'] = np.nan
all_data=data_preprocessing(( pd.concat([train,test])) ).reset_index(drop=True )<train_model>
lr = 1e-02
learn.fit_one_cycle(5, slice(lr))
train=all_data.loc[:train.shape[0]-1,:]
test=all_data.loc[train.shape[0]:,:]
print("Train shape",train.shape)
 print("Test shape",train.shape )<define_variables>
learn.save('model-1-rn50' )
features = list(train.columns)
 features = [i for i in features if i != 'id' and i != 'revenue']<compute_test_metric>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
def score(data, y):
validation_res = pd.DataFrame(
 {"id": data["id"].values,
"transactionrevenue": data["revenue"].values,
"predictedrevenue": np.expm1(y)})
 validation_res = validation_res.groupby("id")["transactionrevenue", "predictedrevenue"].sum().reset_index()
return np.sqrt(mean_squared_error(np.log1p(validation_res["transactionrevenue"].values),
np.log1p(validation_res["predictedrevenue"].values)) )<categorify>
learn.fit_one_cycle(5, slice(1e-05, lr/5))
class KFoldValidation() :
def __init__(self, data, n_splits=5):
unique_vis = np.array(sorted(data['id'].astype(str ).unique()))
folds = GroupKFold(n_splits)
 ids = np.arange(data.shape[0])
 self.fold_ids = []
for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):
self.fold_ids.append([
ids[data['id'].astype(str ).isin(unique_vis[trn_vis])],
ids[data['id'].astype(str ).isin(unique_vis[val_vis])]
])
 def validate(self, train, test, features, model, name="", prepare_stacking=False,
fit_params={"early_stopping_rounds": 500, "verbose": 100, "eval_metric": "rmse"}):
model.FI = pd.DataFrame(index=features)
 full_score = 0
if prepare_stacking:
test[name] = 0
train[name] = np.NaN
for fold_id,(trn, val)in enumerate(self.fold_ids):
devel = train[features].iloc[trn]
y_devel = np.log1p(train["revenue"].iloc[trn])
 valid = train[features].iloc[val]
y_valid = np.log1p(train["revenue"].iloc[val])
 print("Fold ", fold_id, ":")
 model.fit(devel, y_devel, eval_set=[(valid, y_valid)], **fit_params)
 if len(model.feature_importances_)== len(features):
model.FI['fold' + str(fold_id)] = model.feature_importances_ / model.feature_importances_.sum()
predictions = model.predict(valid)
 predictions[predictions < 0] = 0
print("Fold ", fold_id, " error: ", mean_squared_error(y_valid, predictions)**0.5)
 fold_score = score(train.iloc[val], predictions)
 full_score += fold_score / len(self.fold_ids)
 print("Fold ", fold_id, " score: ", fold_score)
 if prepare_stacking:
train[name].iloc[val] = predictions
test_predictions = model.predict(test[features])
 test_predictions[test_predictions < 0] = 0
test[name] += test_predictions / len(self.fold_ids)
 print("Final score: ", full_score)
 return full_score<choose_model_class>
learn.save('model-2-rn50' )
Kfolder = KFoldValidation(train )<choose_model_class>
data =(src.transform(tfms, size = 96)
.databunch().normalize(imagenet_stats))
learn.data = data
data.train_ds[0][0].shape
lgbmodel = lgb.LGBMRegressor(n_estimators=10000,
objective='regression',
metric='rmse',
max_depth = 5,
num_leaves=30,
min_child_samples=100,
learning_rate=0.01,
boosting = 'gbdt',
min_data_in_leaf= 10,
feature_fraction = 0.9,
bagging_freq = 1,
bagging_fraction = 0.9,
importance_type='gain',
lambda_l1 = 0.2,
bagging_seed=random_seed,
subsample=.8,
colsample_bytree=.9,
use_best_model=True )<train_model>
learn.freeze()
learn.lr_find()
learn.recorder.plot()
Kfolder.validate(train, test, features , lgbmodel, name="lgbfinal", prepare_stacking=True )<choose_model_class>
lr = 1e-03
learn.fit_one_cycle(5, slice(lr))
xgbmodel = xgb.XGBRegressor(max_depth=5,
learning_rate=0.01,
n_estimators=10000,
objective='reg:linear',
gamma=1.45,
seed=random_seed,
silent=True,
subsample=0.8,
colsample_bytree=0.7,
colsample_bylevel=0.5 )<compute_test_metric>
learn.save('model-big-1-rn50' )
Kfolder.validate(train, test, features, xgbmodel, name="xgbfinal", prepare_stacking=True )<choose_model_class>
learn.unfreeze()
learn.lr_find()
learn.recorder.plot()
catmodel = cat.CatBoostRegressor(iterations=10000,
learning_rate=0.01,
depth=5,
eval_metric='RMSE',
colsample_bylevel=0.8,
bagging_temperature = 0.2,
metric_period = None,
early_stopping_rounds=200,
random_seed=random_seed )<train_model>
learn.fit_one_cycle(5, slice(3e-06, lr/5))
Kfolder.validate(train, test, features , catmodel, name="catfinal", prepare_stacking=True,
fit_params={"use_best_model": True, "verbose": 100} )<compute_test_metric>
learn.save('model-big-2-rn50' )
train['Revenue_lgb'] = train["lgbfinal"]
print("RMSE model lgb :" ,score(train, train.Revenue_lgb),)
 train['Revenue_xgb'] = train["xgbfinal"]
print("RMSE model xgb :" ,score(train, train.Revenue_xgb))
train['Revenue_cat'] = train["catfinal"]
print("RMSE model cat :" ,score(train, train.Revenue_cat))
train['Revenue_Dragon1'] = 0.4 * train["lgbfinal"] + \
0.2 * train["xgbfinal"] + \
0.4 * train["catfinal"]
print("RMSE model Dragon1 :" ,score(train, train.Revenue_Dragon1))
train['Revenue_Dragon2'] = 0.35 * train["lgbfinal"] + \
0.3 * train["xgbfinal"] + \
0.35 * train["catfinal"]
print("RMSE model Dragon2 :" ,score(train, train.Revenue_Dragon2))<save_to_csv>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
test['revenue'] = np.expm1(test["lgbfinal"])
 test[['id','revenue']].to_csv('submission_lgb.csv', index=False)
 test[['id','revenue']].head()<save_to_csv>
losses,idxs = interp.top_losses()
test['revenue'] = np.expm1(test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_xgb.csv', index=False)
 test[['id','revenue']].head()<save_to_csv>
preds,y = learn.TTA()
acc = accuracy(preds, y)
 print('The validation accuracy is {} %.'.format(acc * 100))
test['revenue'] = np.expm1(test["catfinal"])
 test[['id','revenue']].to_csv('submission_cat.csv', index=False)
 test[['id','revenue']].head()<save_to_csv>
pred_score = auc_score(preds,y ).item()
print('The validation AUC is {}.'.format(pred_score))
test['revenue'] = np.expm1(0.4 * test["lgbfinal"]+ 0.4 * test["catfinal"] + 0.2 * test["xgbfinal"])
 test[['id','revenue']].to_csv('submission_Dragon1.csv', index=False)
 test[['id','revenue']].head()<save_to_csv>
df1 = pd.read_csv(path/'sample_submission.csv')
 id_list = list(df1.id)
test['revenue'] = np.expm1(( test["lgbfinal"] + test["catfinal"] + test["xgbfinal"])/3)
 test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 test[['id','revenue']].head()<load_from_csv>
preds,y = learn.TTA(ds_type=DatasetType.Test)
 pred_list = list(preds[:,1] )
Data=pd.read_csv('.. /input/digit-recognizer/train.csv' )<prepare_x_and_y>
pred_dict = dict(( key, value.item())for(key, value)in zip(learn.data.test_ds.items,pred_list))
pred_ordered = [pred_dict[Path('.. /input/test/' + id + '.tif')] for id in id_list]
<set_options><EOS> 
submissions = pd.DataFrame({'id':id_list,'label':pred_ordered})
 submissions.to_csv("submission_{}.csv".format(pred_score),index = False)