Kaggle competition: Histopathologic Cancer Detection
<SOS> metric: AUC Kaggle data source: histopathologic-cancer-detection<prepare_x_and_y>
! pip install albumentations
! pip install pretrainedmodels
dtrain = xgb.DMatrix(x, y)
 dtest = xgb.DMatrix(X_test )<train_on_grid>
%matplotlib inline
train_on_gpu = True
cudnn.benchmark = True
xgb_params = {
'eta': 0.08,
'max_depth': 15,
'subsample': 0.7,
'colsample_bytree': 0.7,
'objective': 'reg:linear',
'eval_metric': 'rmse',
'silent': 1
}
cv_output = xgb.cv(xgb_params, dtrain, num_boost_round=1000, early_stopping_rounds=400, verbose_eval=50, show_stdv=False)
 cv_output[['train-rmse-mean', 'test-rmse-mean']].plot()<predict_on_test>
SEED = 323
base_dir = '.. /input/'
def seed_everything(seed=SEED):
random.seed(seed)
 os.environ['PYHTONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(SEED )
y_predict1 = abs(model.predict(dtest))<prepare_x_and_y>
labels = pd.read_csv(base_dir+'train_labels.csv' )
train=x
y_train=np.log1p(y)
 test=X_test
y=y_train<compute_train_metric>
labels.label.value_counts()
n_folds = 5
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train.values)
 rmse = np.sqrt(-cross_val_score(model, train.values,y_train.values, scoring="neg_mean_squared_error", cv=kf))
return(rmse)
 def eval_model(model, name):
start_time = time.time()
score = rmsle_cv(model)
 print("{} score: {:.4f}({:.4f}), execution time: {:.1f}".format(name, score.mean() , score.std() , time.time() -start_time))<choose_model_class>
tr, val = train_test_split(labels.label, stratify=labels.label, test_size=0.15, random_state=SEED )
mod_lasso = make_pipeline(RobustScaler() , Lasso(alpha=0.005, random_state=1))
eval_model(mod_lasso, "lasso")
 mod_enet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
eval_model(mod_enet, "enet")
 mod_cat = CatBoostRegressor(iterations=10000, learning_rate=0.01,
depth=5, eval_metric='RMSE',
colsample_bylevel=0.7, random_seed = 17, silent=True,
bagging_temperature = 0.2, early_stopping_rounds=200)
 eval_model(mod_cat, "cat")
 mod_gboost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=5, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state=5)
 eval_model(mod_gboost, "gboost")
 mod_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state=7, nthread=-1)
 eval_model(mod_xgb, "xgb")
 mod_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=8,
learning_rate=0.05, n_estimators=650,
max_bin=58, bagging_fraction=0.80,
bagging_freq=5, feature_fraction=0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf=7, min_sum_hessian_in_leaf=11)
 eval_model(mod_lgb, "lgb" )<train_model>
img_class_dict = {k:v for k, v in zip(labels.id, labels.label)}
class StackingAveragedModels(BaseEstimator, RegressorMixin, TransformerMixin):
def __init__(self, base_models, meta_model, n_folds=5):
self.base_models = base_models
self.meta_model = meta_model
self.n_folds = n_folds
def fit(self, X, y):
self.base_models_ = [list() for x in self.base_models]
self.meta_model_ = clone(self.meta_model)
 kfold = KFold(n_splits=self.n_folds, shuffle=True, random_state=156)
 out_of_fold_predictions = np.zeros(( X.shape[0], len(self.base_models)))
 for i, model in enumerate(self.base_models):
for train_index, holdout_index in kfold.split(X, y):
instance = clone(model)
 self.base_models_[i].append(instance)
 instance.fit(X[train_index], y[train_index])
 y_pred = instance.predict(X[holdout_index])
 out_of_fold_predictions[holdout_index, i] = y_pred
self.meta_model_.fit(out_of_fold_predictions, y)
 return self
def predict(self, X):
meta_features = np.column_stack([
np.column_stack([model.predict(X)for model in base_models] ).mean(axis=1)
 for base_models in self.base_models_ ])
 return self.meta_model_.predict(meta_features)
 mod_stacked = StackingAveragedModels(base_models =(mod_cat, mod_xgb, mod_gboost, mod_lgb), meta_model = mod_lasso)
 eval_model(mod_stacked, "stacked" )<predict_on_test>
class CancerDataset(Dataset):
def __init__(self, datafolder, datatype='train', transform = transforms.Compose([transforms.CenterCrop(64),transforms.ToTensor() ]), labels_dict={}):
self.datafolder = datafolder
self.datatype = datatype
self.image_files_list = [s for s in os.listdir(datafolder)]
self.transform = transform
self.labels_dict = labels_dict
if self.datatype == 'train':
self.labels = [labels_dict[i.split('.')[0]] for i in self.image_files_list]
else:
self.labels = [0 for _ in range(len(self.image_files_list)) ]
def __len__(self):
return len(self.image_files_list)
 def __getitem__(self, idx):
img_name = os.path.join(self.datafolder, self.image_files_list[idx])
 img = cv2.imread(img_name)
 img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
 image = self.transform(image=img)
 image = image['image']
img_name_short = self.image_files_list[idx].split('.')[0]
if self.datatype == 'train':
label = self.labels_dict[img_name_short]
else:
label = 0
return image, label
def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def predict(model):
model.fit(train.values,y_train.values)
 train_pred = model.predict(train.values)
 pred = np.expm1(model.predict(test.values))
print(rmsle(y_train, train_pred))
return(pred )<predict_on_test>
data_transforms = albumentations.Compose([
albumentations.RandomRotate90(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.Flip(p=0.5),
albumentations.OneOf([
albumentations.CLAHE(clip_limit=2), albumentations.IAASharpen() , albumentations.IAAEmboss() ,
albumentations.RandomBrightness() , albumentations.RandomContrast() ,
albumentations.JpegCompression() , albumentations.Blur() , albumentations.GaussNoise() ], p=0.5),
albumentations.HueSaturationValue(p=0.5),
albumentations.ShiftScaleRotate(shift_limit=0.15, scale_limit=0.15, rotate_limit=45, p=0.5),
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_test = albumentations.Compose([
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_tta0 = albumentations.Compose([
albumentations.RandomRotate90(p=0.5),
albumentations.Transpose(p=0.5),
albumentations.Flip(p=0.5),
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_tta1 = albumentations.Compose([
albumentations.RandomRotate90(p=1),
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_tta2 = albumentations.Compose([
albumentations.Transpose(p=1),
albumentations.Normalize() ,
AT.ToTensor()
])
 data_transforms_tta3 = albumentations.Compose([
albumentations.Flip(p=1),
albumentations.Normalize() ,
AT.ToTensor()
])
 dataset = CancerDataset(datafolder=base_dir+'train/', datatype='train', transform=data_transforms, labels_dict=img_class_dict)
 val_set = CancerDataset(datafolder=base_dir+'train/', datatype='train', transform=data_transforms_test, labels_dict=img_class_dict)
 test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_test)
 train_sampler = SubsetRandomSampler(list(tr.index))
valid_sampler = SubsetRandomSampler(list(val.index))
batch_size = 256
num_workers = 0
train_loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers)
 valid_loader = torch.utils.data.DataLoader(val_set, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers )
prediction = predict(mod_lasso)
 prediction = predict(mod_enet)
 prediction = predict(mod_xgb)
 prediction = predict(mod_gboost)
 prediction = predict(mod_lgb)
 prediction = predict(mod_stacked)
 prediction<save_to_csv>
model_conv = pretrainedmodels.se_resnext50_32x4d()
model_conv.avg_pool = nn.AdaptiveAvgPool2d(( 1, 1))
model_conv.last_linear = nn.Linear(in_features=2048, out_features=1, bias=True )
new_test['id']=new_test.index
new_test['revenue']=prediction
new_test[['id','revenue']].to_csv('submission_Dragon2.csv', index=False)
 new_test[['id','revenue']].head()<set_options>
model_conv.cuda()
criterion = nn.BCEWithLogitsLoss()
optimizer = optim.Adam(model_conv.parameters() , lr=0.00005)
 scheduler = StepLR(optimizer, 1, gamma=0.5 )
%matplotlib inline
warnings.filterwarnings("ignore")
<load_from_csv>
valid_loss_min = np.Inf
patience = 5
p = 0
stop = False
n_epochs = 6
for epoch in range(1, n_epochs+1):
print(time.ctime() , 'Epoch:', epoch)
 if stop:
print("Training stop.")
 break
train_loss = []
scheduler.step(epoch)
 train_auc = []
for tr_batch_i,(data, target)in enumerate(train_loader):
model_conv.train()
data, target = data.cuda() , target.cuda()
optimizer.zero_grad()
output = model_conv(data)
 loss = criterion(output[:,0], target.float())
 train_loss.append(loss.item())
 a = target.data.cpu().numpy()
b = output[:,0].detach().cpu().numpy()
train_auc.append(roc_auc_score(a, b))
loss.backward()
optimizer.step()
if(tr_batch_i+1)%100 == 0:
model_conv.eval()
val_loss = []
val_auc = []
for val_batch_i,(data, target)in enumerate(valid_loader):
data, target = data.cuda() , target.cuda()
output = model_conv(data)
 loss = criterion(output[:,0], target.float())
 val_loss.append(loss.item())
 a = target.data.cpu().numpy()
b = output[:,0].detach().cpu().numpy()
val_auc.append(roc_auc_score(a, b))
print('Epoch %d, batches:%d, train loss: %.4f, valid loss: %.4f.'%(epoch, tr_batch_i, np.mean(train_loss), np.mean(val_loss)) +
' train auc: %.4f, valid auc: %.4f'%(np.mean(train_auc),np.mean(val_auc)))
 train_loss = []
train_auc = []
valid_loss = np.mean(val_loss)
 if valid_loss <= valid_loss_min:
print('Validation loss decreased({:.6f} --> {:.6f} ).Saving model...'.format(
 valid_loss_min,
valid_loss))
torch.save(model_conv.state_dict() , 'model.pt')
 valid_loss_min = valid_loss
p = 0
else:
p += 1
if p > patience:
print('Early stop training')
 stop = True
break
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train.index = train['id']
test.index = test['id']
print(train.shape, test.shape)
 train.head()<feature_engineering>
torch.cuda.empty_cache()
train['release_date'] = pd.to_datetime(train['release_date'], dayfirst=True)
 train['release_year'] = train['release_date'].dt.year
train['release_month'] = train['release_date'].dt.month
train['release_day'] = train['release_date'].dt.day
train['release_dayofweek'] = train['release_date'].dt.dayofweek
train['release_quarter'] = train['release_date'].dt.quarter<groupby>
model_conv.eval()
train['release_year'].groupby(train['release_year'][train['release_year']>2010] ).count()<categorify>
saved_dict = torch.load('model.pt')
 model_conv.load_state_dict(saved_dict)
 preds = []
for batch_i,(data, target)in enumerate(test_loader):
data, target = data.cuda() , target.cuda()
output = model_conv(data ).detach()
pr = output[:,0].cpu().numpy()
for i in pr:
preds.append(i )
dict_columns = ['belongs_to_collection', 'genres', 'production_companies',
'production_countries', 'spoken_languages', 'Keywords', 'cast', 'crew']
def text_to_dict(df):
for column in dict_columns:
df[column] = df[column].apply(lambda x: {} if pd.isna(x)else eval(x))
return df<categorify>
test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})
 test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')
 sub = sub[['id', 'preds']]
sub.columns = ['id', 'label']
sub.head()
temp = copy.deepcopy(train)
 temp = text_to_dict(temp )<count_values>
sub.to_csv('sub.csv', index=False )
temp['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0 ).value_counts() [:10]<count_values>
NUM_TTA = 8
temp['genres'].apply(lambda x: x[0]['name'] if x != {} else 0 ).value_counts() [:10]<feature_engineering>
sigmoid = lambda x: scipy.special.expit(x )
runtime_mean = train['runtime'].mean()
budget_mean =train['budget'].mean()
def prepare(df):
df['runtime'][(df['runtime'].isnull())|(df['runtime'] == 0)] = runtime_mean
df['budget'][df['budget']==0] = budget_mean
df['hasHP']=0
df.loc[df['homepage'].isnull() == False, 'hasHP'] = 1
df['hasTL']=0
df.loc[df['tagline'].isnull() == False, 'hasTL'] = 1
df['hasCollection']=0
df.loc[df['belongs_to_collection'].isnull() == False, 'hasCollection'] = 1
df['isEng']=0
df.loc[df['original_language'] == 'en', 'isEng'] = 1
df['isReleased'] = 1
df.loc[ df['status'] != "Released" ,"isReleased"] = 0
df['overview_word_count'] = df['overview'].str.split().str.len()
df['tagline_word_count'] = df['tagline'].str.split().str.len()
df['title_word_count'] = df['title'].str.split().str.len()
df['original_title_word_count'] = df['original_title'].str.split().str.len()
df['release_date'] = pd.to_datetime(df['release_date'], dayfirst=True)
 df['release_year'] = df['release_date'].dt.year
df['release_month'] = df['release_date'].dt.month
df['release_day'] = df['release_date'].dt.day
df['release_year'][df['release_year']>2018] = df['release_year']-100
df['release_dayofweek'] = df['release_date'].dt.dayofweek
df['release_quarter'] = df['release_date'].dt.quarter
df['budget'] = np.log1p(df['budget'])
 df = pd.get_dummies(df, columns=['original_language'], prefix='original_language')
 df = text_to_dict(df)
 df['_collection_name'] = df['belongs_to_collection'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['_collection_name'].fillna('')))
 df['_collection_name'] = le.transform(df['_collection_name'].fillna('' ).astype(str))
df['_genres_name'] = df['genres'].apply(lambda x: x[0]['name'] if x != {} else 0)
 le = LabelEncoder()
le.fit(list(df['_genres_name'].fillna('')))
 df['_genres_name'] = le.transform(df['_genres_name'].fillna('' ).astype(str))
df['_num_Keywords'] = df['Keywords'].apply(lambda x: len(x)if x != {} else 0)
 df['_num_cast'] = df['cast'].apply(lambda x: len(x)if x != {} else 0)
 df['production_countries_count'] = df['production_countries'].apply(lambda x : len(x))
df['production_companies_count'] = df['production_companies'].apply(lambda x : len(x))
df['cast_count'] = df['cast'].apply(lambda x : len(x))
df['crew_count'] = df['crew'].apply(lambda x : len(x))
df = df.drop(['imdb_id','poster_path','status','homepage','tagline','belongs_to_collection',\
'genres', 'production_companies','production_countries', 'spoken_languages',\
'Keywords', 'cast', 'crew','overview','title','original_title','release_date'],axis=1)
 return df<concatenate>
for num_tta in range(NUM_TTA):
if num_tta==0:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_test)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta==1:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta1)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta==2:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta2)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 elif num_tta==3:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta3)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 else:
test_set = CancerDataset(datafolder=base_dir+'test/', datatype='test', transform=data_transforms_tta0)
 test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, num_workers=num_workers)
 preds = []
for batch_i,(data, target)in enumerate(test_loader):
data, target = data.cuda() , target.cuda()
output = model_conv(data ).detach()
pr = output[:,0].cpu().numpy()
for i in pr:
preds.append(sigmoid(i)/NUM_TTA)
 if num_tta==0:
test_preds = pd.DataFrame({'imgs': test_set.image_files_list, 'preds': preds})
 test_preds['imgs'] = test_preds['imgs'].apply(lambda x: x.split('.')[0])
 else:
test_preds['preds']+=np.array(preds)
 sub = pd.read_csv('.. /input/sample_submission.csv')
 sub = pd.merge(sub, test_preds, left_on='id', right_on='imgs')
 sub = sub[['id', 'preds']]
sub.columns = ['id', 'label']
sub.to_csv('sub_tta.csv', index=False )
<define_variables><EOS> 
print("Finished" )