Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<drop_column>
warnings.filterwarnings("ignore")
 sys.path.insert(0, '.. /input/semisupervised-imagenet-models/semi-supervised-ImageNet1K-models-master/')
test_merge= test_merge.drop(unique_test_cols, axis= 1 )<save_to_csv>
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
TEST = '.. /input/prostate-cancer-grade-assessment/test.csv'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
MODELS = [f'.. /input/mypandago/RNXT50_200_reg_nozoomin_0.pth']
sz = 128*4
img_size = 200
bs = 2
N = 12
nworkers = 2
<train_model>
df = pd.read_csv('.. /input/mypanda/train_5folds.csv')
 df = df[df.fold == 0]
df
clf = ensemble.RandomForestClassifier(n_estimators=1000, min_samples_split= 2, random_state= 42)
 clf.fit(train_merge, labels )<predict_on_test>
def _resnext(url, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 return model
class Model(nn.Module):
def __init__(self, arch='resnext50_32x4d', n=1, pre=True):
super().__init__()
m = _resnext(semi_supervised_model_urls[arch], Bottleneck, [3, 4, 6, 3], False,
progress=False,groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(AdaptiveConcatPool2d() ,Flatten() ,nn.Linear(2*nc,512),
Mish() ,nn.BatchNorm1d(512),nn.Dropout(0.5),nn.Linear(512,n))
def forward(self, x):
shape = x.shape
n = shape[1]
x = x.view(-1,shape[2],shape[3],shape[4])
 x = self.enc(x)
 shape = x.shape
x = x.view(-1,n,shape[1],shape[2],shape[3] ).permute(0,2,1,3,4 ).contiguous() \ 
.view(-1,shape[1],shape[2]*n,shape[3])
 x = self.head(x)
 return x
predictions_randfor = clf.predict_proba(test_merge)[:,1]<train_model>
models = []
for path in MODELS:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = Model()
new_state_dict = OrderedDict()
for k, v in state_dict.items() :
name = k[7:]
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
 model.float()
model.eval()
model.cuda()
models.append(model)
 del state_dict
xgbc = xgb.XGBClassifier(seed= 42)
 xgbc.fit(train_merge, labels)
 fig, ax = pyplot.subplots(figsize=(10, 15))
xgb.plot_importance(xgbc, ax=ax)
<not_enough_vertices>
def crop_white(image: np.ndarray)-> np.ndarray:
assert image.shape[2] == 3
assert image.dtype == np.uint8
ys, =(image.min(( 1, 2)) != 255 ).nonzero()
xs, =(image.min(0 ).min(1)!= 255 ).nonzero()
if len(xs)== 0 or len(ys)== 0:
return image
return image[ys.min() :ys.max() + 1, xs.min() :xs.max() + 1]
xgbc.get_fscore() 
<not_enough_vertices>
def tile(img):
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],
constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
img_ = []
for i in range(len(img)) :
img_tmp = img[i]
img_tmp = cv2.resize(img_tmp,(256,256))
img_tmp = cv2.resize(img_tmp,(img_size,img_size))
img_.append(img_tmp)
 img_ = np.stack(img_)
 return img_
mean = torch.tensor([1.0-0.90949707, 1.0-0.8188697, 1.0-0.87795304])
 std = torch.tensor([0.36357649, 0.49984502, 0.40477625])
 class PandaDataset(Dataset):
def __init__(self, path, test, is_train=True):
self.path = path
if is_train:
self.names = list(pd.read_csv(test ).image_id)
 else:
self.names = list(df.image_id)
 def __len__(self):
return len(self.names)
 def __getitem__(self, idx):
name = self.names[idx]
img = skimage.io.MultiImage(os.path.join(DATA,name+'.tiff')) [-2]
imgs = tile(img)
 tiles = torch.Tensor(1.0 - imgs/255.0)
 tiles =(tiles - mean)/std
return tiles.permute(0,3,1,2), name
def calc_roc_auc(y, predict_probs):

 fpr, tpr, thresholds = metrics.roc_curve(y, predict_probs)
 roc_auc = metrics.auc(fpr, tpr)
 return roc_auc<split>
TRAINDATA = '.. /input/prostate-cancer-grade-assessment/train_images'
TRAIN = '.. /input/prostate-cancer-grade-assessment/train.csv'
train_split, val_split, label_train_split, label_val_split = model_selection.train_test_split(train_merge,
labels, test_size = 0.33, random_state = 42, stratify= labels )<features_selection>
import numpy as np
import pandas as pd
import os
import scipy as sp
from functools import partial
from sklearn import metrics
from collections import Counter
import json
def select_features_by_importance_threshold(model, X_train, y_train, selection_model, X_test, y_test,
minimum = False):
if minimum:
thresholds= np.unique(model.feature_importances_[model.feature_importances_ > minimum])
 thresholds = np.insert(thresholds, 0, 0.)
else:
thresholds= np.unique(model.feature_importances_)
 print(thresholds)
 for thresh in thresholds:
	
selection = feature_selection.SelectFromModel(model, threshold=thresh, prefit=True)
 select_X_train = selection.transform(X_train)
 	
selection_model = selection_model
selection_model.fit(select_X_train, y_train)
 	
select_X_test = selection.transform(X_test)
 y_pred = selection_model.predict_proba(select_X_test)[:,1]
predictions = y_pred
auc = calc_roc_auc(y_test, predictions)
 print("Thresh=%.3f, n=%d, AUC: %.2f%%" %(thresh, select_X_train.shape[1], auc))<normalization>
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
ll = metrics.cohen_kappa_score(y, X_p, weights='quadratic')
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5, 4.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(-loss_partial(self.coef_['x']))
def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
elif pred >= coef[3] and pred < coef[4]:
X_p[i] = 4
else:
X_p[i] = 5
return X_p
def coefficients(self):
return self.coef_['x']
sfm = feature_selection.SelectFromModel(xgbc, threshold=0.023, prefit= True)
 sfm_train= sfm.transform(train_merge)
 n_features = sfm_train.shape[1]
print(n_features )<train_model>
optR = OptimizedRounder()
xgb_clf= xgb.XGBClassifier(seed= 42)
 xgb_clf.fit(sfm_train, labels )<predict_on_test>
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
ds = PandaDataset(DATA,TEST)
 dl = DataLoader(ds, batch_size=bs, num_workers=nworkers, shuffle=False)
 names,preds = [],[]
with torch.no_grad() :
for x,y in tqdm(dl):
x = x.cuda()
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),
x.transpose(-1,-2),x.transpose(-1,-2 ).flip(-1),
x.transpose(-1,-2 ).flip(-2),x.transpose(-1,-2 ).flip(-1,-2)],1)
 x = x.view(-1,N,3,img_size,img_size)
 p = [model(x)for model in models]
p = torch.stack(p,1)
 p = p.view(bs,8*len(models),-1 ).mean(1 ).cpu()
names.append(y)
 preds.append(p)
 names = np.concatenate(names)
 preds = torch.cat(preds ).numpy()
sfm_test = sfm.transform(test_merge)
 predictions_xgb = xgb_clf.predict_proba(sfm_test)[:,1]<train_model>
coefficients = [0.538597, 1.585673, 2.633171, 3.240498, 4.070715]
X_train= train_split
X_test= val_split
y_train= label_train_split
y_test= label_val_split
model= xgb.XGBClassifier(seed= 42)
 eval_set = [(X_train, y_train),(X_test, y_test)]
model.fit(X_train, y_train, eval_metric="auc", eval_set=eval_set, verbose=True )<compute_train_metric>
test_predictions = optR.predict(preds, coefficients )
results = model.evals_result()
print(results )<train_model>
test_predictions = test_predictions.reshape(1,-1 ).tolist()
eval_set = [(X_test, y_test)]
model.fit(X_train, y_train, eval_metric=["auc"], eval_set=eval_set, early_stopping_rounds=10)
 results = model.evals_result()
print(results )<compute_test_metric>
names = names.reshape(1,-1 ).tolist()
def report(results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
print("Model with rank: {0}".format(i))
print("Mean validation score: {0:.3f}(std: {1:.3f})".format(
 results['mean_test_score'][candidate],
results['std_test_score'][candidate]))
print("Parameters: {0}".format(results['params'][candidate]))
print("" )<define_search_space>
sub_df = pd.DataFrame({'image_id': names[0], 'isup_grade': test_predictions[0]})
 sub_df["isup_grade"] = sub_df["isup_grade"].astype(int )
<train_on_grid><EOS> 
sub_df.to_csv("submission.csv", index=False)
 sub_df.head()