Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<set_options>
warnings.filterwarnings("ignore")
 sys.path.insert(0, '.. /input/semisupervised-imagenet-models/semi-supervised-ImageNet1K-models-master/')
%matplotlib inline<groupby>
DATA = '.. /input/prostate-cancer-grade-assessment/test_images'
TEST = '.. /input/prostate-cancer-grade-assessment/test.csv'
SAMPLE = '.. /input/prostate-cancer-grade-assessment/sample_submission.csv'
MODELS = [f'.. /input/panda-init-class-model1/RNXT50_128k_0_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128k_3_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128kr_1_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128kr_2_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128kr_2feature_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128kr_9_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128kr_7c_{i}.pth' for i in range(4)] + \
[f'.. /input/panda-init-class-model1/RNXT50_128kr_3_{i}.pth' for i in range(4)]
ws = [1.0,1.0,1.0,7.0,1.0,1.0,1.0,1.0]
sz = 128
bs = 1
N = 128
nworkers = 2
def LeaveOneOut(data1, data2, groupcolumns, columnName, useLOO=False, cut=1, addNoise=False):
features = list([])
 for a in groupcolumns:
features.append(a)
 if(columnName is not None):
features.append(columnName)
 grpCount = data1.groupby(features)['visitors'].count().reset_index().rename(columns={'visitors': 'Count'})
 if(useLOO):
grpCount = grpCount[(grpCount.Count > cut)]
grpMean = data1.groupby(features)['visitors'].mean().reset_index().rename(columns={'visitors': 'Mean'})
 grpMedian = data1.groupby(features)['visitors'].median().reset_index().rename(columns={'visitors': 'Median'})
 grpMin = data1.groupby(features)['visitors'].min().reset_index().rename(columns={'visitors': 'Min'})
 grpMax = data1.groupby(features)['visitors'].max().reset_index().rename(columns={'visitors': 'Max'})
 grpStd = data1.groupby(features)['visitors'].std().reset_index().rename(columns={'visitors': 'Std'})
 grpOutcomes = grpCount.merge(grpMean, on=features)
 grpOutcomes = grpOutcomes.merge(grpMedian, on=features)
 grpOutcomes = grpOutcomes.merge(grpMin, on=features)
 grpOutcomes = grpOutcomes.merge(grpMax, on=features)
 grpOutcomes = grpOutcomes.merge(grpStd, on=features)
 x = pd.merge(data2[features], grpOutcomes,
suffixes=('x_', ''),
how='left',
on=features,
left_index=True)[['Count','Mean','Median','Max','Min','Std']]
x['Outcomes'] = data2['visitors'].values
if(useLOO):
nonnulls =(~x.Count.isnull())
 x.loc[nonnulls,'Mean'] =(( x[nonnulls].Mean*x[nonnulls].Count)-x[nonnulls].Outcomes)
 x.loc[nonnulls,'Median'] =(( x[nonnulls].Median*x[nonnulls].Count)-x[nonnulls].Outcomes)
 if(addNoise is True):
x.loc[nonnulls&(x.Std>0),'Mean'] += np.random.normal(0,x[nonnulls&(x.Std>0)].Std,x[nonnulls&(x.Std>0)].shape[0])
 x.loc[nonnulls&(x.Std>0),'Median'] += np.random.normal(0,x[nonnulls&(x.Std>0)].Std,x[nonnulls&(x.Std>0)].shape[0])
 else:
x.loc[nonnulls,'Count'] -= 1
x.loc[nonnulls,'Mean'] /=(x[nonnulls].Count)
 x.loc[nonnulls,'Median'] /=(x[nonnulls].Count)
 x.Count = np.log1p(x.Count)
 x = x.replace(np.inf, np.nan)
 x = x.replace(-np.inf, np.nan)
 return x[['Count','Mean','Median','Max','Min', 'Std']]
def MungeTrain() :
air_visit_data = pd.read_csv('.. /input/air_visit_data.csv',parse_dates=['visit_date'])
 air_store_info = pd.read_csv('.. /input/air_store_info.csv')
 hpg_store_info = pd.read_csv('.. /input/hpg_store_info.csv')
 hpg_store_info.drop(['latitude','longitude'],inplace=True,axis=1)
 store_id_relation = pd.read_csv('.. /input/store_id_relation.csv')
 storeinfo = air_store_info.merge(store_id_relation,on='air_store_id',how='left')
 storeinfo = storeinfo.merge(hpg_store_info,on='hpg_store_id',how='left')
 air_reserve = pd.read_csv('.. /input/air_reserve.csv',parse_dates=['visit_datetime'])
 air_reserve['visit_date'] = air_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
hpg_reserve =pd.read_csv('.. /input/hpg_reserve.csv',parse_dates=['visit_datetime'])
 hpg_reserve['visit_date'] = hpg_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
date_info = pd.read_csv('.. /input/date_info.csv',parse_dates=['calendar_date'] ).rename(columns={'calendar_date':'visit_date'})
 air_reserve_by_date = air_reserve.groupby(['air_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 hpg_reserve_by_date = hpg_reserve.groupby(['hpg_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 train = air_visit_data.merge(storeinfo,on='air_store_id',how='left')
 train = train.merge(air_reserve_by_date,on=['air_store_id','visit_date'],how='left')
 train = train.merge(hpg_reserve_by_date,on=['hpg_store_id','visit_date'],how='left')
 train = train.merge(date_info,on='visit_date',how='left')
 train['year'] = train.visit_date.dt.year
train['month'] = train.visit_date.dt.month
train.reserve_visitors_x = train.reserve_visitors_x.fillna(0)
 train.reserve_visitors_y = train.reserve_visitors_y.fillna(0)
 train.reserve_visitors_x = np.log1p(train.reserve_visitors_x)
 train.reserve_visitors_y = np.log1p(train.reserve_visitors_y)
 train.visitors = np.log1p(train.visitors)
 train = train.fillna(-1)
 train = train.sort_values(by=['visit_date','air_store_id'],ascending=False)
 train = train.reset_index(drop=True)
 return train
def MungeTest(columns):
air_visit_data = pd.read_csv('.. /input/sample_submission.csv')
 air_visit_data['visit_date'] = air_visit_data.id.apply(lambda x: datetime.datetime(year=int(x[-10:-6]), month=int(x[-5:-3]), day=int(x[-2:])))
 air_visit_data['air_store_id'] = air_visit_data.id.apply(lambda x: x[:-11])
 air_store_info = pd.read_csv('.. /input/air_store_info.csv')
 hpg_store_info = pd.read_csv('.. /input/hpg_store_info.csv')
 hpg_store_info.drop(['latitude','longitude'],inplace=True,axis=1)
 store_id_relation = pd.read_csv('.. /input/store_id_relation.csv')
 storeinfo = air_store_info.merge(store_id_relation,on='air_store_id',how='left')
 storeinfo = storeinfo.merge(hpg_store_info,on='hpg_store_id',how='left')
 air_reserve = pd.read_csv('.. /input/air_reserve.csv',parse_dates=['visit_datetime'])
 air_reserve['visit_date'] = air_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
hpg_reserve =pd.read_csv('.. /input/hpg_reserve.csv',parse_dates=['visit_datetime'])
 hpg_reserve['visit_date'] = hpg_reserve.visit_datetime.apply(lambda df :
datetime.datetime(year=df.year, month=df.month, day=df.day))
date_info = pd.read_csv('.. /input/date_info.csv',parse_dates=['calendar_date'] ).rename(columns={'calendar_date':'visit_date'})
 air_reserve_by_date = air_reserve.groupby(['air_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 hpg_reserve_by_date = hpg_reserve.groupby(['hpg_store_id','visit_date'] ).reserve_visitors.sum().reset_index(drop=False)
 test = air_visit_data.merge(storeinfo,on='air_store_id',how='left')
 test = test.merge(air_reserve_by_date,on=['air_store_id','visit_date'],how='left')
 test = test.merge(hpg_reserve_by_date,on=['hpg_store_id','visit_date'],how='left')
 test = test.merge(date_info,on='visit_date',how='left')
 test['year'] = test.visit_date.dt.year
test['month'] = test.visit_date.dt.month
test.reserve_visitors_x = test.reserve_visitors_x.fillna(0)
 test.reserve_visitors_y = test.reserve_visitors_y.fillna(0)
 test.reserve_visitors_x = np.log1p(test.reserve_visitors_x)
 test.reserve_visitors_y = np.log1p(test.reserve_visitors_y)
 test.visitors = np.log1p(test.visitors)
 test = test.fillna(-1)
 test = test.sort_values(by=['visit_date','air_store_id'],ascending=False)
 test = test.reset_index(drop=True)
 return test[list(['id'])+list(columns)]<remove_duplicates>
def _resnext(url, block, layers, pretrained, progress, **kwargs):
model = ResNet(block, layers, **kwargs)
 return model
class Model(nn.Module):
def __init__(self, arch='resnext50_32x4d', n=6, pre=True):
super().__init__()
m = _resnext(semi_supervised_model_urls[arch], Bottleneck, [3, 4, 6, 3], False, progress=False,\
groups=32,width_per_group=4)
 self.enc = nn.Sequential(*list(m.children())[:-2])
 nc = list(m.children())[-1].in_features
self.head = nn.Sequential(AdaptiveConcatPool2d() ,Flatten() ,
nn.Linear(2*nc,512),Mish() ,nn.GroupNorm(32,512),
nn.Dropout(0.5),nn.Linear(512,n))
def forward(self, x):
shape = x.shape
n = shape[1]
x = x.view(-1,shape[2],shape[3],shape[4])
 x = self.enc(x)
 shape = x.shape
x = x.view(-1,n,shape[1],shape[2],shape[3] ).permute(0,2,1,3,4 ).contiguous() \ 
.view(-1,shape[1],shape[2]*n,shape[3])
 x = self.head(x)
 return x[:,:1]
train = MungeTrain()
delta = train.visit_date.max() -pd.Timedelta(weeks=5)
 lastfiveweekstrain = train[train.visit_date>=delta].copy()
lastfiveweekstrain = lastfiveweekstrain.reset_index(drop=True)
 train = train[train.visit_date<delta].copy()
train = train.reset_index(drop=True)
 test = MungeTest(train.columns )<define_variables>
models = []
for path in MODELS:
state_dict = torch.load(path,map_location=torch.device('cpu'))
model = Model(n=1+10)
 model.load_state_dict(state_dict)
 model.float()
model.eval()
model.cuda()
models.append(model)
 del state_dict
deltavals = [1,2,3,4,5]
xgbtrainpreds = None
xgbtestpreds = None
for i,deltaweek in enumerate(deltavals):
print(i)
 delta = pd.Timedelta(weeks=deltaweek)
 blindmin = train.visit_date.max() -delta
blindmax = train.visit_date.max()
vismin = blindmin-delta-pd.Timedelta(days=1)
 vismax = blindmax-delta-pd.Timedelta(days=1)
 btrain = None
for x in range(int(60./deltaweek)) :
vistrain = train[(train.visit_date<=vismax)].copy()
blindtrain = train[(train.visit_date>=blindmin)&(train.visit_date<=blindmax)].copy()
features = ['air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
blindtrain[c+'_Count_Store'] = np.nan
blindtrain[c+'_Mean_Store'] = np.nan
blindtrain[c+'_Median_Store'] = np.nan
blindtrain[c+'_Max_Store'] = np.nan
blindtrain[c+'_Min_Store'] = np.nan
blindtrain[c+'_Std_Store'] = np.nan
blindtrain[[c+'_Count_Store',c+'_Mean_Store',
c+'_Median_Store',c+'_Max_Store',
c+'_Min_Store', c+'_Std_Store']] = LeaveOneOut(vistrain,
blindtrain,
list(['air_store_id']),
c,
useLOO=False,
cut=0 ).values
features = ['air_store_id',
'air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
blindtrain[c+'_Count'] = np.nan
blindtrain[c+'_Mean'] = np.nan
blindtrain[c+'_Median'] = np.nan
blindtrain[c+'_Max'] = np.nan
blindtrain[c+'_Min'] = np.nan
blindtrain[c+'_Std'] = np.nan
blindtrain[[c+'_Count',c+'_Mean',
c+'_Median',c+'_Max',
c+'_Min', c+'_Std']] = LeaveOneOut(vistrain,
blindtrain,
list([]),
c,
useLOO=False,
cut=0,
addNoise=False ).values
if('air_store_id'!=c):
blindtrain.drop(c,inplace=True,axis=1)
 if(btrain is None):
btrain = blindtrain.copy()
else:
btrain = pd.concat([btrain,blindtrain])
 vismax -= pd.Timedelta(weeks=deltaweek)
 vismin -= pd.Timedelta(weeks=deltaweek)
 blindmin -= pd.Timedelta(weeks=deltaweek)
 blindmax -= pd.Timedelta(weeks=deltaweek)
 vistrain = train.copy()
features = ['air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
lastfiveweekstrain[c+'_Count_Store'] = np.nan
lastfiveweekstrain[c+'_Mean_Store'] = np.nan
lastfiveweekstrain[c+'_Median_Store'] = np.nan
lastfiveweekstrain[c+'_Max_Store'] = np.nan
lastfiveweekstrain[c+'_Min_Store'] = np.nan
lastfiveweekstrain[c+'_Std_Store'] = np.nan
lastfiveweekstrain[[c+'_Count_Store',c+'_Mean_Store',
c+'_Median_Store',c+'_Max_Store',
c+'_Min_Store', c+'_Std_Store']] = LeaveOneOut(vistrain,
lastfiveweekstrain,
list(['air_store_id']),
c,
useLOO=False,
cut=0 ).values
test[c+'_Count_Store'] = np.nan
test[c+'_Mean_Store'] = np.nan
test[c+'_Median_Store'] = np.nan
test[c+'_Max_Store'] = np.nan
test[c+'_Min_Store'] = np.nan
test[c+'_Std_Store'] = np.nan
test[[c+'_Count_Store',c+'_Mean_Store',
c+'_Median_Store',c+'_Max_Store',
c+'_Min_Store', c+'_Std_Store']] = LeaveOneOut(vistrain,
test,
list(['air_store_id']),
c,
useLOO=False,
cut=0 ).values
features = ['air_store_id',
'air_genre_name',
'air_area_name', 'hpg_store_id',
'hpg_genre_name', 'hpg_area_name',
'day_of_week', 'holiday_flg']
for c in features:
lastfiveweekstrain[c+'_Count'] = np.nan
lastfiveweekstrain[c+'_Mean'] = np.nan
lastfiveweekstrain[c+'_Median'] = np.nan
lastfiveweekstrain[c+'_Max'] = np.nan
lastfiveweekstrain[c+'_Min'] = np.nan
lastfiveweekstrain[c+'_Std'] = np.nan
lastfiveweekstrain[[c+'_Count',c+'_Mean',
c+'_Median',c+'_Max',
c+'_Min', c+'_Std']] = LeaveOneOut(vistrain,
lastfiveweekstrain,
list([]),
c,
useLOO=False,
cut=0,
addNoise=False ).values
test[c+'_Count'] = np.nan
test[c+'_Mean'] = np.nan
test[c+'_Median'] = np.nan
test[c+'_Max'] = np.nan
test[c+'_Min'] = np.nan
test[c+'_Std'] = np.nan
test[[c+'_Count',c+'_Mean',
c+'_Median',c+'_Max',
c+'_Min', c+'_Std']] = LeaveOneOut(vistrain,
test,
list([]),
c,
useLOO=False,
cut=0,
addNoise=False ).values
d_train = xgb.DMatrix(btrain[btrain.columns[3:]], label=btrain.visitors)
 d_valid = xgb.DMatrix(lastfiveweekstrain[btrain.columns[3:]], label=lastfiveweekstrain.visitors)
 d_test = xgb.DMatrix(test[btrain.columns[3:]])
 params = {}
params['objective'] = 'reg:linear'
params['eval_metric'] = 'rmse'
params['eta'] = 0.1
params['max_depth'] = 7
params['subsample']= 0.8
params['colsample_bytree']= 0.8
params['silent'] = 1
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 1000, watchlist, early_stopping_rounds=20, verbose_eval=10)
 xgbx = clf.predict(d_valid,ntree_limit=clf.best_iteration+1)
 xgby = clf.predict(d_test,ntree_limit=clf.best_iteration+1)
 trainpreds = pd.DataFrame()
trainpreds['visit_date'] = lastfiveweekstrain.visit_date.values
trainpreds['air_store_id'] = lastfiveweekstrain.air_store_id.values
trainpreds['visitors'] = lastfiveweekstrain.visitors.values
trainpreds['weeks_'+str(deltaweek)] = xgbx
testpreds = pd.DataFrame()
testpreds['visit_date'] = test.visit_date.values
testpreds['air_store_id'] = test.air_store_id.values
testpreds['weeks_'+str(deltaweek)] = xgby
if(xgbtrainpreds is None):
xgbtrainpreds = trainpreds.copy()
xgbtestpreds = testpreds.copy()
else:
xgbtrainpreds = xgbtrainpreds.merge(trainpreds[['air_store_id','visit_date','weeks_'+str(deltaweek)]],on=['air_store_id','visit_date'])
 xgbtestpreds = xgbtestpreds.merge(testpreds[['air_store_id','visit_date','weeks_'+str(deltaweek)]],on=['air_store_id','visit_date'] )<split>
def tile(img):
shape = img.shape
pad0,pad1 =(sz - shape[0]%sz)%sz,(sz - shape[1]%sz)%sz
img = np.pad(img,[[pad0//2,pad0-pad0//2],[pad1//2,pad1-pad1//2],[0,0]],constant_values=255)
 img = img.reshape(img.shape[0]//sz,sz,img.shape[1]//sz,sz,3)
 img = img.transpose(0,2,1,3,4 ).reshape(-1,sz,sz,3)
 if len(img)< N:
img = np.pad(img,[[0,N-len(img)],[0,0],[0,0],[0,0]],constant_values=255)
 idxs = np.argsort(img.reshape(img.shape[0],-1 ).sum(-1)) [:N]
img = img[idxs]
return img
mean = torch.Tensor([1.0-0.85506157, 1.0-0.7035249, 1.0-0.80203127])
 std = torch.Tensor([0.40011922, 0.52504386, 0.42675745])
 class PandaDataset(Dataset):
def __init__(self, path, test):
self.path = path
self.names = list(pd.read_csv(test ).image_id)
 def __len__(self):
return len(self.names)
 def __getitem__(self, idx):
name = self.names[idx]
img = skimage.io.MultiImage(os.path.join(DATA,name+'.tiff')) [1]
tiles = torch.Tensor(( 255 - tile(img)) /255.0)
 tiles =(tiles - mean)/std
return tiles.permute(0,3,1,2), name
X_train, X_test, y_train, y_test = train_test_split(xgbtrainpreds[['weeks_1', 'weeks_2', 'weeks_3', 'weeks_4', 'weeks_5']],
xgbtrainpreds.visitors, test_size=0.2, random_state=42)
 d_train = xgb.DMatrix(X_train, label=y_train)
 d_valid = xgb.DMatrix(X_test, label=y_test)
 d_test = xgb.DMatrix(xgbtestpreds[['weeks_1', 'weeks_2', 'weeks_3', 'weeks_4', 'weeks_5']])
 params = {}
params['objective'] = 'reg:linear'
params['eval_metric'] = 'rmse'
params['eta'] = 0.2
params['max_depth'] = 3
params['subsample']=0.8
params['colsample_bytree']=0.8
params['silent'] = 1
watchlist = [(d_train, 'train'),(d_valid, 'valid')]
clf = xgb.train(params, d_train, 250, watchlist, early_stopping_rounds=20, verbose_eval=10)
 xgbx = clf.predict(d_valid,ntree_limit=clf.best_iteration+1)
 xgby = clf.predict(d_test,ntree_limit=clf.best_iteration+1 )<predict_on_test>
ths = np.array([1.03125,1.03125,0.8125,0.90625,1.0859375] ).cumsum()
ws = torch.Tensor(ws ).cuda()
sub_df = pd.read_csv(SAMPLE)
 if os.path.exists(DATA):
bs=2
ds = PandaDataset(DATA,TEST)
 dl = DataLoader(ds, batch_size=bs, num_workers=nworkers, shuffle=False)
 names,preds = [],[]
with torch.no_grad() :
for x,y in tqdm(dl):
x = x.cuda()
b = x.shape[0]
x = torch.stack([x,x.flip(-1),x.flip(-2),x.flip(-1,-2),x.transpose(-1,-2),\
x.transpose(-1,-2 ).flip(-1)],1)
 n_tta = 6
x = x.view(-1,N,3,sz,sz)
 p = [model(x)for model in models]
p = torch.stack(p,1)
 p = p.view(b,n_tta,len(models)//4,4 ).permute(0,2,3,1 ).reshape(b,len(models)//4,4*n_tta)
 w = ws/torch.clamp(p.std(-1),0.04)
 w /= w.sum(-1 ).unsqueeze(-1)
 p =(p*w.unsqueeze(-1)).mean(-1 ).sum(-1 ).cpu()
p = 6.0*torch.sigmoid(p ).view(-1)
 for pi,yi in zip(p,y):
preds.append(bisect_right(ths, pi.numpy()))
names.append(yi)
 sub_df = pd.DataFrame({'image_id': names, 'isup_grade': preds} )
<compute_test_metric><EOS> 
sub_df.to_csv("submission.csv", index=False)
 sub_df.head()