Kaggle competition: Prostate cANcer graDe Assessment (PANDA) Challenge
<SOS> metric: QuadraticWeightedKappa Kaggle data source: prostate-cancer-grade-assessment-(panda)-challenge<filter>
!pip install /kaggle/input/timm-package/timm-0.1.26-py3-none-any.whl
PKGPATH='.. /input/kprostate111/prostatev111'
sys.path.append(PKGPATH)
 os.environ['KMP_DUPLICATE_LIB_OK']='True'
warnings.filterwarnings("ignore" )
check_stat1 = weather[cols_not_date][weather.Station == '1'].apply(pd.value_counts, axis=0 ).fillna(0)
 check_stat1.loc[['M', '-', 'T']]<filter>
from utils.imageutils import rotatecrop, pairrot, cropcoords, noisecrop, Mish
from utils.imageutils import padsplit, padimg, pixctr, cropper, condenseImgls
from utils.imageutils import padsplitv2, chunk_squaresv2, condenseImglsv2, cropperv2
from utils.imageutils import chunk_squares, frame_combine, balancedSampler
from utils.utilities import get_logger, dumpobj, loadobj, Identity
from utils.utilities import SpatialDropout, qwk5
check_stat2 = weather[cols_not_date][weather.Station == '2'].apply(pd.value_counts, axis=0 ).fillna(0)
 check_stat2.loc[['M', '-', 'T']]<feature_engineering>
logger = get_logger('Sequence model :', 'INFO')
 device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 try:
logger.info('Device : {}'.format(torch.cuda.get_device_name(0)))
 except:
logger.info('Device : Not available')
 logger.info('Cuda available : {}'.format(torch.cuda.is_available()))
n_gpu = torch.cuda.device_count()
logger.info('Cuda n_gpus : {}'.format(n_gpu))
check.loc[['M', '-', 'T']]/(len(weather)) * 100<feature_engineering>
NPIXELS=20*10**6
NSAMP=300
DATATYPE = 'test'
INPATH = '.. /input/prostate-cancer-grade-assessment'
SAMPLE = f'{INPATH}/sample_submission.csv'
DATA = f'{INPATH}/{DATATYPE}_images'
subdf = pd.read_csv(f'{INPATH}/sample_submission.csv')
 tstdf = pd.read_csv(f'{INPATH}/{DATATYPE}.csv')
 if DATATYPE=='train':
folds= pd.read_csv(f'{INPATH}/.. /train-clust/train_clust.csv.gzip',
usecols = ['image_id', 'fold'],
compression = 'gzip')
 removedf = pd.read_csv('.. /input/remove/removedf.csv')
 alldf = pd.read_csv(f'{INPATH}/{DATATYPE}.csv')
 alldf = alldf[~alldf.image_id.isin(removedf.imgname)]
alldf = pd.merge(alldf, folds, on='image_id')
 tstdf = alldf.query('fold == 0' ).reset_index(drop=True)[:NSAMP]
print(tstdf.isup_grade.value_counts())
 print(tstdf.shape )
check_stat1.loc[['M', '-', 'T']]/(len(weather)) * 100<feature_engineering>
class options:
max_len = 36
tilesize = 224
data_path = f'{DATATYPE}_images'
bsize = 2
check_stat2.loc[['M', '-', 'T']]/(len(weather)) * 100<correct_missing_values>
def cropload(imname, thresh = 255):
img = MultiImage(imname)[1]
krows = np.where(np.min(img, 0)< 255)[0]
kcols = np.where(np.min(img, 1)< 255)[0]
img = img[kcols[0]: kcols[-1] + 1, krows[0]: krows[-1] + 1]
return img
val_transforms = A.Compose([
A.RandomCrop(224*6, 224*6, always_apply=True, p=1),
A.NoOp() ,
])
 def get_tiles(img, mode, n_tiles , tile_size, cutoff = 230, tile_cutoff = 0.0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 slice_size = 2
img3 = img2.reshape(\
img2.shape[0] // tile_size, \
tile_size , \
img2.shape[1]* slice_size // tile_size, \
tile_size // slice_size, \
3 \ 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size//slice_size,3)
 bb, hh, ww, cc = img3.shape
idx =(img3[:,:,:,1].reshape(bb,-1)< cutoff ).sum(1)>(hh*ww)*tile_cutoff
img3 = img3[idx]
img3 = np.pad(img3,[[0, slice_size -(img3.shape[0] % slice_size)],[0,0],[0,0],[0,0]], constant_values=255)
 img3 = np.concatenate(img3, 1)\ 
.reshape(tile_size, img3.shape[0]//slice_size, tile_size, 3)\ 
.transpose(1,0,2,3 ).reshape(-1, tile_size, tile_size,3)
 if len(img3)< n_tiles:
padder = n_tiles-len(img3)
 img3 = np.pad(img3,[[padder//2,padder-(padder//2)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = sorted(np.argsort(( img3.reshape(img3.shape[0],-1)< cutoff ).sum(-1)) [-n_tiles:])
 img3 = img3[idxs]
return img3, 0
def chunks(lst, n):

 for i in range(0, len(lst), n):
yield lst[i:i + n]
tst_transforms = A.Compose([
A.Transpose(p=0.5),
A.VerticalFlip(p=0.5),
A.HorizontalFlip(p=0.5),
])
 class ProstateDataset(Dataset):
def __init__(self, df, path, transform,
tile_mode=0, modtype='train'):
self.path = path
self.data_path = options.data_path
self.df = df
logger.info(f'Dataset {modtype} {self.df.shape}')
 if modtype!='test':
self.df.gleason_score = self.df.gleason_score.str.replace('negative', '0+0')
 self.gleasonmap = {'0': 0, '3': 1, '4': 2, '5': 3}
self.tile_mode = tile_mode
self.basesz = 224*6
self.transform = transform
self.type = modtype
def __len__(self):
return len(self.df)
 def __getitem__(self, idx):
imname = self.df.iloc[idx].image_id
imgpath = f'{INPATH}/{self.data_path}/{imname}.tiff'
mskpath = f'{INPATH}/{self.data_path}/{imname}_mask.png'
imgorig = cropload(imgpath, thresh = 255)
 BASESZ = 224*6
sample = {}
for iii in range(4):
img = imgorig.copy()
for _ in range(iii):
img = np.rot90(img)
 hhh, www, _ = img.shape
thresh = 230
BASE =(img[:,:, 1]<thresh ).sum()
FILLED_RATIO= 0
n_tiles = 1
image_size = 2000
while(( FILLED_RATIO)<0.85)and(image_size>228):
n_tiles += 1
image_size = BASESZ//n_tiles
image_size = int(image_size +(4-image_size%4))
tiles, OK = get_tiles(img, 2*random.randint(0,1), n_tiles**2, image_size, cutoff=thresh)
 FILLED =(tiles[:,:,:, 1]<thresh ).sum()
FILLED_RATIO = FILLED/BASE
if FILLED_RATIO < 0.8:
dsize =(1+FILLED_RATIO)/2
img = cv2.resize(img,(int(www * dsize), int(hhh * dsize)) , \
interpolation=cv2.INTER_CUBIC)
 BASE =(img[:,:, 1]<thresh ).sum()
FILLED_RATIO= 0
n_tiles = 1
image_size = 2000
while(( FILLED_RATIO)<0.85)and(image_size>228):
n_tiles += 1
image_size = BASESZ//n_tiles
image_size = int(image_size +(4-image_size%4))
tiles, OK = get_tiles(img, 2*random.randint(0,1), n_tiles**2, image_size, cutoff=thresh)
 FILLED =(tiles[:,:,:, 1]<thresh ).sum()
FILLED_RATIO = FILLED/BASE
fillls = []
if FILLED_RATIO < 0.9:
tilels = []
for tile_cutoff in [0.0, 0.1, 0.2]:
tilels.append(get_tiles(img, 2*random.randint(0,1), n_tiles**2, image_size, cutoff=thresh, tile_cutoff = tile_cutoff)[0])
 fillls.append(( tilels[-1][:,:,:, 1]<thresh ).sum() /BASE)
 fillls = np.array(fillls ).round(3)
 tiles = tilels[np.argmax(fillls)]
else:
fillls = np.array([FILLED_RATIO])
 logtxt = f'flip {iii} img size {image_size} init filled {FILLED_RATIO:.4f}'
image_size = tiles.shape[1]
n_row_tiles = int(np.sqrt(tiles.shape[0]))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
this_img = tiles[i]
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.uint8)
 images = 255 - images
images = self.transform(image=images)['image']
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 sample[f'image{iii}'] = images
if self.type=='train':
sample['isup_grade'] = torch.zeros(5)
 sample['isup_grade'][: self.df.iloc[idx].isup_grade ] = 1.
 sample['gleason1'], sample['gleason2'] = map(self.gleasonmap.get, \
self.df.gleason_score.iloc[idx].split('+'))
return sample
def collatefn(batch):
batch = [b for b in batch if b is not None]
b = dict(( f'image{i}', torch.stack([torch.tensor(l[f'image{i}'])for l in batch])) for i in range(4))
if 'isup_grade' in batch[0]:
b['isup_grade'] = torch.stack([i['isup_grade'] for i in batch])
 for ii in ['gleason1', 'gleason2']:
b[ii] = torch.tensor([l[ii] for l in batch])
 return b
weather = weather.replace('M', np.NaN)
 weather = weather.replace('-', np.NaN)
 weather = weather.replace('T', 0.005)
 weather.Tmax = weather.Tmax.fillna(method = 'ffill')
 weather.Tmin = weather.Tmin.fillna(method = 'ffill')
 weather.Depart = weather.Depart.fillna(method = 'ffill')
 weather.Heat = weather.Heat.fillna(method = 'ffill')
 weather.Cool = weather.Cool.fillna(method = 'ffill')
 weather.PrecipTotal = weather.PrecipTotal.fillna(method = 'ffill' )<data_type_conversions>
trndataargs = {'path': INPATH}
tstdataset = ProstateDataset(tstdf, modtype=DATATYPE, transform = val_transforms, **trndataargs )
to_numeric = ['Tmax','Tmin','Tavg', 'Depart', 'Heat', 'Cool', 'PrecipTotal']
for col in to_numeric:
weather[col]= pd.to_numeric(weather[col] )<correct_missing_values>
weather.Sunrise = weather.Sunrise.fillna(method = 'ffill')
 weather.Sunset = weather.Sunset.fillna(method = 'ffill' )<data_type_conversions>
loaderls = []
trnloaderargs = {'num_workers' : 4, 'collate_fn' : collatefn}
tstloader = DataLoader(tstdataset, shuffle=False, batch_size=options.bsize, **trnloaderargs )
counter = 0
tracker = []
for index, val in enumerate(weather.Sunset):
try:
pd.to_datetime(val, format = '%H%M' ).time()
except:
counter += 1
tracker.append(( index, val, val[2:], counter))
print(tracker[-1])
<data_type_conversions>
class SeqNet(nn.Module):
def __init__(self, architecture = 'mixnet_m', pretrained=True, \
dense_units = 256, dropout = 0.2, isuplabels = 5, \
nblocks=4, concatfinal = 1,
glabels = 4):
super(SeqNet, self ).__init__()
logger.info('Architecture {} dense {} dropout {}'.format(\
architecture, dense_units, dropout))
self.arch=architecture
if 'eff' in self.arch:
self.bbmod = timm.create_model(architecture, pretrained)
 self.dense_units = dense_units
self.bbmod.classifier = Identity()
self.myctfc = nn.Linear(self.bbmod.num_features, isuplabels)
 def forward(self, x):
emb = self.bbmod(x)
 out_isup = self.myctfc(emb)
 return out_isup
time_func = lambda x: pd.Timestamp(pd.to_datetime(x, format = '%H%M'))<feature_engineering>
models = []
for cpt_path in sorted(glob.glob('.. /input/mnetlv33/*')) :
logger.info(f'Load {cpt_path}')
 models.append(SeqNet(architecture = 'tf_efficientnet_b0_ns', pretrained=False, nblocks=4, concatfinal = 1,\
dense_units = 256, dropout = 0.0, isuplabels = 5, glabels = 4))
cpt = torch.load(cpt_path, map_location=torch.device(device))
models[-1].load_state_dict(cpt)
 models[-1].to(device);
models[-1].eval() ;
del cpt
gc.collect()
weather.Sunrise = weather.Sunrise.apply(time_func )<feature_engineering>
ttafns = [
lambda x: x,
lambda x: x.flip(-1),
lambda x: x.flip(-2),
lambda x: x.flip(-1, -2),
lambda x: x.transpose(-1, -2),
lambda x: x.transpose(-1, -2 ).flip(-1),
lambda x: x.transpose(-1, -2 ).flip(-2),
lambda x: x.transpose(-1, -2 ).flip(-1, -2),
]
iters = 4
modlens = len(models)
 if os.path.exists(f'{INPATH}/{DATATYPE}_images/'):
predsls = []
for loader in [tstloader]:
with torch.no_grad() :
ypredval = []
for step, batch in enumerate(loader):
batchls = []
for i in range(iters):
x = batch[f'image{i}'].to(device, dtype=torch.float)
 for midx in range(modlens):
for f in ttafns:
outisup = models[midx](f(x))
outisup = torch.sigmoid(outisup)
 batchls.append(outisup)
 out = sum(batchls)/len(batchls)
 out = out.cpu().sum(1 ).detach().numpy()
ypredval.append(out)
 del x, batch
if step%10==0:
logger.info('Val step {} of {}'.format(step, len(loader)))
 torch.cuda.empty_cache()
gc.collect()
predsls.append(np.concatenate(ypredval, 0))
weather.Sunset = weather.Sunset.apply(time_func )<data_type_conversions>
subdf = tstloader.df[['image_id']]
subdf['isup_grade'] =(sum(predsls)/len(predsls)).round().astype(np.int32 )
minutes=(weather.Sunset - weather.Sunrise ).astype('timedelta64[m]' )<feature_engineering>
subdf.to_csv("submission.csv", index=False)
 subdf.head(20 )
hours = minutes/60<data_type_conversions>
if DATATYPE=='train':
logger.info(( subdf.isup_grade == tstloader.df.isup_grade ).value_counts() )
<feature_engineering><EOS> 
if DATATYPE=='train':
logger.info(qwk5(subdf.isup_grade, tstloader.df.isup_grade))