Kaggle competition: WiDS Datathon 2019
<SOS> metric: auc Kaggle data source: wids-datathon-2019<load_from_csv>
import fastai
from fastai.vision import *
data_dir = '.. /input/prostate-cancer-grade-assessment'
df_train = pd.read_csv(os.path.join(data_dir, 'train.csv'))
df_test = pd.read_csv(os.path.join(data_dir, 'test.csv'))
df_sub = pd.read_csv(os.path.join(data_dir, 'sample_submission.csv'))
model_dir = '.. /input/panda-public-models'
image_folder = os.path.join(data_dir, 'test_images')
 is_test = os.path.exists(image_folder)
 image_folder = image_folder if is_test else os.path.join(data_dir, 'train_images')
 df = df_test if is_test else df_train.loc[:100]
tile_size = 256
image_size = 256
n_tiles = 36
batch_size = 8
num_workers = 4
device = torch.device('cuda')
 print(image_folder )<define_search_model>
work_dir = Path('/kaggle/working/')
 path = Path('.. /input' )
class enetv2(nn.Module):
def __init__(self, backbone, out_dim):
super(enetv2, self ).__init__()
self.enet = enet.EfficientNet.from_name(backbone)
 self.myfc = nn.Linear(self.enet._fc.in_features, out_dim)
 self.enet._fc = nn.Identity()
def extract(self, x):
return self.enet(x)
 def forward(self, x):
x = self.extract(x)
 x = self.myfc(x)
 return x
def load_models(model_files):
models = []
for model_f in model_files:
model_f = os.path.join(model_dir, model_f)
 backbone = 'efficientnet-b0'
model = enetv2(backbone, out_dim=5)
 model.load_state_dict(torch.load(model_f, map_location=lambda storage, loc: storage), strict=True)
 model.eval()
model.to(device)
 models.append(model)
 print(f'{model_f} loaded!')
 return models
model_files = [
'cls_effnet_b0_Rand36r36tiles256_big_bce_lr0.3_augx2_30epo_model_fold0.pth'
]
models = load_models(model_files )<categorify>
train = 'train_images/train_images'
test = path/'leaderboard_test_data/leaderboard_test_data'
holdout = path/'leaderboard_holdout_data/leaderboard_holdout_data'
sample_sub = path/'SampleSubmission.csv'
labels = path/'traininglabels.csv'
def get_tiles(img, mode=0):
result = []
h, w, c = img.shape
pad_h =(tile_size - h % tile_size)% tile_size +(( tile_size * mode)// 2)
 pad_w =(tile_size - w % tile_size)% tile_size +(( tile_size * mode)// 2)
 img2 = np.pad(img,[[pad_h // 2, pad_h - pad_h // 2], [pad_w // 2,pad_w - pad_w//2], [0,0]], constant_values=255)
 img3 = img2.reshape(
 img2.shape[0] // tile_size,
tile_size,
img2.shape[1] // tile_size,
tile_size,
3 
)
 img3 = img3.transpose(0,2,1,3,4 ).reshape(-1, tile_size, tile_size,3)
 n_tiles_with_info =(img3.reshape(img3.shape[0],-1 ).sum(1)< tile_size ** 2 * 3 * 255 ).sum()
if len(img)< n_tiles:
img3 = np.pad(img3,[[0,N-len(img3)],[0,0],[0,0],[0,0]], constant_values=255)
 idxs = np.argsort(img3.reshape(img3.shape[0],-1 ).sum(-1)) [:n_tiles]
img3 = img3[idxs]
for i in range(len(img3)) :
result.append({'img':img3[i], 'idx':i})
 return result, n_tiles_with_info >= n_tiles
class PANDADataset(Dataset):
def __init__(self,
df,
image_size,
n_tiles=n_tiles,
tile_mode=0,
rand=False,
sub_imgs=False 
):
self.df = df.reset_index(drop=True)
 self.image_size = image_size
self.n_tiles = n_tiles
self.tile_mode = tile_mode
self.rand = rand
self.sub_imgs = sub_imgs
def __len__(self):
return self.df.shape[0]
def __getitem__(self, index):
row = self.df.iloc[index]
img_id = row.image_id
tiff_file = os.path.join(image_folder, f'{img_id}.tiff')
 image = skimage.io.MultiImage(tiff_file)[1]
tiles, OK = get_tiles(image, self.tile_mode)
 if self.rand:
idxes = np.random.choice(list(range(self.n_tiles)) , self.n_tiles, replace=False)
 else:
idxes = list(range(self.n_tiles))
idxes = np.asarray(idxes)+ self.n_tiles if self.sub_imgs else idxes
n_row_tiles = int(np.sqrt(self.n_tiles))
images = np.zeros(( image_size * n_row_tiles, image_size * n_row_tiles, 3))
for h in range(n_row_tiles):
for w in range(n_row_tiles):
i = h * n_row_tiles + w
if len(tiles)> idxes[i]:
this_img = tiles[idxes[i]]['img']
else:
this_img = np.ones(( self.image_size, self.image_size, 3)).astype(np.uint8)* 255
this_img = 255 - this_img
h1 = h * image_size
w1 = w * image_size
images[h1:h1+image_size, w1:w1+image_size] = this_img
images = images.astype(np.float32)
 images /= 255
images = images.transpose(2, 0, 1)
 return torch.tensor(images)
<load_pretrained>
df = pd.read_csv(labels)
 df_sample = pd.read_csv(sample_sub )
dataset = PANDADataset(df, image_size, n_tiles, 0)
 loader = DataLoader(dataset, batch_size=batch_size, num_workers=num_workers, shuffle=False)
 dataset2 = PANDADataset(df, image_size, n_tiles, 2)
 loader2 = DataLoader(dataset2, batch_size=batch_size, num_workers=num_workers, shuffle=False )<save_to_csv>
df[df['score']<0.75]
LOGITS = []
LOGITS2 = []
with torch.no_grad() :
for data in tqdm(loader):
data = data.to(device)
 logits = models[0](data)
 LOGITS.append(logits)
 for data in tqdm(loader2):
data = data.to(device)
 logits = models[0](data)
 LOGITS2.append(logits)
 LOGITS =(torch.cat(LOGITS ).sigmoid().cpu() + torch.cat(LOGITS2 ).sigmoid().cpu())/ 2
PREDS = LOGITS.sum(1 ).round().numpy()
df['isup_grade'] = PREDS.astype(int)
 df[['image_id', 'isup_grade']].to_csv('submission.csv', index=False)
 print(df.head())
 print()
print(df.isup_grade.value_counts() )<install_modules>
test_names = [f for f in test.iterdir() ]
holdout_names = [f for f in holdout.iterdir() ]
!pip install.. /input/installation/Keras_Applications-1.0.8-py3-none-any.whl -f./ --no-index --no-deps
!pip install.. /input/install-2/efficientnet-1.1.0-py3-none-any.whl -f./ --no-index --no-deps<set_options>
data =(src.transform(get_transforms() , size=128)
.databunch(bs=64)
.normalize(imagenet_stats))
warnings.filterwarnings('ignore')
 print('tensorflow version:', tf.__version__ )<set_options>
def auc_score(preds,targets):
return torch.tensor(roc_auc_score(targets,preds[:,1]))
gpus = tf.config.experimental.list_physical_devices('GPU')
 if gpus:
try:
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
 logical_gpus = tf.config.experimental.list_logical_devices('GPU')
 print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
 except RuntimeError as e:
print(e)
 else:
print('no gpus' )<define_variables>
learn = create_cnn(data, models.resnet18,
metrics=[accuracy],
model_dir='/kaggle/working/models' )
DATA_PATH = '.. /input/prostate-cancer-grade-assessment'
MODELS_PATH = '.'
IMG_SIZE = 128
SEQ_LEN = 36
TARGET_SIZE = [528, 528]
BATCH_SIZE = 16
MDL_VERSION = 'v0'
SEED = 80
LEVEL = 1<train_model>
lr = 1e-2
def get_axis_max_min(array, axis=0):
one_axis = list(( array != 255 ).sum(axis=tuple([x for x in(0, 1, 2)if x != axis])))
 axis_min = next(( i for i, x in enumerate(one_axis)if x), 0)
 axis_max = len(one_axis)- next(( i for i, x in enumerate(one_axis[::-1])if x), 0)
 return axis_min, axis_max
class DataGenPanda(Sequence):
def __init__(self, imgs_path, df, batch_size=32,
mode='fit', shuffle=False, aug=None,
seq_len=SEQ_LEN, img_size=IMG_SIZE, n_classes=6):
self.imgs_path = imgs_path
self.df = df
self.shuffle = shuffle
self.mode = mode
self.aug = aug
self.batch_size = batch_size
self.img_size = img_size
self.seq_len = seq_len
self.n_classes = n_classes
self.side = int(seq_len **.5)
 self.on_epoch_end()
def __len__(self):
return int(np.floor(len(self.df)/ self.batch_size))
def on_epoch_end(self):
self.indexes = np.arange(len(self.df))
if self.shuffle:
np.random.shuffle(self.indexes)
 def __getitem__(self, index):
X = np.zeros(( self.batch_size, self.side * self.img_size, self.side * self.img_size, 3), dtype=np.float32)
 imgs_batch = self.df[index * self.batch_size :(index + 1)* self.batch_size]['image_id'].values
for i, img_name in enumerate(imgs_batch):
img_path = '{}/{}.tiff'.format(self.imgs_path, img_name)
 img_patches = self.get_patches(img_path)
 X[i, ] = self.glue_to_one(img_patches)
 X = tf.image.convert_image_dtype(X, tf.float32)
 X = tf.image.resize(X, size = TARGET_SIZE)
 if self.mode == 'fit':
y = np.zeros(( self.batch_size, self.n_classes), dtype=np.float32)
 lbls_batch = self.df[index * self.batch_size :(index + 1)* self.batch_size]['isup_grade'].values
for i in range(self.batch_size):
y[i, lbls_batch[i]] = 1
return X, y
elif self.mode == 'predict':
return X
else:
raise AttributeError('mode parameter error')
 def get_patches(self, img_path):
num_patches = self.seq_len
p_size = self.img_size
img = skimage.io.MultiImage(img_path)[LEVEL]
a0min, a0max = get_axis_max_min(img, axis=0)
 a1min, a1max = get_axis_max_min(img, axis=1)
 img = img[a0min:a0max, a1min:a1max, :].astype(np.float32)/ 255
if self.aug:
img = self.aug(image=img)['image']
pad0, pad1 =(p_size - img.shape[0] % p_size)% p_size,(p_size - img.shape[1] % p_size)% p_size
img = np.pad(
 img,
[
[pad0 // 2, pad0 - pad0 // 2],
[pad1 // 2, pad1 - pad1 // 2],
[0, 0]
],
constant_values=1 
)
 img = img.reshape(img.shape[0] // p_size, p_size, img.shape[1] // p_size, p_size, 3)
 img = img.transpose(0, 2, 1, 3, 4 ).reshape(-1, p_size, p_size, 3)
 if len(img)< num_patches:
img = np.pad(
 img,
[
[0, num_patches - len(img)],
[0, 0],
[0, 0],
[0, 0]
],
constant_values=1 
)
 idxs = np.argsort(img.reshape(img.shape[0], -1 ).sum(-1)) [:num_patches]
return np.array(img[idxs])
 def glue_to_one(self, imgs_seq):
img_glue = np.zeros(( self.img_size * self.side, self.img_size * self.side, 3), dtype=np.float32)
 for i, ptch in enumerate(imgs_seq):
x = i // self.side
y = i % self.side
img_glue[x * self.img_size :(x + 1)* self.img_size,
y * self.img_size :(y + 1)* self.img_size, :] = ptch
return img_glue<split>
learn.fit_one_cycle(6, lr )
train = pd.read_csv('{}/train.csv'.format(DATA_PATH))
print('train: ', train.shape, '| unique ids:', sum(train['isup_grade'].value_counts()))
X_train, X_val = train_test_split(train, test_size=.2, stratify=train['isup_grade'], random_state=SEED)
 lbl_value_counts = X_train['isup_grade'].value_counts()
class_weights = {i: max(lbl_value_counts)/ v for i, v in lbl_value_counts.items() }
print('classes weigths:', class_weights )<train_model>
Then we unfreeze and train the whole model, with lower lr .
aug = albu.Compose(
 [
albu.OneOf([albu.RandomBrightness(limit=.15), albu.RandomContrast(limit=.3), albu.RandomGamma() ], p=.25),
albu.HorizontalFlip(p=.25),
albu.VerticalFlip(p=.25),
albu.ShiftScaleRotate(shift_limit=.1, scale_limit=.1, rotate_limit=20, p=.25)
 ] 
)
 train_datagen = DataGenPanda(
 imgs_path='{}/train_images'.format(DATA_PATH),
df=X_train,
batch_size=BATCH_SIZE,
mode='fit',
shuffle=True,
aug=aug,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6 
)
 val_datagen = DataGenPanda(
 imgs_path='{}/train_images'.format(DATA_PATH),
df=X_val,
batch_size=BATCH_SIZE,
mode='fit',
shuffle=False,
aug=None,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6
)<load_pretrained>
learn.unfreeze()
learn.fit_one_cycle(3, slice(1e-4, 1e-3))
MODEL_DIR = '.. /input/pandamodels/'
models = {'B0': 'EfficientNetB0.h5',
'B2':'EfficientNetB2.h5',
'B4':'EfficientNetB4.h5',
'B6':'EfficientNetB6.h5',
'Xception':'Xception.h5'}
model = tf.keras.models.load_model(MODEL_DIR + models['B6'], compile=False)
 print('Model loaded:', model )<load_from_csv>
learn.save('128' )
test = pd.read_csv('{}/test.csv'.format(DATA_PATH))
preds = [[0] * 6] * len(test)
 if os.path.exists('.. /input/prostate-cancer-grade-assessment/test_images'):
print('Inference on test images...')
 subm_datagen = DataGenPanda(
 imgs_path='{}/test_images'.format(DATA_PATH),
df=test,
batch_size=1,
mode='predict',
shuffle=False,
aug=None,
seq_len=SEQ_LEN,
img_size=IMG_SIZE,
n_classes=6 
)
 preds = model.predict_generator(subm_datagen)
 print('Total preds test:', len(preds))
else:
print('Inference on validation images...')
 preds = model.predict_generator(val_datagen.__getitem__(1)) [:len(preds)]<save_to_csv>
p,t = learn.get_preds()
auc_score(p,t )
test['isup_grade'] = np.argmax(preds, axis=1)
 test.drop('data_provider',inplace= True, axis=1)
 test.to_csv('submission.csv', index=False)
 test.head()<set_options>
interp = ClassificationInterpretation.from_learner(learn)
 losses,idxs = interp.top_losses()
warnings.simplefilter(action='ignore', category=FutureWarning)
<load_from_csv>
p,t = learn.get_preds(ds_type=DatasetType.Test )
new_transactions = pd.read_csv('.. /input/elo-merchant-category-recommendation/new_merchant_transactions.csv', parse_dates=['purchase_date'])
 historical_transactions = pd.read_csv('.. /input/elo-merchant-category-recommendation/historical_transactions.csv', parse_dates=['purchase_date'])
 def binarize(df):
for col in ['authorized_flag', 'category_1']:
df[col] = df[col].map({'Y':1, 'N':0})
 return df
historical_transactions = binarize(historical_transactions)
 new_transactions = binarize(new_transactions )<load_from_csv>
sub = pd.DataFrame(np.stack([ids, p[:,1]], axis=1), columns=df_sample.columns )
%%time
def read_data(input_file):
df = pd.read_csv(input_file)
 df['first_active_month'] = pd.to_datetime(df['first_active_month'])
 df['elapsed_time'] =(datetime.date(2018, 2, 1)- df['first_active_month'].dt.date ).dt.days
return df
train = read_data('.. /input/elo-merchant-category-recommendation/train.csv')
 test = read_data('.. /input/elo-merchant-category-recommendation/test.csv')
 target = train['target']
del train['target']
gc.collect()<categorify>
sub.to_csv(work_dir/'sub.csv', index=False )