Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<categorify>
%matplotlib inline
np.random.seed(2)
 sns.set(style='white', context='notebook', palette='deep')
 for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
test = post_process1(test)
 test<predict_on_test>
train_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
 test_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv' )
test_image = test.groupby("image_id" ).first().reset_index()
if LOCAL:
pass
else:
test_loader = DataLoader(
 ImageDataset(test_image),
batch_size=BATCH_SIZE,
num_workers=MAX_THRE,
pin_memory=True, 
)
 models = []
for p in MODEL_PATHS_IMAGE:
model = get_model(p)
 model.eval()
models.append(model)
 preds = predict(models, test_loader, GPU)
 display(pd.crosstab(test_image["image_id"], preds.argmax(1)))
 test_image[COLS_TARGET] = preds
del models, model, test_loader
torch.cuda.empty_cache()
gc.collect()
display(test_image )<predict_on_test>
X = train_data.drop('label', axis=1)
 y = train_data['label']
if LOCAL:
pass
else:
test_loader = DataLoader(
 MyDataset(test, mode="test", w_mask=W_MASK),
batch_size=BATCH_SIZE,
num_workers=MAX_THRE,
pin_memory=True, 
)
 models = []
for p in MODEL_PATHS:
model = get_model(p)
 model.eval()
models.append(model)
 preds = predict_weighted(models, test_loader, GPU, WEIGHT_1ST_2ND)
 display(pd.crosstab(test["image_id"], preds.argmax(1)))
 test[COLS_TARGET] = preds
test = post_process2(test)
 test = post_process3(test, test_image)
 del models, model
torch.cuda.empty_cache()
gc.collect()
display(test)
<load_from_csv>
sns.countplot(x=y)
 print(y.value_counts().sort_index(ascending=True))
cols = sample_submission.columns
if COMPUTE_PUBLIC is False and COMPUTE_PRIVATE is False:
print("dryrun, replace with local computed file")
 test_2nd = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS[0])
 for p in TEST_LOCAL_COMPUTED_PATHS[1:]:
test_2nd.loc[:, COLS_TARGET] += pd.read_csv(p ).loc[:, COLS_TARGET]
test_2nd.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS)
 test_1st = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_1ST[0])
 for p in TEST_LOCAL_COMPUTED_PATHS_1ST[1:]:
test_1st.loc[:, COLS_TARGET] += pd.read_csv(p ).loc[:, COLS_TARGET]
test_1st.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_1ST)
 test = test_2nd.copy()
test_1st.iloc[:, 11] = 0
test.loc[:, COLS_TARGET] = test_1st.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[0] + test_2nd.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[1]
test_image = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_IMAGE[0])
 for p in TEST_LOCAL_COMPUTED_PATHS_IMAGE[1:]:
test_image.loc[:, COLS_TARGET] += pd.read_csv(p ).loc[:, COLS_TARGET]
test_image.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_IMAGE)
 test = post_process1(test)
 test = post_process2(test)
 test = post_process3(test, test_image)
 write_submission(test, fill_shortage=True)
 elif COMPUTE_PUBLIC is False and COMPUTE_PRIVATE is True:
print("only private")
 test_2nd = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS[0])
 for p in TEST_LOCAL_COMPUTED_PATHS[1:]:
test_2nd.loc[:, COLS_TARGET] += pd.read_csv(p ).loc[:, COLS_TARGET]
test_2nd.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS)
 test_1st = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_1ST[0])
 for p in TEST_LOCAL_COMPUTED_PATHS_1ST[1:]:
test_1st.loc[:, COLS_TARGET] += pd.read_csv(p ).loc[:, COLS_TARGET]
test_1st.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_1ST)
 test_local = test_2nd.copy()
test_1st.iloc[:, 11] = 0
test_local.loc[:, COLS_TARGET] = test_1st.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[0] + test_2nd.loc[:, COLS_TARGET] * RATE_OF_WEIGHT_1ST_2ND[1]
test_image_local = pd.read_csv(TEST_LOCAL_COMPUTED_PATHS_IMAGE[0])
 for p in TEST_LOCAL_COMPUTED_PATHS_IMAGE[1:]:
test_image_local.loc[:, COLS_TARGET] += pd.read_csv(p ).loc[:, COLS_TARGET]
test_image_local.loc[:, COLS_TARGET] /= len(TEST_LOCAL_COMPUTED_PATHS_IMAGE)
 test_local = post_process1(test_local)
 test_local = post_process2(test_local)
 test_local = post_process3(test_local, test_image_local)
 if len(sample_submission)== 559:
write_submission(test_local, fill_shortage=False)
 else:
test2 = pd.concat([test_local, test[test_local.columns]], ignore_index=True)
 write_submission(test2, fill_shortage=False)
 elif COMPUTE_PUBLIC is True and COMPUTE_PRIVATE is True:
print("full compute")
 write_submission(test, fill_shortage=False)
 elif COMPUTE_PUBLIC is True and COMPUTE_PRIVATE is False:
print("only public")
 write_submission(test, fill_shortage=True)
<load_from_csv>
X.isna().sum().value_counts()
if len(sample_submission)== 559:
sub = pd.read_csv("submission.csv")
 display(sub )<load_from_csv>
y.isna().sum()
if len(sample_submission)== 559:
sub = pd.read_csv("submission.csv")
 for index, row in sub.head(3 ).iterrows() :
image_id = row["ID"]
w = row["ImageWidth"]
h = row["ImageHeight"]
pred_strs = row["PredictionString"].split()
pred_strs = list(split_list(pred_strs, 3))
for i, pred in enumerate(pred_strs):
class_id, cnf, encoded_mask = pred
class_id = int(class_id)
 cnf = float(cnf)
 print(f"class_id:{class_id}, image_id:{image_id}, confidence:{cnf}")
 mask = decode_binary_mask(encoded_mask, w, h)
 print_masked_img(image_id, mask)
 if i == 9:
break 
<install_modules>
X = X / 255.0
X = X.values.reshape(-1, 28, 28, 1)
 y = to_categorical(y, num_classes = 10 )
!pip install -q.. /input/pycocotools/pycocotools-2.0-cp37-cp37m-linux_x86_64.whl<define_variables>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42 )
sys.path = [
'.. /input/smp20210127/pytorch-image-models-master/pytorch-image-models-master',
'.. /input/hpapytorchzoozip/pytorch_zoo-master/',
'.. /input/hpa-seg/HPA-Cell-Segmentation/hpacellseg',
'.. /input/hpafinal'
] + sys.path
warnings.filterwarnings("ignore" )<import_modules>
cnn_model = Sequential()
cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu', input_shape =(28,28,1)))
 cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu'))
cnn_model.add(MaxPool2D(pool_size=(2, 2)))
 cnn_model.add(Dropout(0.25))
cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu'))
cnn_model.add(Conv2D(filters = 32, kernel_size =(5, 5), padding = 'same', activation ='relu'))
cnn_model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
 cnn_model.add(Dropout(0.25))
cnn_model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'same', activation ='relu'))
cnn_model.add(Conv2D(filters = 64, kernel_size =(3, 3), padding = 'same', activation ='relu'))
cnn_model.add(MaxPool2D(pool_size=(2, 2), strides=(2, 2)))
 cnn_model.add(Dropout(0.25))
cnn_model.add(Flatten())
 cnn_model.add(Dense(256, activation = "relu"))
cnn_model.add(Dropout(0.5))
cnn_model.add(Dense(10, activation = "softmax"))
print(cnn_model.summary() )
remove_small_holes, remove_small_objects)
 device = torch.device('cuda' )<define_variables>
cnn_model.compile(
 optimizer = 'adam',
loss = "categorical_crossentropy",
metrics=["accuracy"]
)
seg_size = 512
seg_bs = 8388608 // seg_size ** 2
seg_TTA = 8
small_th_dict = {
2048: 500,
1024: 125,
512 : 32,
}
small_th = small_th_dict[seg_size]
mask_dir = 'test_mask_npz_fullsize_cell_mask'
model_dirs = [
'.. /input/bo-hpa-models',
'.. /input/bo-hpa-models-3d256',
'.. /input/hpa-models',
'.. /input/hpa-models-qishen',
]
TTA = {
'orig': 2,
'masked': 3,
'cells_128': 8,
'cells_256': 6,
'center_cells': 3,
}
n_ch = 4
num_classes = 19
image_size = 512
orig_mean = [239.93038613, 246.05603962, 250.16871503, 250.50623682]
data_dir = '.. /input/hpa-single-cell-image-classification/test/'
df_sub = pd.read_csv('.. /input/hpa-single-cell-image-classification/sample_submission.csv')
 df_sub = df_sub.head(10)if df_sub.shape[0] == 559 else df_sub
df_sub.shape<normalization>
early_stopping = keras.callbacks.EarlyStopping(
 patience=5,
min_delta=0.001,
restore_best_weights=True,
)
NORMALIZE = {"mean": [124 / 255, 117 / 255, 104 / 255], "std": [1 /(0.0167 * 255)] * 3}
def get_trans_seg(img, I, rev=False):
if I >= 4 and not rev:
img = img.transpose(2,3)
 if I % 4 == 0:
pass
elif I % 4 == 1:
img = img.flip(2)
 elif I % 4 == 2:
img = img.flip(3)
 elif I % 4 == 3:
img = img.flip(2 ).flip(3)
 if I >= 4 and rev:
img = img.transpose(2,3)
 return img
class CellSegmentator(object):

 def __init__(
 self,
nuclei_model=".. /input/hpa-seg/dpn_unet_nuclei_v1.pth",
cell_model=".. /input/hpa-seg/dpn_unet_cell_3ch_v1.pth",
scale_factor=1.0,
device="cuda",
multi_channel_model=True, 
):

 if device != "cuda" and device != "cpu" and "cuda" not in device:
raise ValueError(f"{device} is not a valid device(cuda/cpu)")
 if device != "cpu":
try:
assert torch.cuda.is_available()
except AssertionError:
print("No GPU found, using CPU.", file=sys.stderr)
 device = "cpu"
self.device = device
if isinstance(nuclei_model, str):
if not os.path.exists(nuclei_model):
print(
 f"Could not find {nuclei_model}.Downloading it now",
file=sys.stderr, 
)
 raise
nuclei_model = torch.load(
 nuclei_model, map_location=torch.device(self.device)
)
 if isinstance(nuclei_model, torch.nn.DataParallel)and device == "cpu":
nuclei_model = nuclei_model.module
self.nuclei_model = nuclei_model.to(self.device ).eval()
self.multi_channel_model = multi_channel_model
if isinstance(cell_model, str):
if not os.path.exists(cell_model):
print(
 f"Could not find {cell_model}.Downloading it now", file=sys.stderr 
)
 raise
cell_model = torch.load(cell_model, map_location=torch.device(self.device))
self.cell_model = cell_model.to(self.device ).eval()
self.scale_factor = scale_factor
def _image_conversion(self, images):

 microtubule_imgs, er_imgs, nuclei_imgs = images
if self.multi_channel_model:
if not isinstance(er_imgs, list):
raise ValueError("Please speicify the image path(s)for er channels!")
 else:
if not er_imgs is None:
raise ValueError(
 "second channel should be None for two channel model predition!" 
)
 if not isinstance(microtubule_imgs, list):
raise ValueError("The microtubule images should be a list")
 if not isinstance(nuclei_imgs, list):
raise ValueError("The microtubule images should be a list")
 if er_imgs:
if not len(microtubule_imgs)== len(er_imgs)== len(nuclei_imgs):
raise ValueError("The lists of images needs to be the same length")
 else:
if not len(microtubule_imgs)== len(nuclei_imgs):
raise ValueError("The lists of images needs to be the same length")
 if not all(isinstance(item, np.ndarray)for item in microtubule_imgs):
microtubule_imgs = [
os.path.expanduser(item)for _, item in enumerate(microtubule_imgs)
 ]
nuclei_imgs = [
os.path.expanduser(item)for _, item in enumerate(nuclei_imgs)
 ]
microtubule_imgs = list(
 map(lambda item: imageio.imread(item), microtubule_imgs)
)
 nuclei_imgs = list(map(lambda item: imageio.imread(item), nuclei_imgs))
if er_imgs:
er_imgs = [os.path.expanduser(item)for _, item in enumerate(er_imgs)]
er_imgs = list(map(lambda item: imageio.imread(item), er_imgs))
if not er_imgs:
er_imgs = [
np.zeros(item.shape, dtype=item.dtype)
 for _, item in enumerate(microtubule_imgs)
 ]
cell_imgs = list(
 map(
 lambda item: np.dstack(( item[0], item[1], item[2])) ,
list(zip(microtubule_imgs, er_imgs, nuclei_imgs)) , 
)
)
 return cell_imgs
def pred_nuclei(self, images):

 def _preprocess(image):
self.target_shape = image.shape
if len(image.shape)== 2:
image = np.dstack(( image, image, image))
image = transform.rescale(image, self.scale_factor, multichannel=True)
 nuc_image = np.dstack(( image[..., 2], image[..., 2], image[..., 2]))
nuc_image = nuc_image.transpose([2, 0, 1])
 return nuc_image
def _segment_helper(imgs):
with torch.no_grad() :
mean = torch.as_tensor(NORMALIZE["mean"], device=self.device)
 std = torch.as_tensor(NORMALIZE["std"], device=self.device)
 imgs = torch.tensor(imgs ).float()
imgs = imgs.to(self.device)
 imgs = imgs.sub_(mean[:, None, None] ).div_(std[:, None, None])
 imgs = torch.stack([get_trans_seg(self.nuclei_model(get_trans_seg(imgs, I)) , I, True ).softmax(1)for I in range(1)], 0 ).mean(0)
 return imgs
preprocessed_imgs = list(map(_preprocess, images))
bs = 24
predictions = []
for i in range(0, len(preprocessed_imgs), bs):
start = i
end = min(len(preprocessed_imgs), i+bs)
 x = preprocessed_imgs[start:end]
pred = _segment_helper(x ).cpu().numpy()
predictions.append(pred)
 predictions = list(np.concatenate(predictions, axis=0))
predictions = map(util.img_as_ubyte, predictions)
 predictions = list(map(self._restore_scaling_padding, predictions))
return predictions
def _restore_scaling_padding(self, n_prediction):

 n_prediction = n_prediction.transpose([1, 2, 0])
 if not self.scale_factor == 1:
n_prediction[..., 0] = 0
n_prediction = cv2.resize(
 n_prediction, 
(self.target_shape[0], self.target_shape[1]),
interpolation=cv2.INTER_AREA, 
)
 return n_prediction
def pred_cells(self, images, precombined=False):

 def _preprocess(image):
self.target_shape = image.shape
if not len(image.shape)== 3:
raise ValueError("image should has 3 channels")
 cell_image = transform.rescale(image, self.scale_factor, multichannel=True)
 cell_image = cell_image.transpose([2, 0, 1])
 return cell_image
def _segment_helper(imgs):
with torch.no_grad() :
mean = torch.as_tensor(NORMALIZE["mean"], device=self.device)
 std = torch.as_tensor(NORMALIZE["std"], device=self.device)
 imgs = torch.tensor(imgs ).float()
imgs = imgs.to(self.device)
 imgs = imgs.sub_(mean[:, None, None] ).div_(std[:, None, None])
 imgs = torch.stack([get_trans_seg(self.cell_model(get_trans_seg(imgs, I)) , I, True ).softmax(1)for I in range(seg_TTA)], 0 ).mean(0)
 return imgs
if not precombined:
images = self._image_conversion(images)
 preprocessed_imgs = list(map(_preprocess, images))
bs = 24
predictions = []
for i in range(0, len(preprocessed_imgs), bs):
start = i
end = min(len(preprocessed_imgs), i+bs)
 x = preprocessed_imgs[start:end]
pred = _segment_helper(x ).cpu().numpy()
predictions.append(pred)
 predictions = list(np.concatenate(predictions, axis=0))
predictions = map(self._restore_scaling_padding, predictions)
 predictions = list(map(util.img_as_ubyte, predictions))
return predictions
def __fill_holes(image):

 boundaries = segmentation.find_boundaries(image)
 image = np.multiply(image, np.invert(boundaries))
image = ndi.binary_fill_holes(image > 0)
 image = ndi.label(image)[0]
return image
def label_cell(nuclei_pred, cell_pred):

 def __wsh(
 mask_img,
threshold,
border_img,
seeds,
threshold_adjustment=0.35,
small_object_size_cutoff=10, 
):
img_copy = np.copy(mask_img)
 m = seeds * border_img
img_copy[m <= threshold + threshold_adjustment] = 0
img_copy[m > threshold + threshold_adjustment] = 1
img_copy = img_copy.astype(np.bool)
 img_copy = remove_small_objects(img_copy, small_object_size_cutoff ).astype(
 np.uint8 
)
 mask_img[mask_img <= threshold] = 0
mask_img[mask_img > threshold] = 1
mask_img = mask_img.astype(np.bool)
 mask_img = remove_small_holes(mask_img, 63)
 mask_img = remove_small_objects(mask_img, 1 ).astype(np.uint8)
 markers = ndi.label(img_copy, output=np.uint32)[0]
labeled_array = segmentation.watershed(
 mask_img, markers, mask=mask_img, watershed_line=True 
)
 return labeled_array
nuclei_label = __wsh(
 nuclei_pred[..., 2] / 255.0,
0.4,
1 -(nuclei_pred[..., 1] + cell_pred[..., 1])/ 255.0 > 0.05,
nuclei_pred[..., 2] / 255,
threshold_adjustment=-0.25,
small_object_size_cutoff=small_th, 
)
 nuclei_label = remove_small_objects(nuclei_label, 157)
 nuclei_label = measure.label(nuclei_label)
 threshold_value = max(0.22, filters.threshold_otsu(cell_pred[..., 2] / 255)* 0.5)
 cell_region = np.multiply(
 cell_pred[..., 2] / 255 > threshold_value,
np.invert(np.asarray(cell_pred[..., 1] / 255 > 0.05, dtype=np.int8)) , 
)
 sk = np.asarray(cell_region, dtype=np.int8)
 distance = np.clip(cell_pred[..., 2], 255 * threshold_value, cell_pred[..., 2])
 cell_label = segmentation.watershed(-distance, nuclei_label, mask=sk)
 cell_label = remove_small_objects(cell_label, 344 ).astype(np.uint8)
 selem = disk(2)
 cell_label = closing(cell_label, selem)
 cell_label = __fill_holes(cell_label)
 sk = np.asarray(
 np.add(
 np.asarray(cell_label > 0, dtype=np.int8),
np.asarray(cell_pred[..., 1] / 255 > 0.05, dtype=np.int8), 
)
 > 0,
dtype=np.int8, 
)
 cell_label = segmentation.watershed(-distance, cell_label, mask=sk)
 cell_label = __fill_holes(cell_label)
 cell_label = np.asarray(cell_label > 0, dtype=np.uint8)
 cell_label = measure.label(cell_label)
 cell_label = remove_small_objects(cell_label, 344)
 cell_label = measure.label(cell_label)
 cell_label = np.asarray(cell_label, dtype=np.uint16)
 nuclei_label = np.multiply(cell_label > 0, nuclei_label)> 0
nuclei_label = measure.label(nuclei_label)
 nuclei_label = remove_small_objects(nuclei_label, 157)
 nuclei_label = np.multiply(cell_label, nuclei_label > 0)
 return nuclei_label, cell_label<prepare_x_and_y>
datagen = ImageDataGenerator(
 rotation_range=10,
width_shift_range=0.2,
height_shift_range=0.2,
zoom_range = 0.1, 
)
 datagen.fit(X_train )
class HPADatasetSeg(Dataset):
def __init__(self, df, root='.. /input/hpa-single-cell-image-classification/test/'):
self.df = df.reset_index(drop=True)
 self.root = root
def __len__(self):
return len(self.df)
 def __getitem__(self, index):
row = self.df.loc[index]
r = os.path.join(self.root, f'{row.ID}_red.png')
 y = os.path.join(self.root, f'{row.ID}_yellow.png')
 b = os.path.join(self.root, f'{row.ID}_blue.png')
 r = cv2.imread(r, 0)
 y = cv2.imread(y, 0)
 b = cv2.imread(b, 0)
 target_shape = r.shape
gray_image = cv2.resize(b,(seg_size, seg_size))
rgb_image = cv2.resize(np.stack(( r, y, b), axis=2),(seg_size, seg_size))
return gray_image, rgb_image, target_shape, row.ID
def collate_fn(batch):
gray = []
rgb_image = []
target_shape = []
IDs = []
for data_point in batch:
gray.append(data_point[0])
 rgb_image.append(data_point[1])
 target_shape.append(data_point[2])
 IDs.append(data_point[3])
 return gray, rgb_image, target_shape, IDs
dataset_seg = HPADatasetSeg(df_sub)
 loader_seg = DataLoader(dataset_seg, batch_size=seg_bs, num_workers=2, collate_fn=collate_fn )<categorify>
history = cnn_model.fit_generator(
 datagen.flow(X_train,y_train, batch_size=64),
validation_data=(X_test, y_test),
steps_per_epoch=X_train.shape[0] // 64,
epochs=30,
callbacks=[early_stopping]
)
for gray, rgb, target_shapes, IDs in tqdm(loader_seg):
nuc_segmentations = cellsegmentor.pred_nuclei(gray)
 cell_segmentations = cellsegmentor.pred_cells(rgb, precombined=True)
 for data_id, target_shape, nuc_seg, cell_seg in zip(IDs, target_shapes, nuc_segmentations, cell_segmentations):
nuc, cell = label_cell(nuc_seg, cell_seg)
 np.savez_compressed(f'./{mask_dir}/{data_id}', cell.astype(np.uint8))
print('---- finish mask write ----' )<set_options>
test_data = test_data / 255.0
test_data = test_data.values.reshape(-1, 28, 28, 1)
 results = cnn_model.predict(test_data)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
del cellsegmentor
gc.collect()
torch.cuda.empty_cache()<set_options>
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )