Kaggle competition: Digit Recognizer
<SOS> metric: categorizationaccuracy Kaggle data source: digit-recognizer<load_from_csv>
%matplotlib inline
np.random.seed(2)
 sns.set(style = 'white', context= 'notebook', palette = 'deep' )
print("torch", torch.__version__)
 flags = Flags().update(flags_dict)
 print("flags", flags)
 debug = flags.debug
outdir = Path(flags.outdir)
 os.makedirs(str(outdir), exist_ok=True)
 flags_dict = dataclasses.asdict(flags)
 save_yaml(str(outdir / "flags.yaml"), flags_dict)
 inputdir = Path("/kaggle/input")
 datadir = inputdir / "vinbigdata-chest-xray-abnormalities-detection"
imgdir = inputdir / flags.imgdir_name
train = pd.read_csv(datadir / "train.csv")
<groupby>
train = pd.read_csv('.. /input/digit-recognizer/train.csv')
 test = pd.read_csv('.. /input/digit-recognizer/test.csv' )
is_normal_df = train.groupby("image_id")["class_id"].agg(lambda s:(s == 14 ).sum() ).reset_index().rename({"class_id": "num_normal_annotations"}, axis=1)
 is_normal_df.head()<categorify>
Y_train = train['label']
X_train = train.drop(labels = ['label'], axis = 1)
 del train
g = sns.countplot(Y_train)
 g
num_normal_anno_counts_df = num_normal_anno_counts.reset_index()
num_normal_anno_counts_df["name"] = num_normal_anno_counts_df["index"].map({0: "Abnormal", 3: "Normal"})
 num_normal_anno_counts_df<define_variables>
X_train = X_train/255.0
test = test/255.0
def get_vinbigdata_dicts(
 imgdir: Path,
train_df: pd.DataFrame,
train_data_type: str = "original",
use_cache: bool = True,
debug: bool = True,
target_indices: Optional[np.ndarray] = None, 
):
debug_str = f"_debug{int(debug)}"
train_data_type_str = f"_{train_data_type}"
cache_path = Path(".")/ f"dataset_dicts_cache{train_data_type_str}{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 train_meta = pd.read_csv(imgdir / "train_meta.csv")
 if debug:
train_meta = train_meta.iloc[:500]
image_id = train_meta.loc[0, "image_id"]
image_path = str(imgdir / "train" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, train_meta_row in tqdm(train_meta.iterrows() , total=len(train_meta)) :
record = {}
image_id, height, width = train_meta_row.values
filename = str(imgdir / "train" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
objs = []
for index2, row in train_df.query("image_id == @image_id" ).iterrows() :
class_id = row["class_id"]
if class_id == 14:
pass
else:
h_ratio = resized_height / height
w_ratio = resized_width / width
bbox_resized = [
int(row["x_min"])* w_ratio,
int(row["y_min"])* h_ratio,
int(row["x_max"])* w_ratio,
int(row["y_max"])* h_ratio,
]
obj = {
"bbox": bbox_resized,
"bbox_mode": BoxMode.XYXY_ABS,
"category_id": class_id,
}
objs.append(obj)
 record["annotations"] = objs
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 if target_indices is not None:
dataset_dicts = [dataset_dicts[i] for i in target_indices]
return dataset_dicts
def get_vinbigdata_dicts_test(
 imgdir: Path, test_meta: pd.DataFrame, use_cache: bool = True, debug: bool = True, 
):
debug_str = f"_debug{int(debug)}"
cache_path = Path(".")/ f"dataset_dicts_cache_test{debug_str}.pkl"
if not use_cache or not cache_path.exists() :
print("Creating data...")
 if debug:
test_meta = test_meta.iloc[:500]
image_id = test_meta.loc[0, "image_id"]
image_path = str(imgdir / "test" / f"{image_id}.png")
 image = cv2.imread(image_path)
 resized_height, resized_width, ch = image.shape
print(f"image shape: {image.shape}")
 dataset_dicts = []
for index, test_meta_row in tqdm(test_meta.iterrows() , total=len(test_meta)) :
record = {}
image_id, height, width = test_meta_row.values
filename = str(imgdir / "test" / f"{image_id}.png")
 record["file_name"] = filename
record["image_id"] = image_id
record["height"] = resized_height
record["width"] = resized_width
dataset_dicts.append(record)
 with open(cache_path, mode="wb")as f:
pickle.dump(dataset_dicts, f)
 print(f"Load from cache {cache_path}")
 with open(cache_path, mode="rb")as f:
dataset_dicts = pickle.load(f)
 return dataset_dicts 
<categorify>
X_train = X_train.values.reshape(-1,28,28,1)
 test= test.values.reshape(-1,28,28,1 )
class DatasetMixin(Dataset):
def __init__(self, transform=None):
self.transform = transform
def __getitem__(self, index):

 if torch.is_tensor(index):
index = index.tolist()
if isinstance(index, slice):
current, stop, step = index.indices(len(self))
return [self.get_example_wrapper(i)for i in
six.moves.range(current, stop, step)]
elif isinstance(index, list)or isinstance(index, numpy.ndarray):
return [self.get_example_wrapper(i)for i in index]
else:
return self.get_example_wrapper(index)
 def __len__(self):

 raise NotImplementedError
def get_example_wrapper(self, i):

 example = self.get_example(i)
 if self.transform:
example = self.transform(example)
 return example
def get_example(self, i):

 raise NotImplementedError 
<categorify>
Y_train = to_categorical(Y_train,num_classes= 10 )
class VinbigdataTwoClassDataset(DatasetMixin):
def __init__(self, dataset_dicts, image_transform=None, transform=None, train: bool = True,
mixup_prob: float = -1.0, label_smoothing: float = 0.0):
super(VinbigdataTwoClassDataset, self ).__init__(transform=transform)
 self.dataset_dicts = dataset_dicts
self.image_transform = image_transform
self.train = train
self.mixup_prob = mixup_prob
self.label_smoothing = label_smoothing
def _get_single_example(self, i):
d = self.dataset_dicts[i]
filename = d["file_name"]
img = cv2.imread(filename)
 if self.image_transform:
img = self.image_transform(img)
 img = torch.tensor(np.transpose(img,(2, 0, 1)).astype(np.float32))
if self.train:
label = int(len(d["annotations"])> 0)
 if self.label_smoothing > 0:
if label == 0:
return img, float(label)+ self.label_smoothing
else:
return img, float(label)- self.label_smoothing
else:
return img, float(label)
 else:
return img, None
def get_example(self, i):
img, label = self._get_single_example(i)
 if self.mixup_prob > 0.and np.random.uniform() < self.mixup_prob:
j = np.random.randint(0, len(self.dataset_dicts))
p = np.random.uniform()
img2, label2 = self._get_single_example(j)
 img = img * p + img2 *(1 - p)
 if self.train:
label = label * p + label2 *(1 - p)
 if self.train:
label_logit = torch.tensor([1 - label, label], dtype=torch.float32)
 return img, label_logit
else:
return img
def __len__(self):
return len(self.dataset_dicts )<create_dataframe>
random_seed = 2
dataset_dicts = get_vinbigdata_dicts(imgdir, train, debug=debug)
 dataset = VinbigdataTwoClassDataset(dataset_dicts )<normalization>
X_train, X_val, Y_train, Y_val = \
train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed )
class Transform:
def __init__(
 self, hflip_prob: float = 0.5, ssr_prob: float = 0.5, random_bc_prob: float = 0.5 
):
self.transform = A.Compose(
 [
A.HorizontalFlip(p=hflip_prob),
A.ShiftScaleRotate(
 shift_limit=0.0625, scale_limit=0.1, rotate_limit=10, p=ssr_prob 
),
A.RandomBrightnessContrast(p=random_bc_prob),
] 
)
 def __call__(self, image):
image = self.transform(image=image)["image"]
return image 
<create_dataframe>
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(BatchNormalization())
 model.add(AveragePooling2D(pool_size=(2,2)))
 model.add(Dropout(0.1))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(AveragePooling2D(pool_size=(2,2), strides=(2,2)))
 model.add(BatchNormalization())
 model.add(Dropout(0.1))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.25))
model.add(Dense(10, activation = "softmax"))
aug_dataset = VinbigdataTwoClassDataset(dataset_dicts, image_transform=Transform() )<categorify>
optimizer = Adam(
 learning_rate=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-07)
 learning_rate_reduction = ReduceLROnPlateau(monitor='val_Accuracy',
patience=3,
verbose=1,
factor=5e-7,
min_lr=0.00001 )
class Transform:
def __init__(self, aug_kwargs: Dict):
self.transform = A.Compose(
 [getattr(A, name )(**kwargs)for name, kwargs in aug_kwargs.items() ] 
)
 def __call__(self, image):
image = self.transform(image=image)["image"]
return image<init_hyperparams>
model.compile(optimizer = optimizer , loss = "categorical_crossentropy", metrics=["accuracy"])
 epochs = 20
batch_size = 128
class CNNFixedPredictor(nn.Module):
def __init__(self, cnn: nn.Module, num_classes: int = 2):
super(CNNFixedPredictor, self ).__init__()
self.cnn = cnn
self.lin = Linear(cnn.num_features, num_classes)
 print("cnn.num_features", cnn.num_features)
 for param in self.cnn.parameters() :
param.requires_grad = False
def forward(self, x):
feat = self.cnn(x)
 return self.lin(feat)
<choose_model_class>
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )
def build_predictor(model_name: str, model_mode: str = "normal"):
if model_mode == "normal":
return timm.create_model(model_name, pretrained=True, num_classes=2, in_chans=3)
 elif model_mode == "cnn_fixed":
timm_model = timm.create_model(model_name, pretrained=True, num_classes=0, in_chans=3)
 return CNNFixedPredictor(timm_model, num_classes=2)
 else:
raise ValueError(f"[ERROR] Unexpected value model_mode={model_mode}")
<compute_test_metric>
history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size, callbacks=[learning_rate_reduction]
)
def accuracy(y: torch.Tensor, t: torch.Tensor)-> torch.Tensor:

 assert y.shape[:-1] == t.shape, f"y {y.shape}, t {t.shape} is inconsistent."
pred_label = torch.max(y.detach() , dim=-1)[1]
count = t.nelement()
correct =(pred_label == t ).sum().float()
acc = correct / count
return acc
def accuracy_with_logits(y: torch.Tensor, t: torch.Tensor)-> torch.Tensor:

 assert y.shape == t.shape
gt_label = torch.max(t.detach() , dim=-1)[1]
return accuracy(y, gt_label )<compute_train_metric>
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )
<find_best_params><EOS> 
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False)