Preparing logging for training and evaluation, and adding evaluation step [WIP]

dvc.lock

@@ -3,8 +3,8 @@ process_data:
     src/data/processed
   deps:
   - path: src/code/make_dataset.py
-    md5: fd5076d53909a47ce3b6598c26af6c97
-    size: 3783
+    md5: 3a0dee3a1ba9c587b8ca6ea6f0447ada
+    size: 5227
   - path: src/data/raw/nyu_depth_v2_labeled.mat
     md5: 520609c519fba3ba5ac58c8fefcc3530
     size: 2972037809
@@ -13,8 +13,8 @@ process_data:
     size: 2626
   outs:
   - path: src/data/processed/
-    md5: d98a9647a37ab431bfa35815eb4afda0.dir
-    size: 232903470
+    md5: 38e5a1b51dfd29c0f313779b2f3d4540.dir
+    size: 234696354
     nfiles: 2898
 train:
   cmd: python3 src/code/training.py src/data/processed

dvc.yaml

@@ -9,9 +9,17 @@ stages:
     outs:
     - src/data/processed/
   train:
-    cmd: python3 src/code/training.py src/data/processed
+    cmd: python3 src/code/training.py src/data/processed/train
     deps:
     - src/code/training.py
-    - src/data/processed/
+    - src/data/processed/train
     outs:
     - src/models/
+  eval:
+    cmd: python3 src/code/eval.py src/data/processed/test
+    deps:
+    - src/code/eval.py
+    - src/models/model.pth
+    - src/data/processed/test
+    outs:
+    - src/eval/

requirements.txt

@@ -1,8 +1,9 @@
-dvc==1.10.1
-fastai==2.1.5
+dvc==1.11.15
+fastai==2.2.5
 torch==1.7.0
 h5py==2.10.0
 opencv-python==4.4.0.42
 tqdm==4.52.0
 numpy==1.19.4
-scikit-learn==0.23.2
+scikit-learn==0.23.2
+dagshub==0.1.5

src/code/custom_data_loading.py

@@ -0,0 +1,47 @@
+from fastai.vision.all import \
+    DataLoaders, \
+    delegates, \
+    DataBlock, \
+    ImageBlock, \
+    PILImage, \
+    PILImageBW, \
+    RandomSplitter, \
+    Path, \
+    get_files
+
+
+class ImageImageDataLoaders(DataLoaders):
+    """Basic wrapper around several `DataLoader`s with factory methods for Image to Image problems"""
+    @classmethod
+    @delegates(DataLoaders.from_dblock)
+    def from_label_func(cls, path, filenames, label_func, is_test, valid_pct=0.2, seed=None, item_transforms=None,
+                        batch_transforms=None, **kwargs):
+        """Create from list of `fnames` in `path`s with `label_func`."""
+        datablock = DataBlock(blocks=(ImageBlock(cls=PILImage), ImageBlock(cls=PILImageBW)),
+                              get_y=label_func,
+                              item_tfms=item_transforms,
+                              batch_tfms=batch_transforms)
+        if not is_test:
+            datablock.splitter = RandomSplitter(valid_pct, seed=seed)
+        res = cls.from_dblock(datablock, filenames, path=path, **kwargs)
+        return res
+
+
+def get_y_fn(x):
+    y = str(x.absolute()).replace('.jpg', '_depth.png')
+    y = Path(y)
+
+    return y
+
+
+def create_data(data_path, is_test=False):
+    filenames = get_files(data_path, extensions='.jpg')
+    if len(filenames) == 0:
+        raise ValueError("Could not find any files in the given path")
+    dataset = ImageImageDataLoaders.from_label_func(data_path,
+                                                    is_test=is_test,
+                                                    seed=42,
+                                                    bs=4, num_workers=0,
+                                                    filenames=filenames,
+                                                    label_func=get_y_fn)
+    return dataset

src/code/eval.py

@@ -0,0 +1,52 @@
+import sys
+from fastai.vision.all import unet_learner, Path, resnet34, MSELossFlat
+import torch
+from src.code.custom_data_loading import create_data
+from dagshub.fastai import DAGsHubLogger
+
+
+def compute_errors(targ, pred):
+    thresh = torch.max((targ / pred), (pred / targ)).numpy()
+    a1 = (thresh < 1.25).mean()
+    a2 = (thresh < 1.25 ** 2).mean()
+    a3 = (thresh < 1.25 ** 3).mean()
+
+    abs_rel = (torch.abs(targ - pred) / targ).mean().item()
+    sq_rel = torch.mean(((targ - pred).pow(2)) / targ).item()
+
+    rmse = torch.sqrt((targ - pred).pow(2).mean()).item()
+
+    rmse_log = torch.sqrt((torch.log(1 + targ) - torch.log(1 + pred)).pow(2).mean()).item()
+
+    err = torch.log(1 + pred) - torch.log(1 + targ)
+    silog = torch.sqrt(torch.mean(err.pow(2)) - torch.mean(err).pow(2)).item() * 100
+
+    log_10 = (torch.abs(torch.log10(1 + targ) - torch.log10(1 + pred))).mean().item()
+    return dict(a1=a1,
+                a2=a2,
+                a3=a3,
+                abs_rel=abs_rel,
+                sq_rel=sq_rel,
+                rmse=rmse,
+                rmse_log=rmse_log,
+                silog=silog,
+                log_10=log_10)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("usage: %s <test_data_path>" % sys.argv[0], file=sys.stderr)
+        sys.exit(0)
+
+    data_path = Path(sys.argv[1])
+    data = create_data(data_path, is_test=True)
+
+    learner = unet_learner(data,
+                           resnet34,
+                           n_out=3,
+                           loss_func=MSELossFlat(),
+                           path='src/',
+                           model_dir='models')
+    learner = learner.load('model')
+    predictions, targets = learner.get_preds()
+    print(compute_errors(targets, predictions))

src/code/make_dataset.py

@@ -26,11 +26,21 @@
 # SOFTWARE.
 #######################################################################################
 #
-# Helper script to convert the NYU Depth v2 dataset Matlab file into a set of
-# PNG and JPEG images.
-#
 # See https://github.com/deeplearningais/curfil/wiki/Training-and-Prediction-with-the-NYU-Depth-v2-Dataset
 
+
+"""Helper script to convert the NYU Depth v2 dataset Matlab file into a set of PNG and JPEG images.
+Receives 3 Files from argparse:
+<h5_file> - Contains the original images, depths maps, and scene types
+<train_test_split> - contains two numpy arrays with the index of the
+                    images based on the split to train and test sets.
+<out_folder> - Name of the folder to save the original and depth images.
+
+Every image in the DB will have it's twine B&W image that indicates the depth
+in the image. the images will be read, converted by the convert_image function
+and finally saved to path based on train test split and Scene types.
+"""
+
 from __future__ import print_function
 
 import h5py
@@ -42,41 +52,57 @@ import cv2
 from tqdm import tqdm
 
 
-def convert_image(i, scene, depth, image, folder):
-    # depth is given in meters (Kinect has a range of around .5m and 4.5m but can sense also at 8m)
-    normalized_depth = cv2.normalize(depth,  None, 0, 255, cv2.NORM_MINMAX)
-    cv2.imwrite("%s/%05d_depth.png" % (folder, i), normalized_depth)
+def convert_image(index, depth_map, img, output_folder):
+    """Processes data images and depth maps
+    :param index: int, image index
+    :param depth_map: numpy array, image depth - 2D array.
+    :param img: numpy array, the original RGB image - 3D array.
+    :param output_folder: path to save the image in.
 
-    image = image[:, :, ::-1]
-    image_black_boundary = np.zeros((480, 640, 3), dtype=np.uint8)
-    image_black_boundary[7:474, 7:632, :] = image[7:474, 7:632, :]
-    cv2.imwrite("%s/%05d.jpg" % (folder, i), image_black_boundary)
+    Receives an image with it's relevant depth map.
+    Normalizes the depth map, and adds a 7 px boundary to the original image.
+    Saves both image and depth map to the appropriate processed data folder.
+    """
+
+    # Normalize the depth image
+    normalized_depth = cv2.normalize(depth_map, None, 0, 255, cv2.NORM_MINMAX)
+    cv2.imwrite("%s/%05d_depth.png" % (output_folder, index), normalized_depth)
+
+    # Adding black frame to original image
+    img = img[:, :, ::-1]  # Flipping the image from RGB to BGR for opencv
+    image_black_boundary = np.zeros(img.shape, dtype=np.uint8)
+    image_black_boundary[7:image_black_boundary.shape[0] - 6, 7:image_black_boundary.shape[1] - 6, :] = \
+        img[7:img.shape[0] - 6, 7:img.shape[1] - 6, :]
+    cv2.imwrite("%s/%05d.jpg" % (output_folder, index), image_black_boundary)
 
 
 if __name__ == "__main__":
 
-    if len(sys.argv) < 4:
+    # Check if got all needed input for argparse
+    if len(sys.argv) != 4:
         print("usage: %s <h5_file> <train_test_split> <out_folder>" % sys.argv[0], file=sys.stderr)
         sys.exit(0)
 
+    # load arguments to variables
     h5_file = h5py.File(sys.argv[1], "r")
-    # h5py is not able to open that file. but scipy is
-    train_test = scipy.io.loadmat(sys.argv[2])
+    train_test = scipy.io.loadmat(sys.argv[2])  # h5py is not able to open that file. but scipy is
     out_folder = sys.argv[3]
 
+    # Extract images *indexes* for train and test data sets
     test_images = set([int(x) for x in train_test["testNdxs"]])
     train_images = set([int(x) for x in train_test["trainNdxs"]])
     print("%d training images" % len(train_images))
     print("%d test images" % len(test_images))
 
+    # Grayscale
     depth = h5_file['depths']
+    print("Reading", sys.argv[1])
+    images = h5_file['images']  # (num_channels, height, width)
 
-    print("reading", sys.argv[1])
-
-    images = h5_file['images']
+    # Extract all sceneTypes per image - "office", "classroom", etc.
     scenes = [u''.join(chr(c[0]) for c in h5_file[obj_ref]) for obj_ref in h5_file['sceneTypes'][0]]
 
-    for i, image in tqdm(enumerate(images), desc="processing images", total=len(images)):
+    for i, image in tqdm(enumerate(images), desc="Processing images", total=len(images)):
         idx = int(i) + 1
         if idx in train_images:
             train_test = "train"
@@ -84,9 +110,11 @@ if __name__ == "__main__":
             assert idx in test_images, "index %d neither found in training set nor in test set" % idx
             train_test = "test"
 
+        # Create path to save image in
         folder = "%s/%s/%s" % (out_folder, train_test, scenes[i])
         if not os.path.exists(folder):
             os.makedirs(folder)
-        convert_image(i, scenes[i], depth[i, :, :].T, image.T, folder)
+
+        convert_image(i, depth[i, :, :].T, image.T, folder)
 
     print("Finished")

src/code/training.py

@@ -1,49 +1,36 @@
-import torch
+"""Trains or fine-tunes a model for the task of monocular depth estimation
+Receives 1 arguments from argparse:
+  <data_path> - Path to the dataset which is split into 2 folders - train and test.
+"""
 import sys
-from fastai.vision.all import *
-from torchvision.utils import save_image
-
-
-class ImageImageDataLoaders(DataLoaders):
-    "Basic wrapper around several `DataLoader`s with factory methods for Image to Image problems"
-
-    @classmethod
-    @delegates(DataLoaders.from_dblock)
-    def from_label_func(cls, path, fnames, label_func, valid_pct=0.2, seed=None, item_tfms=None,
-                        batch_tfms=None, **kwargs):
-        "Create from list of `fnames` in `path`s with `label_func`."
-        dblock = DataBlock(blocks=(ImageBlock(cls=PILImage), ImageBlock(cls=PILImageBW)),
-                           splitter=RandomSplitter(valid_pct, seed=seed),
-                           get_y=label_func,
-                           item_tfms=item_tfms,
-                           batch_tfms=batch_tfms)
-        res = cls.from_dblock(dblock, fnames, path=path, **kwargs)
-        return res
-
-
-def get_y_fn(x):
-    y = str(x.absolute()).replace('.jpg', '_depth.png')
-    y = Path(y)
-
-    return y
-
-
-def create_data(data_path):
-    fnames = get_files(data_path / 'train', extensions='.jpg')
-    data = ImageImageDataLoaders.from_label_func(data_path / 'train', seed=42, bs=4, num_workers=0,
-                                                 fnames=fnames, label_func=get_y_fn)
-    return data
+from fastai.vision.all import unet_learner, Path, resnet34, rmse, MSELossFlat
+from src.code.custom_data_loading import create_data
+from dagshub.fastai import DAGsHubLogger
 
 
 if __name__ == "__main__":
-    if len(sys.argv) < 2:
+    # Check if got all needed input for argparse
+    if len(sys.argv) != 2:
         print("usage: %s <data_path>" % sys.argv[0], file=sys.stderr)
         sys.exit(0)
 
     data = create_data(Path(sys.argv[1]))
-    learner = unet_learner(data, resnet34, metrics=rmse, wd=1e-2, n_out=3, loss_func=MSELossFlat(),
-                           path='src/test/')
+    wd, lr, ep = 1e-2, 1e-3, 1
+    learner = unet_learner(data,
+                           resnet34,
+                           metrics=rmse,
+                           wd=wd,
+                           n_out=3,
+                           loss_func=MSELossFlat(),
+                           path='src/',
+                           model_dir='models',
+                           cbs=DAGsHubLogger(
+                               metrics_path="train_metrics.csv",
+                               hparams_path="train_params.yml"
+                           ))
+
     print("Training model...")
-    learner.fine_tune(1)
+    learner.fine_tune(epochs=ep, base_lr=lr)
     print("Saving model...")
     learner.save('model')
+    print("Done!")

src/data/raw/nyu_depth_v2_labeled.mat.dvc

@@ -6,3 +6,4 @@ deps:
 outs:
 - md5: 520609c519fba3ba5ac58c8fefcc3530
   path: nyu_depth_v2_labeled.mat
+  size: 2972037809

src/data/raw/splits.mat.dvc

@@ -6,3 +6,4 @@ deps:
 outs:
 - md5: 08e3c3aea27130ac7c01ffd739a4535f
   path: splits.mat
+  size: 2626