Changes to almost all of the pipeline:

- Added a different type of preprocessing which multiplies the data values by a constant – giving a human readable label, which is reversible for the calculation of metrics.
- Created an eval and metric calculation stage that logs test metrics
- Added eval stage to dvc.yaml and added metrics
- Added logs folder with logged metrics and params

TODO:
- Add params.yaml and const.py with relevant parameter and constants to reduce room for human error
- Train the model for longer and see if results improve.

dvc.yaml

@@ -15,6 +15,9 @@ stages:
     - src/data/processed/train
     outs:
     - src/models/
+    metrics:
+    - logs/train_metrics.csv:
+        cache: false
   eval:
     cmd: python3 src/code/eval.py src/data/processed/test
     deps:
@@ -22,4 +25,7 @@ stages:
     - src/models/model.pth
     - src/data/processed/test
     outs:
-    - src/eval/
+    - src/eval/
+    metrics:
+    - logs/test_metrics.csv:
+        cache: false

src/code/eval.py

@@ -1,36 +1,8 @@
 import sys
-from fastai.vision.all import unet_learner, Path, resnet34, MSELossFlat, get_files, L
-import torch
+from fastai.vision.all import unet_learner, Path, resnet34, MSELossFlat, get_files, L, tuplify
 from src.code.custom_data_loading import create_data
-from dagshub.fastai import DAGsHubLogger
-
-
-def compute_errors(targ, pred):
-    thresh = torch.max((targ / pred), (pred / targ)).numpy()
-    a1 = (thresh < 1.25).mean()
-    a2 = (thresh < 1.25 ** 2).mean()
-    a3 = (thresh < 1.25 ** 3).mean()
-
-    abs_rel = (torch.abs(targ - pred) / targ).mean().item()
-    sq_rel = torch.mean(((targ - pred).pow(2)) / targ).item()
-
-    rmse = torch.sqrt((targ - pred).pow(2).mean()).item()
-
-    rmse_log = torch.sqrt((torch.log(1 + targ) - torch.log(1 + pred)).pow(2).mean()).item()
-
-    err = torch.log(1 + pred) - torch.log(1 + targ)
-    silog = torch.sqrt(torch.mean(err.pow(2)) - torch.mean(err).pow(2)).item() * 100
-
-    log_10 = (torch.abs(torch.log10(1 + targ) - torch.log10(1 + pred))).mean().item()
-    return dict(a1=a1,
-                a2=a2,
-                a3=a3,
-                abs_rel=abs_rel,
-                sq_rel=sq_rel,
-                rmse=rmse,
-                rmse_log=rmse_log,
-                silog=silog,
-                log_10=log_10)
+from src.code.eval_metric_calculation import compute_eval_metrics
+from dagshub import dagshub_logger
 
 
 if __name__ == "__main__":
@@ -51,5 +23,21 @@ if __name__ == "__main__":
                            path='src/',
                            model_dir='models')
     learner = learner.load('model')
-    predictions, targets = learner.get_preds(dl=test_dl)
-    print(compute_errors(targets, predictions))
+    inputs, predictions, targets, decoded = learner.get_preds(dl=test_dl,
+                                                              with_input=True,
+                                                              with_decoded=True)
+    # FastAI magic to retrieve image values
+    inputs = (inputs,)
+    decoded_predictions = learner.dls.decode(inputs + tuplify(decoded))[1]
+    decoded_targets = learner.dls.decode(inputs + tuplify(targets))[1]
+
+    metrics = compute_eval_metrics(decoded_targets.numpy(), decoded_predictions.numpy())
+
+    with dagshub_logger(
+            metrics_path="logs/test_metrics.csv",
+            should_log_hparams=False
+    ) as logger:
+        # Metric logging
+        logger.log_metrics(metrics)
+
+    print("Evaluation Done!")

src/code/eval_metric_calculation.py

@@ -0,0 +1,71 @@
+import numpy as np
+
+
+def compute_errors(target, prediction):
+    thresh = np.maximum((target / prediction), (prediction / target))
+    a1 = (thresh < 1.25).mean()
+    a2 = (thresh < 1.25 ** 2).mean()
+    a3 = (thresh < 1.25 ** 3).mean()
+
+    abs_rel = np.mean(np.abs(target - prediction) / target)
+    sq_rel = np.mean(((target - prediction) ** 2) / target)
+
+    rmse = (target - prediction) ** 2
+    rmse = np.sqrt(rmse.mean())
+
+    rmse_log = (np.log(target) - np.log(prediction)) ** 2
+    rmse_log = np.sqrt(rmse_log.mean())
+
+    err = np.log(prediction) - np.log(target)
+    silog = np.sqrt(np.mean(err ** 2) - np.mean(err) ** 2) * 100
+
+    log_10 = (np.abs(np.log10(target) - np.log10(prediction))).mean()
+
+    return a1, a2, a3, abs_rel, sq_rel, rmse, rmse_log, silog, log_10
+
+
+def compute_eval_metrics(targets, predictions):
+    targets = targets / 25.0
+    predictions = predictions / 25.0
+
+    min_depth_eval = 1e-3
+    max_depth_eval = 10
+
+    num_samples = predictions.shape[0]
+
+    a1 = np.zeros(num_samples, np.float32)
+    a2 = np.zeros(num_samples, np.float32)
+    a3 = np.zeros(num_samples, np.float32)
+    abs_rel = np.zeros(num_samples, np.float32)
+    sq_rel = np.zeros(num_samples, np.float32)
+    rmse = np.zeros(num_samples, np.float32)
+    rmse_log = np.zeros(num_samples, np.float32)
+    silog = np.zeros(num_samples, np.float32)
+    log10 = np.zeros(num_samples, np.float32)
+
+    for i in range(num_samples):
+        target_depth = targets[i]
+        prediction_depth = predictions[i]
+
+        prediction_depth[prediction_depth < min_depth_eval] = min_depth_eval
+        prediction_depth[prediction_depth > max_depth_eval] = max_depth_eval
+        prediction_depth[np.isinf(prediction_depth)] = max_depth_eval
+
+        target_depth[np.isinf(target_depth)] = 0
+        target_depth[np.isnan(target_depth)] = 0
+
+        valid_mask = np.logical_and(target_depth > min_depth_eval, target_depth < max_depth_eval)
+
+        a1[i], a2[i], a3[i], abs_rel[i], sq_rel[i], rmse[i], rmse_log[i], silog[i], log10[i] = \
+            compute_errors(target_depth[valid_mask], prediction_depth[valid_mask])
+
+    print("{:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}, {:>7}".format(
+        'd1', 'd2', 'd3', 'AbsRel', 'SqRel', 'RMSE', 'RMSElog', 'SILog', 'log10'))
+    print("{:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}, {:7.3f}".format(
+        a1.mean(), a2.mean(), a3.mean(),
+        abs_rel.mean(), sq_rel.mean(), rmse.mean(), rmse_log.mean(), silog.mean(), log10.mean()))
+
+    return dict(a1=a1.mean(), a2=a2.mean(), a3=a3.mean(),
+                abs_rel=abs_rel.mean(), sq_rel=sq_rel.mean(),
+                rmse=rmse.mean(), rmse_log=rmse_log.mean(),
+                log10=log10.mean(), silog=silog.mean())

src/code/make_dataset.py

@@ -65,8 +65,9 @@ def convert_image(index, depth_map, img, output_folder):
     """
 
     # Normalize the depth image
-    normalized_depth = cv2.normalize(depth_map, None, 0, 255, cv2.NORM_MINMAX)
-    cv2.imwrite("%s/%05d_depth.png" % (output_folder, index), normalized_depth)
+    # normalized_depth = cv2.normalize(depth_map, None, 0, 255, cv2.NORM_MINMAX)
+    img_depth = depth_map * 25.0
+    cv2.imwrite("%s/%05d_depth.png" % (output_folder, index), img_depth)
 
     # Adding black frame to original image
     img = img[:, :, ::-1]  # Flipping the image from RGB to BGR for opencv

src/code/training.py

@@ -25,8 +25,8 @@ if __name__ == "__main__":
                            path='src/',
                            model_dir='models',
                            cbs=DAGsHubLogger(
-                               metrics_path="train_metrics.csv",
-                               hparams_path="train_params.yml"
+                               metrics_path="logs/train_metrics.csv",
+                               hparams_path="logs/train_params.yml"
                            ))
 
     print("Training model...")