Repository: lightly-ai/lightly

Issue:
Support subfolders of videos
Support subfolders of videos

The `LightlyDataset` should support subfolders of videos. The code should be adapted in the following places:


https://github.com/lightly-ai/lightly/blob/76aed5b954ffd1b9f3600e481d9288a5a7ccabd0/lightly/data/_helpers.py#L26

https://github.com/lightly-ai/lightly/blob/76aed5b954ffd1b9f3600e481d9288a5a7ccabd0/lightly/data/_video.py#L180


Discussion:

Resolving patch:
diff --git a/lightly/data/_helpers.py b/lightly/data/_helpers.py
--- a/lightly/data/_helpers.py
+++ b/lightly/data/_helpers.py
@@ -23,19 +23,39 @@
                     '.hevc', '.m4v', '.webm', '.mpeg')
 
 
-def _contains_videos(root: str, extensions: tuple):
+def _dir_contains_videos(root: str, extensions: tuple):
     """Checks whether directory contains video files.
 
     Args:
         root: Root directory path.
 
     Returns:
-        True if root contains subdirectories else false.
+        True if root contains video files.
+
     """
     with os.scandir(root) as scan_dir:
         return any(f.name.lower().endswith(extensions) for f in scan_dir)
 
 
+def _contains_videos(root: str, extensions: tuple):
+    """Checks whether directory or any subdirectory contains video files.
+
+    Iterates over all subdirectories of "root" recursively and returns True
+    if any of the subdirectories contains a file with a VIDEO_EXTENSION.
+
+    Args:
+        root: Root directory path.
+
+    Returns:
+        True if "root" or any subdir contains video files.
+
+    """
+    for subdir, _, _ in os.walk(root):
+        if _dir_contains_videos(subdir, extensions):
+            return True
+    return False
+
+
 def _is_lightly_output_dir(dirname: str):
     """Checks whether the directory is a lightly_output directory.
 
diff --git a/lightly/data/_video.py b/lightly/data/_video.py
--- a/lightly/data/_video.py
+++ b/lightly/data/_video.py
@@ -177,15 +177,16 @@ def _is_valid_file(filename):
 
     # find all video instances (no subdirectories)
     video_instances = []
-    for fname in os.listdir(directory):
+    for root, _, files in os.walk(directory):
 
-        # skip invalid files
-        if not _is_valid_file(fname):
-            continue
+        for fname in files:
+            # skip invalid files
+            if not _is_valid_file(fname):
+                continue
 
-        # keep track of valid files
-        path = os.path.join(directory, fname)
-        video_instances.append(path)
+            # keep track of valid files
+            path = os.path.join(root, fname)
+            video_instances.append(path)
 
     # get timestamps
     timestamps, fpss = [], []
diff --git a/lightly/data/dataset.py b/lightly/data/dataset.py
--- a/lightly/data/dataset.py
+++ b/lightly/data/dataset.py
@@ -291,13 +291,16 @@ def dump(self,
         for i, filename in zip(indices, filenames):
             _dump_image(self.dataset, output_dir, filename, i, fmt=format)
 
+
     def get_filepath_from_filename(self, filename: str, image: PIL.Image.Image = None):
         """Returns the filepath given the filename of the image
 
         There are three cases:
         - The dataset is a regular dataset with the images in the input dir.
-        - The dataset is a video dataset, thus the images have to be saved in a temporary folder.
-        - The dataset is a torch dataset, thus the images have to be saved in a temporary folder.
+        - The dataset is a video dataset, thus the images have to be saved in a
+        temporary folder.
+        - The dataset is a torch dataset, thus the images have to be saved in a
+        temporary folder.
         Args:
             filename:
                 The filename of the image
@@ -305,23 +308,32 @@ def get_filepath_from_filename(self, filename: str, image: PIL.Image.Image = Non
                 The image corresponding to the filename
 
         Returns:
-            The filename to the image, either the exiting one (case 1) or a newly created jpg (case 2, 3)
+            The filename to the image, either the existing one (case 1) or a
+            newly created jpg (case 2, 3)
 
         """
 
-        has_input_dir = hasattr(self, 'input_dir') and isinstance(self.input_dir, str)
+        has_input_dir = hasattr(self, 'input_dir') and \
+            isinstance(self.input_dir, str)
         if has_input_dir:
             path_to_image = os.path.join(self.input_dir, filename)
             if os.path.isfile(path_to_image):
-                # Case 1
+                # the file exists, return its filepath
                 return path_to_image
 
         if image is None:
-            raise ValueError("The parameter image must not be None for VideoDatasets and TorchDatasets")
+            raise ValueError(
+                'The parameter image must not be None for'
+                'VideoDatasets and TorchDatasets'
+            )
 
-        # Case 2 and 3
+        # the file doesn't exist, save it as a jpg and return filepath
         folder_path = tempfile.mkdtemp()
-        filepath = os.path.join(folder_path,filename) + '.jpg'
+        filepath = os.path.join(folder_path, filename) + '.jpg'
+        
+        if os.path.dirname(filepath):
+            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+
         image.save(filepath)
         return filepath
 


Test patch: