Repository: Lightning-Universe/lightning-flash

Issue:
VideoClassificationData.from_files() does not work
## 🐛 Bug
`VideoClassificationData.from_files()` does not work. `VideoClassificationData` objects can only be constructed using the `from_folders()` classmethod - which unfortunately doesn't work for multilabel tasks :)

### To Reproduce
I wrote a Colab notebook to reproduce this in a self-contained environment: https://colab.research.google.com/drive/1X7UvZDndCc0dzcUZ_fGdmQz0ZMTTsj_U?usp=sharing

#### Code sample
See link to colab above

### Expected behavior
A `VideoClassificationData` object should be constructed from a list of paths and their labels


Discussion:
Dear @Darktex,

It should be too hard to support. Would you be interested to give it a try and open a PR ? I would help you if needed 

Best,
T.C
Of course! I tried for a bit on my own, but I wasn't able to track down the root cause - lots of delegation going on :(

I think I'd need some pointers to unblock

Resolving patch:
diff --git a/flash/video/classification/data.py b/flash/video/classification/data.py
--- a/flash/video/classification/data.py
+++ b/flash/video/classification/data.py
@@ -54,6 +54,8 @@
 
 _PYTORCHVIDEO_DATA = Dict[str, Union[str, torch.Tensor, int, float, List]]
 
+Label = Union[int, List[int]]
+
 
 class BaseVideoClassification:
     def __init__(
@@ -150,6 +152,76 @@ def _make_encoded_video_dataset(self, data) -> "LabeledVideoDataset":
         return ds
 
 
+class VideoClassificationListDataSource(BaseVideoClassification, PathsDataSource):
+    def __init__(
+        self,
+        clip_sampler: "ClipSampler",
+        video_sampler: Type[Sampler] = torch.utils.data.RandomSampler,
+        decode_audio: bool = True,
+        decoder: str = "pyav",
+    ):
+        super().__init__(
+            clip_sampler,
+            video_sampler=video_sampler,
+            decode_audio=decode_audio,
+            decoder=decoder,
+        )
+        PathsDataSource.__init__(
+            self,
+            extensions=("mp4", "avi"),
+        )
+
+    def _to_multi_hot(self, label_list: List[int]) -> torch.Tensor:
+        v = torch.zeros(len(self.labels_set))
+        for label in label_list:
+            v[label] = 1
+        return v
+
+    def _make_encoded_video_dataset(self, data) -> "LabeledVideoDataset":
+        [paths, str_labels] = data
+        self.is_multilabel = any(isinstance(label, list) for label in str_labels)
+        if self.is_multilabel:
+            self.labels_set = {label for label_list in str_labels for label in label_list}
+            self.label_to_id = {label: i for i, label in enumerate(sorted(self.labels_set))}
+            self.id_to_label = {i: label for label, i in self.label_to_id.items()}
+
+            encoded_labels = [
+                self._to_multi_hot([self.label_to_id[classname] for classname in label_list])
+                for label_list in str_labels
+            ]
+
+            data = list(
+                zip(
+                    paths,
+                    encoded_labels,
+                )
+            )
+        else:
+            self.labels_set = set(str_labels)
+            self.label_to_id = {label: i for i, label in enumerate(sorted(self.labels_set))}
+            self.id_to_label = {i: label for label, i in self.label_to_id.items()}
+            data = list(
+                zip(paths, [self.label_to_id[classname] for classname in str_labels])
+            )  # List[Lists] -> List[Tuples]
+        labeled_video_paths = LabeledVideoPaths(data)
+        ds = LabeledVideoDataset(
+            labeled_video_paths,
+            self.clip_sampler,
+            video_sampler=self.video_sampler,
+            decode_audio=self.decode_audio,
+            decoder=self.decoder,
+        )
+        return ds
+
+    def load_data(self, data: str, dataset: Optional[Any] = None) -> "LabeledVideoDataset":
+        ds = self._make_encoded_video_dataset(data)
+
+        if self.training:
+            self.set_state(LabelsState(self.id_to_label))
+            dataset.num_classes = len(self.labels_set)
+        return ds
+
+
 class VideoClassificationFiftyOneDataSource(
     BaseVideoClassification,
     FiftyOneDataSource,
@@ -238,7 +310,7 @@ def __init__(
             test_transform=test_transform,
             predict_transform=predict_transform,
             data_sources={
-                DefaultDataSources.FILES: VideoClassificationPathsDataSource(
+                DefaultDataSources.FILES: VideoClassificationListDataSource(
                     clip_sampler,
                     video_sampler=video_sampler,
                     decode_audio=decode_audio,


Test patch:
diff --git a/tests/video/classification/test_model.py b/tests/video/classification/test_model.py
--- a/tests/video/classification/test_model.py
+++ b/tests/video/classification/test_model.py
@@ -13,6 +13,7 @@
 # limitations under the License.
 import contextlib
 import os
+import random
 import re
 import tempfile
 from pathlib import Path
@@ -127,7 +128,7 @@ def mock_encoded_video_dataset_folder(tmpdir):
 
 
 @pytest.mark.skipif(not _VIDEO_TESTING, reason="PyTorchVideo isn't installed.")
-def test_video_classifier_finetune(tmpdir):
+def test_video_classifier_finetune_from_folders(tmpdir):
 
     with mock_encoded_video_dataset_file() as (
         mock_csv,
@@ -198,6 +199,93 @@ def test_video_classifier_finetune(tmpdir):
         trainer.finetune(model, datamodule=datamodule)
 
 
+@pytest.mark.skipif(not _VIDEO_TESTING, reason="PyTorchVideo isn't installed.")
+def test_video_classifier_finetune_from_files(tmpdir):
+
+    with mock_encoded_video_dataset_file() as (
+        mock_csv,
+        label_videos,
+        total_duration,
+    ):
+        label_names = ["label_1", "label_2", "label_3", "label_4"]
+
+        half_duration = total_duration / 2 - 1e-9
+
+        files = []
+        labels = []
+        with open(mock_csv) as fin:
+            for line in fin:
+                if not line:
+                    break
+                splits = line.split()
+                fname = splits[0]
+                label = label_names[random.randint(0, len(labels))]
+                files.append(fname)
+                labels.append(label)
+
+        datamodule = VideoClassificationData.from_files(
+            train_files=files,
+            train_targets=labels,
+            clip_sampler="uniform",
+            clip_duration=half_duration,
+            video_sampler=SequentialSampler,
+            decode_audio=False,
+        )
+
+        for sample in datamodule.train_dataset.data:
+            expected_t_shape = 5
+            assert sample["video"].shape[1] == expected_t_shape
+
+        assert len(VideoClassifier.available_backbones()) > 5
+
+        train_transform = {
+            "post_tensor_transform": Compose(
+                [
+                    ApplyTransformToKey(
+                        key="video",
+                        transform=Compose(
+                            [
+                                UniformTemporalSubsample(8),
+                                RandomShortSideScale(min_size=256, max_size=320),
+                                RandomCrop(244),
+                                RandomHorizontalFlip(p=0.5),
+                            ]
+                        ),
+                    ),
+                ]
+            ),
+            "per_batch_transform_on_device": Compose(
+                [
+                    ApplyTransformToKey(
+                        key="video",
+                        transform=K.VideoSequential(
+                            K.Normalize(torch.tensor([0.45, 0.45, 0.45]), torch.tensor([0.225, 0.225, 0.225])),
+                            K.augmentation.ColorJitter(0.1, 0.1, 0.1, 0.1, p=1.0),
+                            data_format="BCTHW",
+                            same_on_frame=False,
+                        ),
+                    ),
+                ]
+            ),
+        }
+
+        datamodule = VideoClassificationData.from_files(
+            train_files=files,
+            train_targets=labels,
+            clip_sampler="uniform",
+            clip_duration=half_duration,
+            video_sampler=SequentialSampler,
+            decode_audio=False,
+            train_transform=train_transform,
+        )
+
+        model = VideoClassifier(num_classes=datamodule.num_classes, pretrained=False, backbone="slow_r50")
+
+        trainer = flash.Trainer(fast_dev_run=True, gpus=torch.cuda.device_count())
+
+        trainer.finetune(model, datamodule=datamodule)
+
+
 @pytest.mark.skipif(not _VIDEO_AVAILABLE, reason="PyTorchVideo isn't installed.")
 @pytest.mark.skipif(not _FIFTYONE_AVAILABLE, reason="fiftyone isn't installed.")
 def test_video_classifier_finetune_fiftyone(tmpdir):