Repository: Lightning-Universe/lightning-flash

Issue:
Error message when `filter_valid_files` finds invalid files.
Currently if you have invalid files they are silently dropped. If all are dropped then this will trigger a cryptic error. Instead we should give a good error message like:
```
Found invalid files: ..., ..., ..... The supported file extensions are: ...
```



Discussion:
I'd like to be assigned to this, please.
Hey @JustinGoheen thanks for offering to work on this! I'll assign you to the issue 😃 The offending function can be found here: https://github.com/Lightning-AI/lightning-flash/blob/9c6fa13e1bc7fd9879ae5aa37d364c2a4df1c63f/flash/core/data/utilities/paths.py#L136

Let us know if you have any questions!
@ethanwharris do you mind if I incorporate [rich](https://rich.readthedocs.io/en/stable/introduction.html).

I've done something similar in a [personal project](https://github.com/JustinGoheen/lightning-pod/blob/475b292928f87eadd0be6026ccbbcbf7f8473cc2/lightning_pod/cli/console.py#L18)
![image](https://user-images.githubusercontent.com/26209687/178286337-837bb3cb-7178-4fd6-ba0f-0c82d88d4088.png)

The above isn't an error message; but I'd like to use Rich for formatting the basic message output, and potentially a table - if that isn't too verbose of a message.

> rich is installed by one of the core requirements of lightning i.e. by installing lighting, a user will have installed rich, and there is no need to add a dependency to flash's base requirements to support rich.

Resolving patch:
diff --git a/flash/core/data/utilities/paths.py b/flash/core/data/utilities/paths.py
--- a/flash/core/data/utilities/paths.py
+++ b/flash/core/data/utilities/paths.py
@@ -14,6 +14,7 @@
 import os
 from typing import Any, Callable, cast, List, Optional, Tuple, TypeVar, Union
 
+from pytorch_lightning.utilities import rank_zero_warn
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
 from flash.core.data.utilities.sort import sorted_alphanumeric
@@ -152,6 +153,12 @@ def filter_valid_files(
     if not isinstance(files, List):
         files = [files]
 
+    if valid_extensions is None:
+        return (files,) + additional_lists
+
+    if not isinstance(valid_extensions, tuple):
+        valid_extensions = tuple(valid_extensions)
+
     additional_lists = tuple([a] if not isinstance(a, List) else a for a in additional_lists)
 
     if not all(len(a) == len(files) for a in additional_lists):
@@ -159,11 +166,23 @@ def filter_valid_files(
             f"The number of files ({len(files)}) and the number of items in any additional lists must be the same."
         )
 
-    if valid_extensions is None:
-        return (files,) + additional_lists
     filtered = list(
         filter(lambda sample: has_file_allowed_extension(sample[0], valid_extensions), zip(files, *additional_lists))
     )
-    if len(additional_lists) > 0:
+
+    filtered_files = [f[0] for f in filtered]
+
+    invalid = [f for f in files if f not in filtered_files]
+
+    if invalid:
+        invalid_extensions = list({"." + f.split(".")[-1] for f in invalid})
+        rank_zero_warn(
+            f"Found invalid file extensions: {', '.join(invalid_extensions)}. "
+            "Files with these extensions will be ignored. "
+            f"The supported file extensions are: {', '.join(valid_extensions)}."
+        )
+
+    if additional_lists:
         return tuple(zip(*filtered))
-    return [f[0] for f in filtered]
+
+    return filtered_files


Test patch:
diff --git a/tests/core/data/utilities/test_paths.py b/tests/core/data/utilities/test_paths.py
new file mode 100644
--- /dev/null
+++ b/tests/core/data/utilities/test_paths.py
@@ -0,0 +1,107 @@
+# Copyright The PyTorch Lightning team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import warnings
+from string import ascii_lowercase
+from typing import List
+
+import pytest
+from numpy import random
+
+from flash.audio.data import AUDIO_EXTENSIONS
+from flash.core.data.utilities.paths import filter_valid_files, PATH_TYPE
+from flash.image.data import IMG_EXTENSIONS, NP_EXTENSIONS
+
+
+def _make_mock_dir(root, mock_files: List) -> List[PATH_TYPE]:
+    mockdir = []
+    for idx, f_ext in enumerate(mock_files):
+        mockdir.append(os.path.join(root, mock_files[idx]))
+    return mockdir
+
+
+def _make_fake_files(mock_extensions, seed: int) -> List[str]:
+    random.seed(seed)
+    fake_files = mock_extensions[:]
+    for idx, f_ext in enumerate(fake_files):
+        idxs = random.randint(0, len(ascii_lowercase), size=3)
+        fake_name = "".join([ascii_lowercase[ix] for ix in idxs])
+        fake_files[idx] = "".join([fake_name, f_ext])
+    return fake_files
+
+
+def _make_fake_extensions(seed: int) -> List[str]:
+    random.seed(seed)
+    fake_extensions = []
+    for i in range(5):
+        idxs = random.randint(0, len(ascii_lowercase), size=3)
+        fake_extensions.append("".join(["."] + [ascii_lowercase[idx] for idx in idxs]))
+    return fake_extensions
+
+
+def _make_valid_extensions() -> tuple:
+    return AUDIO_EXTENSIONS + IMG_EXTENSIONS + NP_EXTENSIONS
+
+
+def test_filter_valid_files(tmpdir) -> None:
+    valid_extensions = _make_valid_extensions()
+    valid_extensions = list(valid_extensions)
+    fake_extensions = _make_fake_extensions(seed=42)
+    mock_extensions = valid_extensions + fake_extensions
+    mock_files = _make_fake_files(mock_extensions, seed=42)
+    mockdir = _make_mock_dir(tmpdir, mock_files)
+    message = "Found invalid file extensions"
+    with pytest.warns(UserWarning, match=message):
+        filtered = filter_valid_files(mockdir, valid_extensions=valid_extensions)
+    assert all(i not in fake_extensions for i in filtered)
+
+
+def test_filter_valid_files_no_invalid(tmpdir):
+    valid_extensions = _make_valid_extensions()
+    valid_extensions = list(valid_extensions)
+    mock_files = _make_fake_files(valid_extensions, seed=42)
+    mockdir = _make_mock_dir(tmpdir, mock_files)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        filtered = filter_valid_files(mockdir, valid_extensions=valid_extensions)
+    assert len(filtered) == len(mockdir)
+
+
+def test_filter_valid_files_with_additional_list(tmpdir) -> None:
+    valid_extensions = _make_valid_extensions()
+    valid_extensions = list(valid_extensions)
+    fake_extensions = _make_fake_extensions(seed=42)
+    mock_extensions = valid_extensions + fake_extensions
+    mock_files = _make_fake_files(mock_extensions, seed=42)
+    mockdir = _make_mock_dir(tmpdir, mock_files)
+    message = "Found invalid file extensions"
+    with pytest.warns(UserWarning, match=message):
+        filtered = filter_valid_files(
+            mockdir,
+            mockdir,
+            valid_extensions=valid_extensions,
+        )
+    assert all(i not in fake_extensions for i in filtered)
+
+
+def test_filter_valid_files_no_invalid_with_additional_list(tmpdir):
+    valid_extensions = _make_valid_extensions()
+    valid_extensions = list(valid_extensions)
+    mock_files = _make_fake_files(valid_extensions, seed=42)
+    mockdir = _make_mock_dir(tmpdir, mock_files)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        filtered_files, filtered_additional = filter_valid_files(mockdir, mockdir, valid_extensions=valid_extensions)
+    assert len(filtered_files) == len(mockdir)
+    assert len(filtered_additional) == len(mockdir)