Repository: pytorch/audio

Issue:
Dataset traversal cannot be deterministic
[Many dataset implementations use `walk_files` function](https://github.com/pytorch/audio/search?q=walk_files&unscoped_q=walk_files) which uses `os.walk` which does not give us choice to make directory traversal in deterministic order.

This makes it harder to test dataset and causes issue on reproducibility. Dataset should have an option to be deterministic.


Discussion:

Resolving patch:
diff --git a/torchaudio/datasets/utils.py b/torchaudio/datasets/utils.py
--- a/torchaudio/datasets/utils.py
+++ b/torchaudio/datasets/utils.py
@@ -264,7 +264,13 @@ def walk_files(root: str,
 
     root = os.path.expanduser(root)
 
-    for dirpath, _, files in os.walk(root):
+    for dirpath, dirs, files in os.walk(root):
+        dirs.sort()
+        # `dirs` is the list used in os.walk function and by sorting it in-place here, we change the
+        # behavior of os.walk to traverse sub directory alphabetically
+        # see also
+        # https://stackoverflow.com/questions/6670029/can-i-force-python3s-os-walk-to-visit-directories-in-alphabetical-order-how#comment71993866_6670926
+        files.sort()
         for f in files:
             if f.endswith(suffix):
 


Test patch:
diff --git a/test/datasets/libritts_test.py b/test/datasets/libritts_test.py
--- a/test/datasets/libritts_test.py
+++ b/test/datasets/libritts_test.py
@@ -49,16 +49,14 @@ def setUpClass(cls):
 
     def test_libritts(self):
         dataset = LIBRITTS(self.root_dir)
-        samples = list(dataset)
-        samples.sort(key=lambda s: s[4])
-
+        n_ites = 0
         for i, (waveform,
                 sample_rate,
                 original_text,
                 normalized_text,
                 speaker_id,
                 chapter_id,
-                utterance_id) in enumerate(samples):
+                utterance_id) in enumerate(dataset):
 
             expected_ids = self.utterance_ids[i]
             expected_data = self.data[i]
@@ -69,3 +67,5 @@ def test_libritts(self):
             assert original_text == self.original_text
             assert normalized_text == self.normalized_text
             assert utterance_id == f'{"_".join(str(u) for u in expected_ids[-4:])}'
+            n_ites += 1
+        assert n_ites == len(self.utterance_ids)
diff --git a/test/datasets/utils_test.py b/test/datasets/utils_test.py
new file mode 100644
--- /dev/null
+++ b/test/datasets/utils_test.py
@@ -0,0 +1,47 @@
+import os
+from pathlib import Path
+
+from torchaudio.datasets import utils as dataset_utils
+
+from ..common_utils import (
+    TempDirMixin,
+    TorchaudioTestCase,
+)
+
+
+class TestWalkFiles(TempDirMixin, TorchaudioTestCase):
+    root = None
+    expected = None
+
+    def _add_file(self, *parts):
+        path = self.get_temp_path(*parts)
+        self.expected.append(path)
+        Path(path).touch()
+
+    def setUp(self):
+        self.root = self.get_temp_path()
+        self.expected = []
+
+        # level 1
+        for filename in ['a.txt', 'b.txt', 'c.txt']:
+            self._add_file(filename)
+
+        # level 2
+        for dir1 in ['d1', 'd2', 'd3']:
+            for filename in ['d.txt', 'e.txt', 'f.txt']:
+                self._add_file(dir1, filename)
+            # level 3
+            for dir2 in ['d1', 'd2', 'd3']:
+                for filename in ['g.txt', 'h.txt', 'i.txt']:
+                    self._add_file(dir1, dir2, filename)
+
+        print('\n'.join(self.expected))
+
+    def test_walk_files(self):
+        """walk_files should traverse files in alphabetical order"""
+        n_ites = 0
+        for i, path in enumerate(dataset_utils.walk_files(self.root, '.txt', prefix=True)):
+            found = os.path.join(self.root, path)
+            assert found == self.expected[i]
+            n_ites += 1
+        assert n_ites == len(self.expected)
diff --git a/test/datasets/yesno_test.py b/test/datasets/yesno_test.py
--- a/test/datasets/yesno_test.py
+++ b/test/datasets/yesno_test.py
@@ -38,11 +38,12 @@ def setUpClass(cls):
 
     def test_yesno(self):
         dataset = yesno.YESNO(self.root_dir)
-        samples = list(dataset)
-        samples.sort(key=lambda s: s[2])
-        for i, (waveform, sample_rate, label) in enumerate(samples):
+        n_ite = 0
+        for i, (waveform, sample_rate, label) in enumerate(dataset):
             expected_label = self.labels[i]
             expected_data = self.data[i]
             self.assertEqual(expected_data, waveform, atol=5e-5, rtol=1e-8)
             assert sample_rate == 8000
             assert label == expected_label
+            n_ite += 1
+        assert n_ite == len(self.data)