Repository: PennyLaneAI/pennylane

Issue:
[Bug] Inconsistent ordering in datasets for lists of parameters
The ordering of lists of datasets when passed lists of parameters is of no canonical ordering, but appears to be random:

```python
bondlengths = ['0.5', '0.54', '0.58', '0.62']
data = qml.data.load("qchem", molname="H2", basis="STO-3G", bondlength=bondlengths)
print(data)
```
would expect output:
```pycon
[<Dataset = description: qchem/H2/STO-3G/0.5, attributes: ['molecule', 'hamiltonian', ...]>, 
<Dataset = description: qchem/H2/STO-3G/0.54, attributes: ['molecule', 'hamiltonian', ...]>, 
<Dataset = description: qchem/H2/STO-3G/0.58, attributes: ['molecule', 'hamiltonian', ...]>, 
<Dataset = description: qchem/H2/STO-3G/0.62, attributes: ['molecule', 'hamiltonian', ...]>]
```
but actually obtain:
```pycon
[<Dataset = description: qchem/H2/STO-3G/0.54, attributes: ['molecule', 'hamiltonian', ...]>, 
<Dataset = description: qchem/H2/STO-3G/0.5, attributes: ['molecule', 'hamiltonian', ...]>, 
<Dataset = description: qchem/H2/STO-3G/0.58, attributes: ['molecule', 'hamiltonian', ...]>, 
<Dataset = description: qchem/H2/STO-3G/0.62, attributes: ['molecule', 'hamiltonian', ...]>]
```


Discussion:
thanks for reporting this! the attribute ordering (per-parameter) gets randomized in [this line](https://github.com/PennyLaneAI/pennylane/blob/6daabc6cd265f5112e426bedcb57f4197b7fbb7c/pennylane/data/data_manager.py#L200) because it uses sets to remove any attributes the user provided that aren't actually available. I agree that the random ordering is unexpected behaviour, and should probably be fixed.

~I had originally used sets here to ensure that we don't request any files that don't exist in S3, but I eventually added more pre-validation after writing that line of code! The set intersection is not really needed here, so I'll remove it.~

EDIT: good thing I wrote unit tests! I was using sets for a specific case and still need to keep that, but I'll still ensure the given order is preserved.

Resolving patch:
diff --git a/pennylane/data/data_manager.py b/pennylane/data/data_manager.py
--- a/pennylane/data/data_manager.py
+++ b/pennylane/data/data_manager.py
@@ -197,7 +197,11 @@ def _generate_folders(node, folders):
     """
 
     next_folders = folders[1:]
-    folders = set(node) if folders[0] == ["full"] else set(folders[0]).intersection(set(node))
+    if folders[0] == ["full"]:
+        folders = node
+    else:
+        values_for_this_node = set(folders[0]).intersection(set(node))
+        folders = [f for f in folders[0] if f in values_for_this_node]
     return (
         [
             os.path.join(folder, child)


Test patch:
diff --git a/tests/data/test_dataset_access.py b/tests/data/test_dataset_access.py
--- a/tests/data/test_dataset_access.py
+++ b/tests/data/test_dataset_access.py
@@ -304,7 +304,7 @@ class TestLoadHelpers:
                     "HeH": {"STO-3G": ["0.50"]},
                 },
                 [["full"], ["full"], ["0.48", "0.50"]],
-                ["H2/6-31G/0.50", "H2/STO-3G/0.48", "H2/STO-3G/0.50", "HeH/STO-3G/0.50"],
+                ["H2/STO-3G/0.48", "H2/STO-3G/0.50", "H2/6-31G/0.50", "HeH/STO-3G/0.50"],
             ),
             (
                 {
@@ -314,11 +314,19 @@ class TestLoadHelpers:
                 [["full"], ["STO-3G"], ["0.50"]],
                 [],
             ),
+            (
+                {
+                    "H2": {"STO-3G": ["0.46", "0.48", "0.50"], "6-31G": ["0.46", "0.50"]},
+                    "HeH": {"STO-3G": ["0.50"]},
+                },
+                [["HeH", "H2"], ["STO-3G"], ["0.50", "0.46"]],
+                ["HeH/STO-3G/0.50", "H2/STO-3G/0.50", "H2/STO-3G/0.46"],
+            ),
         ],
     )
     def test_generate_folders(self, node, folders, output):
         """Test the _generate_folders helper function."""
-        assert sorted(qml.data.data_manager._generate_folders(node, folders)) == output
+        assert qml.data.data_manager._generate_folders(node, folders) == output
 
     @patch("concurrent.futures.ThreadPoolExecutor.submit", return_value=True)
     @patch("pennylane.data.data_manager.wait", return_value=MagicMock(done=[]))
@@ -492,12 +500,26 @@ def test_successful_load_many(self, tmp_path):
             "qchem", molname="H2", basis="6-31G", bondlength="full", folder_path=str(tmp_path)
         )
         assert len(datasets) == 3
-        assert sorted([d.molecule for d in datasets]) == [
+        assert [d.molecule for d in datasets] == [
             "H2_6-31G_0.46_full",
-            "H2_6-31G_1.0_full",
             "H2_6-31G_1.16_full",
+            "H2_6-31G_1.0_full",
         ]
 
+    def test_stable_load_order(self, tmp_path):
+        """Test that the order of returned datasets is stable and sorted."""
+        lengths = [1.0, 1.16, 0.46]
+        path = str(tmp_path)
+        for _ in range(5):  # 5 subsequent calls to suggest stability
+            data = qml.data.load(
+                "qchem", molname="H2", basis="6-31G", bondlength=lengths, folder_path=path
+            )
+            assert [d.molecule for d in data] == [
+                "H2_6-31G_1.0_full",
+                "H2_6-31G_1.16_full",
+                "H2_6-31G_0.46_full",
+            ]
+
     @pytest.mark.parametrize(
         ("attributes", "fullfile"),
         [(["full"], "H2/6-31G/0.46/H2_6-31G_0.46_full.dat"), (["molecule"], None)],