Repository: huggingface/dataset-viewer

Issue:
Ensure the parquet files in /parquet are sorted by "shard" index
And tell it in the docs


Discussion:

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/constants.py b/libs/libcommon/src/libcommon/constants.py
--- a/libs/libcommon/src/libcommon/constants.py
+++ b/libs/libcommon/src/libcommon/constants.py
@@ -17,27 +17,27 @@
 
 DEFAULT_INPUT_TYPE = "dataset"
 DEFAULT_JOB_RUNNER_VERSION = 1
-PROCESSING_STEP_DATASET_CONFIG_NAMES_VERSION = 1
-PROCESSING_STEP_CONFIG_PARQUET_VERSION = 4
-PROCESSING_STEP_CONFIG_PARQUET_METADATA_VERSION = 1
-PROCESSING_STEP_CONFIG_SIZE_VERSION = 2
 PROCESSING_STEP_CONFIG_INFO_VERSION = 2
 PROCESSING_STEP_CONFIG_OPT_IN_OUT_URLS_COUNT_VERSION = 3
+PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_VERSION = 3
+PROCESSING_STEP_CONFIG_PARQUET_METADATA_VERSION = 1
+PROCESSING_STEP_CONFIG_PARQUET_VERSION = 5
+PROCESSING_STEP_CONFIG_SIZE_VERSION = 2
+PROCESSING_STEP_CONFIG_SPLIT_NAMES_FROM_INFO_VERSION = 3
+PROCESSING_STEP_CONFIG_SPLIT_NAMES_FROM_STREAMING_VERSION = 3
+PROCESSING_STEP_DATASET_CONFIG_NAMES_VERSION = 1
 PROCESSING_STEP_DATASET_INFO_VERSION = 2
 PROCESSING_STEP_DATASET_IS_VALID_VERSION = 4
 PROCESSING_STEP_DATASET_OPT_IN_OUT_URLS_COUNT_VERSION = 2
 PROCESSING_STEP_DATASET_PARQUET_VERSION = 2
 PROCESSING_STEP_DATASET_SIZE_VERSION = 2
+PROCESSING_STEP_DATASET_SPLIT_NAMES_VERSION = 3
+PROCESSING_STEP_SPLIT_DUCKDB_INDEX_VERSION = 1
 PROCESSING_STEP_SPLIT_FIRST_ROWS_FROM_PARQUET_VERSION = 2
-PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_VERSION = 3
 PROCESSING_STEP_SPLIT_FIRST_ROWS_FROM_STREAMING_VERSION = 3
-PROCESSING_STEP_CONFIG_SPLIT_NAMES_FROM_INFO_VERSION = 3
-PROCESSING_STEP_CONFIG_SPLIT_NAMES_FROM_STREAMING_VERSION = 3
-PROCESSING_STEP_DATASET_SPLIT_NAMES_VERSION = 3
+PROCESSING_STEP_SPLIT_IMAGE_URL_COLUMNS_VERSION = 1
 PROCESSING_STEP_SPLIT_OPT_IN_OUT_URLS_COUNT_VERSION = 2
 PROCESSING_STEP_SPLIT_OPT_IN_OUT_URLS_SCAN_VERSION = 4
-PROCESSING_STEP_SPLIT_IMAGE_URL_COLUMNS_VERSION = 1
-PROCESSING_STEP_SPLIT_DUCKDB_INDEX_VERSION = 1
 
 PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS = 100
 PROCESSING_STEP_CONFIG_PARQUET_AND_INFO_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS = 100
diff --git a/services/worker/src/worker/job_runners/config/parquet.py b/services/worker/src/worker/job_runners/config/parquet.py
--- a/services/worker/src/worker/job_runners/config/parquet.py
+++ b/services/worker/src/worker/job_runners/config/parquet.py
@@ -35,10 +35,15 @@ def compute_parquet_response(dataset: str, config: str) -> ConfigParquetResponse
         kinds=[previous_step], dataset=dataset, config=config
     )
     content = config_parquet_and_info_best_response.response["content"]
-    if "parquet_files" not in content:
-        raise PreviousStepFormatError("Previous step did not return the expected content: 'parquet_files'.")
 
-    parquet_files = [parquet_file for parquet_file in content["parquet_files"] if parquet_file.get("config") == config]
+    try:
+        parquet_files = [
+            parquet_file for parquet_file in content["parquet_files"] if parquet_file.get("config") == config
+        ]
+        # sort by filename, which ensures the shards are in order: 00000, 00001, 00002, ...
+        parquet_files.sort(key=lambda x: x["filename"])  # type: ignore
+    except KeyError as e:
+        raise PreviousStepFormatError("Previous step did not return the expected content: 'parquet_files'.", e) from e
     return ConfigParquetResponse(parquet_files=parquet_files)
 
 
diff --git a/services/worker/src/worker/job_runners/dataset/parquet.py b/services/worker/src/worker/job_runners/dataset/parquet.py
--- a/services/worker/src/worker/job_runners/dataset/parquet.py
+++ b/services/worker/src/worker/job_runners/dataset/parquet.py
@@ -23,7 +23,7 @@
 from worker.job_runners.dataset.dataset_job_runner import DatasetJobRunner
 
 
-def compute_sizes_response(dataset: str) -> Tuple[DatasetParquetResponse, float]:
+def compute_parquet_response(dataset: str) -> Tuple[DatasetParquetResponse, float]:
     """
     Get the response of dataset-parquet for one specific dataset on huggingface.co.
     Args:
@@ -108,5 +108,5 @@ def get_job_runner_version() -> int:
         return PROCESSING_STEP_DATASET_PARQUET_VERSION
 
     def compute(self) -> JobResult:
-        response_content, progress = compute_sizes_response(dataset=self.dataset)
+        response_content, progress = compute_parquet_response(dataset=self.dataset)
         return JobResult(response_content, progress=progress)


Test patch:
diff --git a/services/worker/tests/job_runners/config/test_parquet.py b/services/worker/tests/job_runners/config/test_parquet.py
--- a/services/worker/tests/job_runners/config/test_parquet.py
+++ b/services/worker/tests/job_runners/config/test_parquet.py
@@ -114,6 +114,70 @@ def _get_job_runner(
             None,
             True,
         ),
+        (
+            "shards_order",
+            "config_1",
+            HTTPStatus.OK,
+            ConfigParquetAndInfoResponse(
+                parquet_files=[
+                    SplitHubFile(
+                        dataset="ok",
+                        config="config_1",
+                        split="train",
+                        url="url2",
+                        filename="parquet-train-00001-of-05534.parquet",
+                        size=0,
+                    ),
+                    SplitHubFile(
+                        dataset="ok",
+                        config="config_1",
+                        split="train",
+                        url="url1",
+                        filename="parquet-train-00000-of-05534.parquet",
+                        size=0,
+                    ),
+                    SplitHubFile(
+                        dataset="ok",
+                        config="config_1",
+                        split="test",
+                        url="url2",
+                        filename="parquet-test-00000-of-00001.parquet",
+                        size=0,
+                    ),
+                ],
+                dataset_info={"description": "value", "dataset_size": 10},
+            ),
+            None,
+            ConfigParquetResponse(
+                parquet_files=[
+                    SplitHubFile(
+                        dataset="ok",
+                        config="config_1",
+                        split="test",
+                        url="url2",
+                        filename="parquet-test-00000-of-00001.parquet",
+                        size=0,
+                    ),
+                    SplitHubFile(
+                        dataset="ok",
+                        config="config_1",
+                        split="train",
+                        url="url1",
+                        filename="parquet-train-00000-of-05534.parquet",
+                        size=0,
+                    ),
+                    SplitHubFile(
+                        dataset="ok",
+                        config="config_1",
+                        split="train",
+                        url="url2",
+                        filename="parquet-train-00001-of-05534.parquet",
+                        size=0,
+                    ),
+                ]
+            ),
+            False,
+        ),
     ],
 )
 def test_compute(