Repository: huggingface/dataset-viewer

Issue:
The API returns the wrong row number
As stated in this post: https://discuss.huggingface.co/t/got-wrong-row-number-of-dataset-viewer/77132, I got the wrong number on the dataset repo page. The numbers are correct if I download the repo, but when I follow the instructions in https://huggingface.co/docs/datasets-server/size?code=curl, the numbers of rows in the response are also wrong. I'd like to know if there is something wrong with my usage or if there are some bugs in the API.


Discussion:
Update:
When I called
```bash
curl https://datasets-server.huggingface.co/statistics\?dataset\=${dataset_path}\&config\=default\&split\=test \                                                                    ─╯
        -X GET \
        -H "Authorization: Bearer ${API_TOKEN}"
```
I will get the correct `number_examples`.

But when I called
```bash
curl https://datasets-server.huggingface.co/size\?dataset\=${dataset_path} \                                                                                                        ─╯
        -X GET \
        -H "Authorization: Bearer ${API_TOKEN}"
```
The `num_rows` attribute gives the wrong number.
Update:
I've checked the API source code and recent commit, and it seems that this function has be changed yesterday: https://github.com/huggingface/datasets-server/blob/476c564b218c98eb9005ecd73c81d62d2d2f2563/services/worker/src/worker/job_runners/config/parquet_and_info.py#L666-L737.

The `num_examples` is miscalculated. I think that maybe #L684 should be in the for-loop? (I'm not sure about this because I'm not familiar with this code base).

If anyone can help to check this? @severo @polinaeterna 
wow, well spotted, thanks!
Reports:
- https://discuss.huggingface.co/t/got-wrong-row-number-of-dataset-viewer/77132/3
- https://discord.com/channels/879548962464493619/1217113204912816148/1217113204912816148
related to https://github.com/huggingface/datasets-server/pull/2564

Resolving patch:
diff --git a/services/worker/src/worker/job_runners/config/parquet_and_info.py b/services/worker/src/worker/job_runners/config/parquet_and_info.py
--- a/services/worker/src/worker/job_runners/config/parquet_and_info.py
+++ b/services/worker/src/worker/job_runners/config/parquet_and_info.py
@@ -681,8 +681,8 @@ def fill_builder_info(
     builder.info.download_size = 0
     builder.info.dataset_size = 0
     logging.info("Start validation of parquet files.")
-    num_examples = 0
     for split, urls in data_files.items():
+        num_examples = 0
         split = str(split)  # in case it's a NamedSplit
         first_url = urls[0]
         try:


Test patch:
diff --git a/services/worker/tests/fixtures/hub.py b/services/worker/tests/fixtures/hub.py
--- a/services/worker/tests/fixtures/hub.py
+++ b/services/worker/tests/fixtures/hub.py
@@ -392,6 +392,38 @@ def hub_public_three_parquet_files_builder(three_parquet_files_paths: list[str])
     delete_hub_dataset_repo(repo_id=repo_id)
 
 
+@pytest.fixture(scope="session")
+def three_parquet_splits_paths(
+    tmp_path_factory: pytest.TempPathFactory, datasets: Mapping[str, Dataset]
+) -> Mapping[str, str]:
+    dataset = datasets["descriptive_statistics_string_text"]
+    path1 = str(tmp_path_factory.mktemp("data") / "train.parquet")
+    data1 = Dataset.from_dict(dataset[:30])
+    with open(path1, "wb") as f:
+        data1.to_parquet(f)
+
+    path2 = str(tmp_path_factory.mktemp("data") / "test.parquet")
+    data2 = Dataset.from_dict(dataset[30:60])
+    with open(path2, "wb") as f:
+        data2.to_parquet(f)
+
+    path3 = str(tmp_path_factory.mktemp("data") / "validation.parquet")
+    data3 = Dataset.from_dict(dataset[60:])
+    with open(path3, "wb") as f:
+        data3.to_parquet(f)
+
+    return {"train": path1, "test": path2, "validation": path3}
+
+
+@pytest.fixture(scope="session")
+def hub_public_three_parquet_splits_builder(three_parquet_splits_paths: Mapping[str, str]) -> Iterator[str]:
+    repo_id = create_hub_dataset_repo(
+        prefix="parquet_builder_three_splits", file_paths=list(three_parquet_splits_paths.values())
+    )
+    yield repo_id
+    delete_hub_dataset_repo(repo_id=repo_id)
+
+
 class HubDatasetTest(TypedDict):
     name: str
     config_names_response: Any
diff --git a/services/worker/tests/job_runners/config/test_parquet_and_info.py b/services/worker/tests/job_runners/config/test_parquet_and_info.py
--- a/services/worker/tests/job_runners/config/test_parquet_and_info.py
+++ b/services/worker/tests/job_runners/config/test_parquet_and_info.py
@@ -3,7 +3,7 @@
 
 import io
 import os
-from collections.abc import Callable, Iterator
+from collections.abc import Callable, Iterator, Mapping
 from dataclasses import replace
 from fnmatch import fnmatch
 from http import HTTPStatus
@@ -828,6 +828,38 @@ def test_fill_builder_info_multiple_parquets(
         assert split_info.num_bytes == pytest.approx(actual_info.splits[split_name].num_bytes, 300)
 
 
+def test_fill_builder_info_multiple_splits(
+    hub_public_three_parquet_splits_builder: str,
+    app_config: AppConfig,
+    tmp_path: Path,
+    three_parquet_splits_paths: Mapping[str, str],
+) -> None:
+    cache_dir = str(tmp_path / "test_fill_builder_info_multiple_splits")
+    name = hub_public_three_parquet_splits_builder
+    builder = load_dataset_builder(name, cache_dir=cache_dir)
+    builder.info = datasets.info.DatasetInfo()
+    validate = ParquetFileValidator(max_row_group_byte_size=100_000).validate
+    fill_builder_info(builder, hf_endpoint=app_config.common.hf_endpoint, hf_token=None, validate=validate)
+
+    # load dataset with `load_dataset` to generate correct values for dataset info to compare
+    dataset = load_dataset("parquet", data_files=three_parquet_splits_paths)
+    actual_info = builder.info
+    assert any(expected_info.features == actual_info.features for expected_info in dataset.values())
+    assert any(expected_info.download_size == actual_info.download_size for expected_info in dataset.values())
+    # uncompressed dataset size is approximate
+    # since we estimate approximate compression ratio by reading only first row group
+    assert any(
+        expected_info.dataset_size == pytest.approx(actual_info.dataset_size, 300)
+        for expected_info in dataset.values()
+    )
+
+    assert set(dataset.keys()) == set(actual_info.splits)
+    assert len(actual_info.splits) == len(three_parquet_splits_paths)
+    for split_name, split in dataset.items():
+        assert split.info.splits[split_name].num_examples == actual_info.splits[split_name].num_examples
+        assert split.info.splits[split_name].num_bytes == pytest.approx(actual_info.splits[split_name].num_bytes, 300)
+
+
 @pytest.mark.parametrize(
     "num_rows, row_group_byte_size, max_row_group_byte_size, expected",
     [