Repository: huggingface/dataset-viewer

Issue:
Update datasets to 2.19.0
Update datasets to 2.19.0: https://github.com/huggingface/datasets/releases/tag/2.19.0


Discussion:

Resolving patch:
diff --git a/services/worker/src/worker/job_runners/dataset/compatible_libraries.py b/services/worker/src/worker/job_runners/dataset/compatible_libraries.py
--- a/services/worker/src/worker/job_runners/dataset/compatible_libraries.py
+++ b/services/worker/src/worker/job_runners/dataset/compatible_libraries.py
@@ -23,9 +23,8 @@
 )
 from datasets.packaged_modules import _MODULE_TO_EXTENSIONS, _PACKAGED_DATASETS_MODULES
 from datasets.utils.file_utils import cached_path
-from datasets.utils.hub import hf_hub_url
 from datasets.utils.metadata import MetadataConfigs
-from huggingface_hub import DatasetCard, DatasetCardData, HfFileSystem
+from huggingface_hub import DatasetCard, DatasetCardData, HfFileSystem, hf_hub_url
 from libcommon.constants import LOADING_METHODS_MAX_CONFIGS
 from libcommon.croissant_utils import get_record_set
 from libcommon.exceptions import DatasetWithTooComplexDataFilesPatternsError, PreviousStepFormatError
@@ -44,12 +43,21 @@
 )
 from worker.job_runners.dataset.dataset_job_runner import DatasetJobRunnerWithDatasetsCache
 
-NON_WORD_GLOB_SEPARATOR = f"[{NON_WORDS_CHARS}/]"
-NON_WORD_REGEX_SEPARATOR = NON_WORD_GLOB_SEPARATOR.replace(".", "\.").replace("/", "\/")
-
-if any(
-    NON_WORD_GLOB_SEPARATOR not in pattern.format(keyword="train", sep=NON_WORDS_CHARS)
-    for pattern in datasets.data_files.KEYWORDS_IN_PATH_NAME_BASE_PATTERNS
+BASE_PATTENS_WITH_SEPARATOR = [
+    pattern
+    for pattern in datasets.data_files.KEYWORDS_IN_FILENAME_BASE_PATTERNS
+    + datasets.data_files.KEYWORDS_IN_DIR_NAME_BASE_PATTERNS
+    if "{sep}" in pattern
+]
+NON_WORD_GLOB_SEPARATOR = f"[{NON_WORDS_CHARS}]"
+NON_WORD_REGEX_SEPARATOR = NON_WORD_GLOB_SEPARATOR.replace(".", "\.")
+
+if (
+    any(
+        NON_WORD_GLOB_SEPARATOR not in pattern.format(keyword="train", sep=NON_WORDS_CHARS)
+        for pattern in BASE_PATTENS_WITH_SEPARATOR
+    )
+    or not BASE_PATTENS_WITH_SEPARATOR
 ):
     raise ImportError(
         f"Current `datasets` version is not compatible with simplify_data_files_patterns() which expects as keyword separator {NON_WORD_GLOB_SEPARATOR} for glob patterns. "
@@ -84,7 +92,7 @@ def get_builder_configs_with_simplified_data_files(
     download_config = DownloadConfig(token=hf_token)
     try:
         dataset_readme_path = cached_path(
-            hf_hub_url(dataset, datasets.config.REPOCARD_FILENAME),
+            hf_hub_url(dataset, datasets.config.REPOCARD_FILENAME, repo_type="dataset"),
             download_config=download_config,
         )
         dataset_card_data = DatasetCard.load(Path(dataset_readme_path)).data
@@ -92,7 +100,7 @@ def get_builder_configs_with_simplified_data_files(
         dataset_card_data = DatasetCardData()
     try:
         standalone_yaml_path = cached_path(
-            hf_hub_url(dataset, datasets.config.REPOYAML_FILENAME),
+            hf_hub_url(dataset, datasets.config.REPOYAML_FILENAME, repo_type="dataset"),
             download_config=download_config,
         )
         with open(standalone_yaml_path, "r", encoding="utf-8") as f:
@@ -150,9 +158,9 @@ def simplify_data_files_patterns(
 
         ```
         data/train-[0-9][0-9][0-9][0-9][0-9]-of-[0-9][0-9][0-9][0-9][0-9]*.*        =>      data/train-*-of-*.parquet
-        **/*[-._ 0-9/]train[-._ 0-9/]**                                             =>      **/*_train_*.jsonl
-        test[-._ 0-9/]**                                                            =>      test[0-9].csv
-        train[-._ 0-9/]**                                                           =>      train-*.tar
+        **/*[-._ 0-9]train[-._ 0-9]*/**                                             =>      **/*_train_*.jsonl
+        test[-._ 0-9]*/**                                                            =>      test[0-9].csv
+        train[-._ 0-9]*/**                                                           =>      train-*.tar
         ```
     """
     patterns = DataFilesPatternsList([], allowed_extensions=None)


Test patch:
diff --git a/libs/libcommon/tests/test_parquet_utils.py b/libs/libcommon/tests/test_parquet_utils.py
--- a/libs/libcommon/tests/test_parquet_utils.py
+++ b/libs/libcommon/tests/test_parquet_utils.py
@@ -108,7 +108,7 @@ def ds_image_fs(ds_image: Dataset, tmpfs: AbstractFileSystem) -> Generator[Abstr
 def ds_parquet_metadata_dir(
     ds_fs: AbstractFileSystem, parquet_metadata_directory: StrPath
 ) -> Generator[StrPath, None, None]:
-    parquet_shard_paths = ds_fs.glob("**.parquet")
+    parquet_shard_paths = ds_fs.glob("**/*.parquet")
     for parquet_shard_path in parquet_shard_paths:
         parquet_file_metadata_path = Path(parquet_metadata_directory) / "ds" / "--" / parquet_shard_path
         parquet_file_metadata_path.parent.mkdir(parents=True, exist_ok=True)
@@ -179,7 +179,7 @@ def dataset_with_config_parquet_metadata(
 def ds_empty_parquet_metadata_dir(
     ds_empty_fs: AbstractFileSystem, parquet_metadata_directory: StrPath
 ) -> Generator[StrPath, None, None]:
-    parquet_shard_paths = ds_empty_fs.glob("**.parquet")
+    parquet_shard_paths = ds_empty_fs.glob("**/*.parquet")
     for parquet_shard_path in parquet_shard_paths:
         parquet_file_metadata_path = Path(parquet_metadata_directory) / "ds_empty" / "--" / parquet_shard_path
         parquet_file_metadata_path.parent.mkdir(parents=True, exist_ok=True)
@@ -250,7 +250,7 @@ def dataset_empty_with_config_parquet_metadata(
 def ds_sharded_parquet_metadata_dir(
     ds_sharded_fs: AbstractFileSystem, parquet_metadata_directory: StrPath
 ) -> Generator[StrPath, None, None]:
-    parquet_shard_paths = ds_sharded_fs.glob("**.parquet")
+    parquet_shard_paths = ds_sharded_fs.glob("**/*.parquet")
     for parquet_shard_path in parquet_shard_paths:
         parquet_file_metadata_path = Path(parquet_metadata_directory) / "ds_sharded" / "--" / parquet_shard_path
         parquet_file_metadata_path.parent.mkdir(parents=True, exist_ok=True)
@@ -305,7 +305,7 @@ def dataset_sharded_with_config_parquet_metadata(
                 "num_rows": pq.read_metadata(ds_sharded_fs.open(parquet_file_path)).num_rows,
                 "parquet_metadata_subpath": f"ds_sharded/--/{parquet_file_path}",
             }
-            for parquet_file_path in ds_sharded_fs.glob("default/**.parquet")
+            for parquet_file_path in ds_sharded_fs.glob("default/**/*.parquet")
         ]
     }
     upsert_response(
diff --git a/services/worker/tests/job_runners/dataset/test_compatible_libraries.py b/services/worker/tests/job_runners/dataset/test_compatible_libraries.py
--- a/services/worker/tests/job_runners/dataset/test_compatible_libraries.py
+++ b/services/worker/tests/job_runners/dataset/test_compatible_libraries.py
@@ -404,7 +404,7 @@ def test_compute_error(
         (
             "Anthropic/hh-rlhf",
             "json",
-            {"default": {"test": ["**/*/test.jsonl.gz"], "train": ["**/*/train.jsonl.gz"]}},
+            {"default": {"test": ["**/test.jsonl.gz"], "train": ["**/train.jsonl.gz"]}},
         ),
         ("laion/conceptual-captions-12m-webdataset", "webdataset", {"default": {"train": ["**/*.tar"]}}),
         (
@@ -487,8 +487,8 @@ def test_simplify_data_files_patterns(
             "Anthropic/hh-rlhf",
             "json",
             {
-                "train": ["**/*/train.jsonl.gz"],
-                "test": ["**/*/test.jsonl.gz"],
+                "train": ["**/train.jsonl.gz"],
+                "test": ["**/test.jsonl.gz"],
             },
             "dask",
         ),