Repository: huggingface/dataset-viewer

Issue:
Replace canonical datasets with community ones in the docs/tests
For example, `glue` is now `nyu-mll/glue`

cc @lhoestq @albertvillanova for visibility.


Discussion:

Resolving patch:
diff --git a/front/admin_ui/app.py b/front/admin_ui/app.py
--- a/front/admin_ui/app.py
+++ b/front/admin_ui/app.py
@@ -398,13 +398,20 @@ def query_jobs(pending_jobs_query):
                     processing_step.job_type
                     for processing_step in processing_graph.get_topologically_ordered_processing_steps()
                 ]
+
                 def on_change_refresh_job_type(job_type):
                     return processing_graph.get_processing_step(job_type).difficulty
 
                 refresh_type = gr.Dropdown(
-                    job_types, multiselect=False, type="value", label="job type", value=job_types[0]
+                    job_types,
+                    multiselect=False,
+                    type="value",
+                    label="job type",
+                    value=job_types[0],
+                )
+                refresh_dataset_name = gr.Textbox(
+                    label="dataset", placeholder="allenai/c4"
                 )
-                refresh_dataset_name = gr.Textbox(label="dataset", placeholder="c4")
                 refresh_config_name = gr.Textbox(
                     label="config (optional)", placeholder="en"
                 )
@@ -415,8 +422,17 @@ def on_change_refresh_job_type(job_type):
                     "*you can select multiple values by separating them with commas, e.g. split='train, test'*"
                 )
 
-                refresh_difficulty = gr.Slider(0, 100, processing_graph.get_processing_step(job_types[0]).difficulty, step=10, interactive=True, label="difficulty")
-                refresh_type.change(on_change_refresh_job_type, refresh_type, refresh_difficulty)
+                refresh_difficulty = gr.Slider(
+                    0,
+                    100,
+                    processing_graph.get_processing_step(job_types[0]).difficulty,
+                    step=10,
+                    interactive=True,
+                    label="difficulty",
+                )
+                refresh_type.change(
+                    on_change_refresh_job_type, refresh_type, refresh_difficulty
+                )
 
                 refresh_priority = gr.Dropdown(
                     ["low", "normal", "high"],
@@ -551,7 +567,7 @@ def delete_and_recreate_dataset(
                     outputs=delete_and_recreate_dataset_output,
                 )
             with gr.Tab("Dataset status"):
-                dataset_name = gr.Textbox(label="dataset", placeholder="c4")
+                dataset_name = gr.Textbox(label="dataset", placeholder="allenai/c4")
                 dataset_status_button = gr.Button("Get dataset status")
                 gr.Markdown("### Pending jobs")
                 jobs_table = gr.DataFrame()
diff --git a/services/worker/src/worker/job_runners/config/parquet_and_info.py b/services/worker/src/worker/job_runners/config/parquet_and_info.py
--- a/services/worker/src/worker/job_runners/config/parquet_and_info.py
+++ b/services/worker/src/worker/job_runners/config/parquet_and_info.py
@@ -751,7 +751,7 @@ class limit_parquet_writes:
     Example of usage:
 
     ```python
-    builder = load_dataset_builder("squad")
+    builder = load_dataset_builder("rajpurkar/squad")
     max_dataset_size_bytes = 10_000_000
     with limit_parquet_writes(builder, max_dataset_size_bytes=max_dataset_size_bytes) as limiter:
         builder.download_and_prepare(file_format="parquet")
@@ -762,7 +762,7 @@ class limit_parquet_writes:
     the full dataset:
 
     ```python
-    builder = load_dataset_builder("squad")
+    builder = load_dataset_builder("rajpurkar/squad")
     max_dataset_size_bytes = 10_000_000
     dl_manager = StreamingDownloadManager(...)
     for split_generator in builder._split_generators(dl_manager):


Test patch:
diff --git a/e2e/tests/test_12_splits.py b/e2e/tests/test_12_splits.py
--- a/e2e/tests/test_12_splits.py
+++ b/e2e/tests/test_12_splits.py
@@ -10,7 +10,7 @@
     "status,name,dataset,config,error_code",
     [
         #  (200, "all splits in a dataset", "ibm/duorc", None, None),
-        #  (200, "splits for a single config", "emotion", "unsplit", None)
+        #  (200, "splits for a single config", "dair-ai/emotion", "unsplit", None)
         (
             401,
             "inexistent dataset, and not authenticated",
diff --git a/libs/libcommon/tests/test_duckdb_utils.py b/libs/libcommon/tests/test_duckdb_utils.py
--- a/libs/libcommon/tests/test_duckdb_utils.py
+++ b/libs/libcommon/tests/test_duckdb_utils.py
@@ -3,11 +3,11 @@
 
 def test_duckdb_index_is_partial() -> None:
     assert duckdb_index_is_partial(
-        "https://hf.co/datasets/c4/resolve/refs%2Fconvert%2Fparquet/en/partial-train/index.duckdb"
+        "https://hf.co/datasets/canonical/resolve/refs%2Fconvert%2Fduckdb/en/partial-train/index.duckdb"
     )
     assert duckdb_index_is_partial(
-        "https://hf.co/datasets/bigcode/the-stack/resolve/refs%2Fconvert%2Fparquet/default/train/partial-index.duckdb"
+        "https://hf.co/datasets/organization/not-canonical/resolve/refs%2Fconvert%2Fduckdb/default/train/partial-index.duckdb"
     )
     assert not duckdb_index_is_partial(
-        "https://hf.co/datasets/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/train/index.duckdb"
+        "https://hf.co/datasets/rajpurkar/squad/resolve/refs%2Fconvert%2Fduckdb/plain_text/train/index.duckdb"
     )
diff --git a/libs/libcommon/tests/test_parquet_utils.py b/libs/libcommon/tests/test_parquet_utils.py
--- a/libs/libcommon/tests/test_parquet_utils.py
+++ b/libs/libcommon/tests/test_parquet_utils.py
@@ -402,13 +402,13 @@ def indexer(
 
 def test_parquet_export_is_partial() -> None:
     assert parquet_export_is_partial(
-        "https://hf.co/datasets/c4/resolve/refs%2Fconvert%2Fparquet/en/partial-train/0000.parquet"
+        "https://hf.co/datasets/canonical/resolve/refs%2Fconvert%2Fparquet/en/partial-train/0000.parquet"
     )
     assert not parquet_export_is_partial(
-        "https://hf.co/datasets/bigcode/the-stack/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet"
+        "https://hf.co/datasets/organization/not-canonical/resolve/refs%2Fconvert%2Fparquet/default/train/0000.parquet"
     )
     assert not parquet_export_is_partial(
-        "https://hf.co/datasets/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/train/0000.parquet"
+        "https://hf.co/datasets/rajpurkar/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/train/0000.parquet"
     )
 
 
@@ -494,9 +494,12 @@ def test_indexer_schema_mistmatch_error(
 @pytest.mark.parametrize(
     "parquet_url,expected",
     [
-        ("https://hf.co/datasets/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/train/0000.parquet", "train"),
         (
-            "https://hf.co/datasets/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/partial-test/0000.parquet",
+            "https://hf.co/datasets/rajpurkar/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/train/0000.parquet",
+            "train",
+        ),
+        (
+            "https://hf.co/datasets/rajpurkar/squad/resolve/refs%2Fconvert%2Fparquet/plain_text/partial-test/0000.parquet",
             "partial-test",
         ),
     ],
diff --git a/services/admin/tests/test_app_real.py b/services/admin/tests/test_app_real.py
--- a/services/admin/tests/test_app_real.py
+++ b/services/admin/tests/test_app_real.py
@@ -43,7 +43,7 @@ def test_force_refresh(
     real_app_config: AppConfig,
     real_client: TestClient,
 ) -> None:
-    dataset = "glue"
+    dataset = "nyu-mll/glue"
     first_step = processing_graph.get_processing_steps(order="topological")[0]
     path = first_step.job_type
     response = real_client.request("post", f"/force-refresh/{path}?dataset={dataset}")
diff --git a/services/api/tests/test_app_real.py b/services/api/tests/test_app_real.py
--- a/services/api/tests/test_app_real.py
+++ b/services/api/tests/test_app_real.py
@@ -46,7 +46,7 @@ def test_webhook_untrusted(
 ) -> None:
     payload = {
         "event": "add",
-        "repo": {"type": "dataset", "name": "glue", "gitalyUid": "123", "headSha": "revision"},
+        "repo": {"type": "dataset", "name": "nyu-mll/glue", "gitalyUid": "123", "headSha": "revision"},
         "scope": "repo",
     }
     response = real_client.post("/webhook", json=payload)
@@ -57,7 +57,7 @@ def test_webhook_untrusted(
 def test_webhook_trusted(real_client: TestClient) -> None:
     payload = {
         "event": "add",
-        "repo": {"type": "dataset", "name": "glue", "gitalyUid": "123", "headSha": "revision"},
+        "repo": {"type": "dataset", "name": "nyu-mll/glue", "gitalyUid": "123", "headSha": "revision"},
         "scope": "repo",
     }
     response = real_client.post("/webhook", json=payload, headers={"x-webhook-secret": API_HF_WEBHOOK_SECRET})
diff --git a/services/worker/tests/job_runners/config/test_parquet_and_info.py b/services/worker/tests/job_runners/config/test_parquet_and_info.py
--- a/services/worker/tests/job_runners/config/test_parquet_and_info.py
+++ b/services/worker/tests/job_runners/config/test_parquet_and_info.py
@@ -855,12 +855,12 @@ def test_resolve_trust_remote_code() -> None:
         resolve_trust_remote_code("lhoestq/demo1", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}", "lhoestq/d*"])
         is True
     )
-    assert resolve_trust_remote_code("squad", allow_list=[]) is False
-    assert resolve_trust_remote_code("squad", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}"]) is True
-    assert resolve_trust_remote_code("squad", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}", "lhoestq/s*"]) is True
+    assert resolve_trust_remote_code("mnist", allow_list=[]) is False
+    assert resolve_trust_remote_code("mnist", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}"]) is True
+    assert resolve_trust_remote_code("mnist", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}", "lhoestq/s*"]) is True
     assert (
         resolve_trust_remote_code(
-            "lhoestq/custom_squad", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}", "lhoestq/d*"]
+            "lhoestq/custom_mnist", allow_list=["{{ALL_DATASETS_WITH_NO_NAMESPACE}}", "lhoestq/d*"]
         )
         is False
     )