Repository: huggingface/dataset-viewer

Issue:
Avoid collisions in unicity_id field in the jobs collection
See https://github.com/huggingface/datasets-server/pull/837#pullrequestreview-1315579718


Discussion:
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/datasets-server/blob/main/CONTRIBUTING.md) are likely to be ignored.
Very minor, but lets try to fix it one day (or remove unicity_id altogether)
This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the [contributing guidelines](https://github.com/huggingface/datasets-server/blob/main/CONTRIBUTING.md) are likely to be ignored.

Resolving patch:
diff --git a/libs/libcommon/src/libcommon/queue.py b/libs/libcommon/src/libcommon/queue.py
--- a/libs/libcommon/src/libcommon/queue.py
+++ b/libs/libcommon/src/libcommon/queue.py
@@ -17,7 +17,7 @@
 from mongoengine.queryset.queryset import QuerySet
 
 from libcommon.constants import QUEUE_MONGOENGINE_ALIAS, QUEUE_TTL_SECONDS
-from libcommon.utils import get_datetime
+from libcommon.utils import get_datetime, inputs_to_string
 
 # START monkey patching ### hack ###
 # see https://github.com/sbdchd/mongo-types#install
@@ -245,7 +245,7 @@ def _add_job(
             dataset=dataset,
             config=config,
             split=split,
-            unicity_id=f"Job[{job_type}][{dataset}][{config}][{split}]",
+            unicity_id=inputs_to_string(dataset=dataset, config=config, split=split, prefix=job_type),
             namespace=dataset.split("/")[0],
             force=force,
             priority=priority,
diff --git a/libs/libcommon/src/libcommon/state.py b/libs/libcommon/src/libcommon/state.py
--- a/libs/libcommon/src/libcommon/state.py
+++ b/libs/libcommon/src/libcommon/state.py
@@ -17,6 +17,7 @@
     get_response,
     get_response_metadata,
 )
+from libcommon.utils import inputs_to_string
 
 # TODO: use the term Artifact elsewhere in the code (a Step should produce one or several Artifacts, depending on the
 # input level: one, one per dataset, one per config, or one per split)
@@ -154,7 +155,7 @@ def __post_init__(self) -> None:
                 raise ValueError("Step input type is split, but config or split is None")
         else:
             raise ValueError(f"Invalid step input type: {self.step.input_type}")
-        self.id = ",".join([p for p in (self.step.name, self.dataset, self.config, self.split) if p])
+        self.id = inputs_to_string(dataset=self.dataset, config=self.config, split=self.split, prefix=self.step.name)
 
         self.job_state = JobState(
             job_type=self.step.job_type, dataset=self.dataset, config=self.config, split=self.split
diff --git a/libs/libcommon/src/libcommon/utils.py b/libs/libcommon/src/libcommon/utils.py
--- a/libs/libcommon/src/libcommon/utils.py
+++ b/libs/libcommon/src/libcommon/utils.py
@@ -3,7 +3,7 @@
 
 import base64
 from datetime import datetime, timezone
-from typing import Any
+from typing import Any, Optional
 
 import orjson
 
@@ -24,3 +24,19 @@ def orjson_dumps(content: Any) -> bytes:
 
 def get_datetime() -> datetime:
     return datetime.now(timezone.utc)
+
+
+def inputs_to_string(
+    dataset: str,
+    config: Optional[str] = None,
+    split: Optional[str] = None,
+    prefix: Optional[str] = None,
+) -> str:
+    result = dataset
+    if config is not None:
+        result = f"{result},{config}"
+        if split is not None:
+            result = f"{result},{split}"
+    if prefix is not None:
+        result = f"{prefix},{result}"
+    return result


Test patch:
diff --git a/libs/libcommon/tests/test_utils.py b/libs/libcommon/tests/test_utils.py
new file mode 100644
--- /dev/null
+++ b/libs/libcommon/tests/test_utils.py
@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2022 The HuggingFace Authors.
+
+import pytest
+
+from libcommon.utils import inputs_to_string
+
+
+@pytest.mark.parametrize(
+    "dataset,config,split,prefix,expected",
+    [
+        ("dataset", None, None, None, "dataset"),
+        ("dataset", "config", None, None, "dataset,config"),
+        ("dataset", None, "split", None, "dataset"),
+        ("dataset", "config", "split", None, "dataset,config,split"),
+        ("dataset", None, None, "prefix", "prefix,dataset"),
+        ("dataset", "config", "split", "prefix", "prefix,dataset,config,split"),
+    ],
+)
+def test_inputs_to_string(dataset: str, config: str, split: str, prefix: str, expected: str) -> None:
+    result = inputs_to_string(dataset=dataset, config=config, split=split, prefix=prefix)
+    assert result == expected