Repository: useblocks/sphinx-needs
Pull request #1226 (2024-08-21 06:24:15)

🔧 Replace INTERNALS & JSON_KEY_EXCLUSIONS_NEEDS with NeedsCoreFields
This provides a consistent, centralised place to declare all data about core need fields.

Extracted from #1132

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks) Report
Attention: Patch coverage is `96.96970%` with `1 line` in your changes missing coverage. Please review.
> Project coverage is 86.46%. Comparing base [(`2045121`)](https://app.codecov.io/gh/useblocks/sphinx-needs/commit/20451216479b46272121d8ddd7c43a78824aba72?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks) to head [(`dc17b22`)](https://app.codecov.io/gh/useblocks/sphinx-needs/commit/dc17b22f394d72c0e794a52bb497afdc6d65a6e1?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks).

| [Files](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks) | Patch % | Lines |
|---|---|---|
| [sphinx\_needs/data.py](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226?src=pr&el=tree&filepath=sphinx_needs%2Fdata.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks#diff-c3BoaW54X25lZWRzL2RhdGEucHk=) | 96.15% | [1 Missing :warning: ](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #1226      +/-   ##
==========================================
+ Coverage   86.44%   86.46%   +0.01%     
==========================================
  Files          56       56              
  Lines        6501     6510       +9     
==========================================
+ Hits         5620     5629       +9     
  Misses        881      881              
```

| [Flag](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226/flags?src=pr&el=flags&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks) | Coverage Δ | |
|---|---|---|
| [pytests](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226/flags?src=pr&el=flag&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks) | `86.46% <96.96%> (+0.01%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks#carryforward-flags-in-the-pull-request-comment) to find out more.

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/useblocks/sphinx-needs/pull/1226?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=useblocks).


Patch:
diff --git a/sphinx_needs/api/configuration.py b/sphinx_needs/api/configuration.py
index 1197cd8c..ba3c287f 100644
--- a/sphinx_needs/api/configuration.py
+++ b/sphinx_needs/api/configuration.py
@@ -8,11 +8,10 @@ from __future__ import annotations
 
 from typing import Callable
 
-from docutils.parsers.rst import directives
 from sphinx.application import Sphinx
 from sphinx.util.logging import SphinxLoggerAdapter
 
-from sphinx_needs.api.exceptions import NeedsApiConfigException, NeedsApiConfigWarning
+from sphinx_needs.api.exceptions import NeedsApiConfigException
 from sphinx_needs.config import NEEDS_CONFIG, NeedsSphinxConfig
 from sphinx_needs.data import NeedsInfoType
 from sphinx_needs.functions import register_func
@@ -85,7 +84,9 @@ def add_need_type(
     app.add_directive(directive, sphinx_needs.directives.need.NeedDirective)
 
 
-def add_extra_option(app: Sphinx, name: str) -> None:
+def add_extra_option(
+    app: Sphinx, name: str, *, description: str = "Added by add_extra_option API"
+) -> None:
     """
     Adds an extra option to the configuration. This option can then later be used inside needs or ``add_need``.
 
@@ -101,9 +102,7 @@ def add_extra_option(app: Sphinx, name: str) -> None:
     :param name: Name as string of the extra option
     :return: None
     """
-    if name in NEEDS_CONFIG.extra_options:
-        raise NeedsApiConfigWarning(f"Option {name} already registered.")
-    NEEDS_CONFIG.extra_options[name] = directives.unchanged
+    NEEDS_CONFIG.add_extra_option(name, description)
 
 
 def add_dynamic_function(
diff --git a/sphinx_needs/api/exceptions.py b/sphinx_needs/api/exceptions.py
index 1720441a..406e1e50 100644
--- a/sphinx_needs/api/exceptions.py
+++ b/sphinx_needs/api/exceptions.py
@@ -19,10 +19,6 @@ class NeedsNoIdException(SphinxError):
     pass
 
 
-class NeedsDuplicatedId(SphinxError):
-    pass
-
-
 class NeedsStatusNotAllowed(SphinxError):
     pass
 
diff --git a/sphinx_needs/api/need.py b/sphinx_needs/api/need.py
index ccfe6598..d9c88ce2 100644
--- a/sphinx_needs/api/need.py
+++ b/sphinx_needs/api/need.py
@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import os
 import re
+import warnings
 from contextlib import contextmanager
 from typing import Any, Iterator
 
@@ -16,7 +17,6 @@ from sphinx.environment import BuildEnvironment
 from sphinx_needs.api.configuration import NEEDS_CONFIG
 from sphinx_needs.api.exceptions import (
     NeedsConstraintNotAllowed,
-    NeedsDuplicatedId,
     NeedsInvalidException,
     NeedsInvalidOption,
     NeedsNoIdException,
@@ -35,6 +35,8 @@ from sphinx_needs.utils import jinja_parse
 
 logger = get_logger(__name__)
 
+_deprecated_kwargs = {"constraints_passed", "links_string", "hide_tags", "hide_status"}
+
 
 def add_need(
     app: Sphinx,
@@ -50,13 +52,9 @@ def add_need(
     status: str | None = None,
     tags: None | str | list[str] = None,
     constraints: None | str | list[str] = None,
-    constraints_passed: None | bool = None,
-    links_string: None | str | list[str] = None,
     delete: None | bool = False,
     jinja_content: None | bool = False,
     hide: bool = False,
-    hide_tags: bool = False,
-    hide_status: bool = False,
     collapse: None | bool = None,
     style: None | str = None,
     layout: None | str = None,
@@ -136,11 +134,8 @@ def add_need(
     :param tags: Tags as single string.
     :param constraints: Constraints as single, comma separated, string.
     :param constraints_passed: Contains bool describing if all constraints have passed
-    :param links_string: Links as single string. (Not used)
     :param delete: boolean value (Remove the complete need).
     :param hide: boolean value.
-    :param hide_tags: boolean value. (Not used with Sphinx-Needs >0.5.0)
-    :param hide_status: boolean value. (Not used with Sphinx-Needs >0.5.0)
     :param collapse: boolean value.
     :param style: String value of class attribute of node.
     :param layout: String value of layout definition to use
@@ -150,6 +145,13 @@ def add_need(
 
     :return: node
     """
+
+    if kwargs.keys() & _deprecated_kwargs:
+        warnings.warn(
+            "deprecated key found in kwargs", DeprecationWarning, stacklevel=1
+        )
+        kwargs = {k: v for k, v in kwargs.items() if k not in _deprecated_kwargs}
+
     #############################################################################################
     # Get environment
     #############################################################################################
@@ -312,19 +314,24 @@ def add_need(
 
     if need_id in SphinxNeedsData(env).get_or_create_needs():
         if id:
-            raise NeedsDuplicatedId(
-                f"A need with ID {need_id} already exists! "
-                f"This is not allowed. Document {docname}[{lineno}] Title: {title}."
-            )
+            message = f"A need with ID {need_id} already exists, " f"title: {title!r}."
         else:  # this is a generated ID
-            raise NeedsDuplicatedId(
+            _title = " ".join(title)
+            message = (
                 "Needs could not generate a unique ID for a need with "
-                "the title '{}' because another need had the same title. "
+                f"the title {_title!r} because another need had the same title. "
                 "Either supply IDs for the requirements or ensure the "
                 "titles are different.  NOTE: If title is being generated "
                 "from the content, then ensure the first sentence of the "
-                "requirements are different.".format(" ".join(title))
+                "requirements are different."
             )
+        logger.warning(
+            message + " [needs.duplicate_id]",
+            type="needs",
+            subtype="duplicate_id",
+            location=(docname, lineno) if docname else None,
+        )
+        return []
 
     # Trim title if it is too long
     max_length = needs_config.max_title_length
@@ -620,7 +627,6 @@ def add_external_need(
     status: str | None = None,
     tags: str | None = None,
     constraints: str | None = None,
-    links_string: str | None = None,
     **kwargs: Any,
 ) -> list[nodes.Node]:
     """
@@ -640,7 +646,6 @@ def add_external_need(
     :param status: Status as string.
     :param tags: Tags as single string.
     :param constraints: constraints as single, comma separated string.
-    :param links_string: Links as single string.
     :param external_css: CSS class name as string, which is set for the <a> tag.
 
     """
@@ -667,7 +672,6 @@ def add_external_need(
         status=status,
         tags=tags,
         constraints=constraints,
-        links_string=links_string,
         is_external=True,
         external_url=external_url,
         external_css=external_css,
diff --git a/sphinx_needs/config.py b/sphinx_needs/config.py
index 323115b1..a2924238 100644
--- a/sphinx_needs/config.py
+++ b/sphinx_needs/config.py
@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from dataclasses import MISSING, dataclass, field, fields
-from typing import TYPE_CHECKING, Any, Callable, Dict, Literal, TypedDict
+from typing import TYPE_CHECKING, Any, Callable, Dict, Literal, Mapping, TypedDict
 
+from docutils.parsers.rst import directives
 from sphinx.application import Sphinx
 from sphinx.config import Config as _SphinxConfig
 
@@ -15,6 +16,16 @@ if TYPE_CHECKING:
     from sphinx_needs.data import NeedsInfoType
 
 
+@dataclass
+class ExtraOptionParams:
+    """Defines a single extra option for needs"""
+
+    description: str
+    """A description of the option."""
+    validator: Callable[[str | None], str]
+    """A function to validate the directive option value."""
+
+
 class Config:
     """
     Stores sphinx-needs specific configuration values.
@@ -26,7 +37,7 @@ class Config:
     """
 
     def __init__(self) -> None:
-        self._extra_options: dict[str, Callable[[str], Any]] = {}
+        self._extra_options: dict[str, ExtraOptionParams] = {}
         self._warnings: dict[
             str, str | Callable[[NeedsInfoType, SphinxLoggerAdapter], bool]
         ] = {}
@@ -36,7 +47,7 @@ class Config:
         self._warnings = {}
 
     @property
-    def extra_options(self) -> dict[str, Callable[[str], Any]]:
+    def extra_options(self) -> Mapping[str, ExtraOptionParams]:
         """Options that are dynamically added to `NeedDirective` & `NeedserviceDirective`,
         after the config is initialized.
 
@@ -46,6 +57,25 @@ class Config:
         """
         return self._extra_options
 
+    def add_extra_option(
+        self,
+        name: str,
+        description: str,
+        *,
+        validator: Callable[[str | None], str] | None = None,
+        override: bool = False,
+    ) -> None:
+        """Adds an extra option to the configuration."""
+        if not override and name in self._extra_options:
+            from sphinx_needs.api.exceptions import (
+                NeedsApiConfigWarning,  # avoid circular import
+            )
+
+            raise NeedsApiConfigWarning(f"Option {name} already registered.")
+        self._extra_options[name] = ExtraOptionParams(
+            description, directives.unchanged if validator is None else validator
+        )
+
     @property
     def warnings(
         self,
diff --git a/sphinx_needs/data.py b/sphinx_needs/data.py
index a97d76c6..5b83717d 100644
--- a/sphinx_needs/data.py
+++ b/sphinx_needs/data.py
@@ -4,17 +4,22 @@ which is stored in the Sphinx environment.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Literal, TypedDict
+from typing import TYPE_CHECKING, Any, Final, Literal, Mapping, TypedDict
+
+from sphinx.util.logging import getLogger
 
 if TYPE_CHECKING:
     from docutils.nodes import Element, Text
     from sphinx.application import Sphinx
     from sphinx.environment import BuildEnvironment
-    from typing_extensions import Required
+    from typing_extensions import NotRequired, Required
 
     from sphinx_needs.services.manager import ServiceManager
 
 
+LOGGER = getLogger(__name__)
+
+
 class NeedsFilterType(TypedDict):
     filter: str
     """Filter string."""
@@ -40,20 +45,249 @@ class NeedsPartType(TypedDict):
     """List of need IDs, which are referencing this part."""
 
 
+class CoreFieldParameters(TypedDict):
+    """Parameters for core fields."""
+
+    description: str
+    """Description of the field."""
+    schema: dict[str, Any]
+    """JSON schema for the field."""
+    show_in_layout: NotRequired[bool]
+    """Whether to show the field in the rendered layout of the need by default (False if not present)."""
+    exclude_json: NotRequired[bool]
+    """Whether to exclude the field from the JSON representation (False if not present)."""
+
+
+NeedsCoreFields: Final[Mapping[str, CoreFieldParameters]] = {
+    "target_id": {"description": "ID of the data.", "schema": {"type": "string"}},
+    "id": {"description": "ID of the data.", "schema": {"type": "string"}},
+    "docname": {
+        "description": "Name of the document where the need is defined (None if external).",
+        "schema": {"type": ["string", "null"]},
+    },
+    "lineno": {
+        "description": "Line number where the need is defined (None if external).",
+        "schema": {"type": ["integer", "null"]},
+        "exclude_json": True,
+    },
+    "lineno_content": {
+        "description": "Line number on which the need content starts (None if external).",
+        "schema": {"type": ["integer", "null"]},
+        "exclude_json": True,
+    },
+    "full_title": {
+        "description": "Title of the need, of unlimited length.",
+        "schema": {"type": "string"},
+    },
+    "title": {
+        "description": "Title of the need, trimmed to a maximum length.",
+        "schema": {"type": "string"},
+    },
+    "status": {
+        "description": "Status of the need.",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "tags": {
+        "description": "List of tags.",
+        "schema": {"type": "array", "items": {"type": "string"}},
+        "show_in_layout": True,
+    },
+    "collapse": {
+        "description": "Hide the meta-data information of the need.",
+        "schema": {"type": ["boolean", "null"]},
+        "exclude_json": True,
+    },
+    "hide": {
+        "description": "If true, the need is not rendered.",
+        "schema": {"type": "boolean"},
+        "exclude_json": True,
+    },
+    "delete": {
+        "description": "If true, the need is deleted entirely.",
+        "schema": {"type": "boolean"},
+        "show_in_layout": True,
+    },
+    "layout": {
+        "description": "Key of the layout, which is used to render the need.",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "style": {
+        "description": "Comma-separated list of CSS classes (all appended by `needs_style_`).",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "arch": {
+        "description": "Mapping of uml key to uml content.",
+        "schema": {"type": "object", "additionalProperties": {"type": "string"}},
+    },
+    "is_external": {
+        "description": "If true, no node is created and need is referencing external url.",
+        "schema": {"type": "boolean"},
+    },
+    "external_url": {
+        "description": "URL of the need, if it is an external need.",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "external_css": {
+        "description": "CSS class name, added to the external reference.",
+        "schema": {"type": "string"},
+    },
+    "type": {"description": "Type of the need.", "schema": {"type": "string"}},
+    "type_name": {"description": "Name of the type.", "schema": {"type": "string"}},
+    "type_prefix": {
+        "description": "Prefix of the type.",
+        "schema": {"type": "string"},
+        "exclude_json": True,
+    },
+    "type_color": {
+        "description": "Hexadecimal color code of the type.",
+        "schema": {"type": "string"},
+        "exclude_json": True,
+    },
+    "type_style": {
+        "description": "Style of the type.",
+        "schema": {"type": "string"},
+        "exclude_json": True,
+    },
+    "is_modified": {
+        "description": "Whether the need was modified by needextend.",
+        "schema": {"type": "boolean"},
+    },
+    "modifications": {
+        "description": "Number of modifications by needextend.",
+        "schema": {"type": "integer"},
+    },
+    "is_need": {
+        "description": "Whether the need is a need.",
+        "schema": {"type": "boolean"},
+    },
+    "is_part": {
+        "description": "Whether the need is a part.",
+        "schema": {"type": "boolean"},
+    },
+    "parts": {
+        "description": "Mapping of parts, a.k.a. sub-needs, IDs to data that overrides the need's data",
+        "schema": {"type": "object", "additionalProperties": {"type": "object"}},
+    },
+    "id_parent": {
+        "description": "<parent ID>, or <self ID> if not a part.",
+        "exclude_json": True,
+        "schema": {"type": "string"},
+    },
+    "id_complete": {
+        "description": "<parent ID>.<self ID>, or <self ID> if not a part.",
+        "exclude_json": True,
+        "schema": {"type": "string"},
+    },
+    "jinja_content": {
+        "description": "Whether the content should be pre-processed by jinja.",
+        "schema": {"type": "boolean"},
+        "show_in_layout": True,
+    },
+    "template": {
+        "description": "Template of the need.",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "pre_template": {
+        "description": "Pre-template of the need.",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "post_template": {
+        "description": "Post-template of the need.",
+        "schema": {"type": ["string", "null"]},
+        "show_in_layout": True,
+    },
+    "content": {
+        "description": "Content of the need.",
+        "schema": {"type": "string"},
+        "exclude_json": True,
+    },
+    "pre_content": {
+        "description": "Pre-content of the need.",
+        "schema": {"type": "string"},
+    },
+    "post_content": {
+        "description": "Post-content of the need.",
+        "schema": {"type": "string"},
+    },
+    "content_id": {
+        "description": "ID of the content node.",
+        "schema": {"type": ["string", "null"]},
+    },
+    "content_node": {
+        "description": "Deep copy of the content node.",
+        "schema": {},
+        "exclude_json": True,
+    },
+    "has_dead_links": {
+        "description": "True if any links reference need ids that are not found in the need list.",
+        "schema": {"type": "boolean"},
+    },
+    "has_forbidden_dead_links": {
+        "description": "True if any links reference need ids that are not found in the need list, and the link type does not allow dead links.",
+        "schema": {"type": "boolean"},
+    },
+    "constraints": {
+        "description": "List of constraint names, which are defined for this need.",
+        "schema": {"type": "array", "items": {"type": "string"}},
+    },
+    "constraints_results": {
+        "description": "Mapping of constraint name, to check name, to result.",
+        "schema": {"type": "object", "additionalProperties": {"type": "object"}},
+    },
+    "constraints_passed": {
+        "description": "True if all constraints passed, False if any failed, None if not yet checked.",
+        "schema": {"type": ["boolean", "null"]},
+    },
+    "constraints_error": {
+        "description": "An error message set if any constraint failed, and `error_message` field is set in config.",
+        "schema": {"type": "string"},
+        "show_in_layout": True,
+    },
+    "doctype": {
+        "description": "Type of the document where the need is defined, e.g. '.rst'.",
+        "schema": {"type": "string"},
+    },
+    "sections": {
+        "description": "Sections of the need.",
+        "schema": {"type": "array", "items": {"type": "string"}},
+    },
+    "section_name": {
+        "description": "Simply the first section.",
+        "schema": {"type": ["string", "null"]},
+    },
+    "signature": {
+        "description": "Derived from a docutils desc_name node.",
+        "schema": {"type": "string"},
+        "show_in_layout": True,
+    },
+    "parent_need": {
+        "description": "Simply the first parent id.",
+        "schema": {"type": "string"},
+    },
+}
+
+
 class NeedsInfoType(TypedDict, total=False):
     """Data for a single need."""
 
+    # TODO remove duplication target_id/id
     target_id: Required[str]
     """ID of the data."""
     id: Required[str]
-    """ID of the data (same as target_id)"""
+    """ID of the data."""
 
     docname: Required[str | None]
-    """Name of the document where the need is defined (None if external)"""
+    """Name of the document where the need is defined (None if external)."""
     lineno: Required[int | None]
-    """Line number where the need is defined (None if external)"""
+    """Line number where the need is defined (None if external)."""
     lineno_content: Required[int | None]
-    """Line number on which the need content starts (None if external)"""
+    """Line number on which the need content starts (None if external)."""
 
     # meta information
     full_title: Required[str]
@@ -65,7 +299,7 @@ class NeedsInfoType(TypedDict, total=False):
 
     # rendering information
     collapse: Required[None | bool]
-    """hide the meta-data information of the need."""
+    """Hide the meta-data information of the need."""
     hide: Required[bool]
     """If true, the need is not rendered."""
     delete: Required[None | bool]
@@ -81,7 +315,7 @@ class NeedsInfoType(TypedDict, total=False):
 
     # external reference information
     is_external: Required[bool]
-    """If true, no node is created and need is referencing external url"""
+    """If true, no node is created and need is referencing external url."""
     external_url: Required[None | str]
     """URL of the need, if it is an external need."""
     external_css: Required[str]
@@ -107,12 +341,9 @@ class NeedsInfoType(TypedDict, total=False):
     parts: Required[dict[str, NeedsPartType]]
     # additional information required for compatibility with parts
     id_parent: Required[str]
-    """ID of the parent need, or self ID if not a part"""
+    """<parent ID>, or <self ID> if not a part."""
     id_complete: Required[str]
-    """ID of the parent need, followed by the part ID, 
-    delimited by a dot: ``<id_parent>.<id>``,
-    or self ID if not a part
-    """
+    """<parent ID>.<self ID>, or <self ID> if not a part."""
 
     # content creation information
     jinja_content: Required[None | bool]
@@ -123,9 +354,9 @@ class NeedsInfoType(TypedDict, total=False):
     pre_content: str
     post_content: str
     content_id: Required[None | str]
-    """ID of the content node (set after parsing)."""
+    """ID of the content node."""
     content_node: Required[None | Element]
-    """deep copy of the content node (set after parsing)."""
+    """Deep copy of the content node."""
 
     # these default to False and are updated in check_links post-process
     has_dead_links: Required[bool]
@@ -148,15 +379,15 @@ class NeedsInfoType(TypedDict, total=False):
 
     # additional source information
     doctype: Required[str]
-    """Type of the document where the need is defined, e.g. '.rst'"""
+    """Type of the document where the need is defined, e.g. '.rst'."""
     # set in analyse_need_locations transform
     sections: Required[list[str]]
     section_name: Required[str]
-    """Simply the first section"""
+    """Simply the first section."""
     signature: Required[str | Text]
-    """Derived from a docutils desc_name node"""
+    """Derived from a docutils desc_name node."""
     parent_need: Required[str]
-    """Simply the first parent id"""
+    """Simply the first parent id."""
 
     # link information
     # Note, there is more dynamically added link information;
@@ -612,7 +843,7 @@ class SphinxNeedsData:
 
 
 def merge_data(
-    _app: Sphinx, env: BuildEnvironment, _docnames: list[str], other: BuildEnvironment
+    _app: Sphinx, env: BuildEnvironment, docnames: list[str], other: BuildEnvironment
 ) -> None:
     """
     Performs data merge of parallel executed workers.
@@ -621,13 +852,32 @@ def merge_data(
     Needs to update env manually for all data Sphinx-Needs collect during read phase
     """
 
-    # Update global needs dict
-    needs = SphinxNeedsData(env).get_or_create_needs()
-    other_needs = SphinxNeedsData(other).get_or_create_needs()
-    needs.update(other_needs)
     if SphinxNeedsData(other).has_export_filters:
         SphinxNeedsData(env).has_export_filters = True
 
+    # Update needs
+    needs = SphinxNeedsData(env).get_or_create_needs()
+    other_needs = SphinxNeedsData(other).get_or_create_needs()
+    for other_id, other_need in other_needs.items():
+        if other_id in needs:
+            # we only want to warn if the need comes from one of the docs parsed in this worker
+            _docname = other_need["docname"]
+            if _docname in docnames:
+                message = (
+                    f"A need with ID {other_id} already exists, "
+                    f"title: {other_need['title']!r}."
+                )
+                LOGGER.warning(
+                    message + " [needs.duplicate_id]",
+                    type="needs",
+                    subtype="duplicate_id",
+                    location=(_docname, other_need["lineno"]) if _docname else None,
+                )
+        else:
+            needs[other_id] = other_need
+
+    # update other data
+
     def _merge(name: str, is_complex_dict: bool = False) -> None:
         # Update global needs dict
         if not hasattr(env, name):
diff --git a/sphinx_needs/external_needs.py b/sphinx_needs/external_needs.py
index eaddfe78..3ecc1f3f 100644
--- a/sphinx_needs/external_needs.py
+++ b/sphinx_needs/external_needs.py
@@ -11,7 +11,6 @@ from sphinx.application import Sphinx
 from sphinx.environment import BuildEnvironment
 
 from sphinx_needs.api import add_external_need, del_need
-from sphinx_needs.api.exceptions import NeedsDuplicatedId
 from sphinx_needs.config import NeedsSphinxConfig
 from sphinx_needs.data import SphinxNeedsData
 from sphinx_needs.logging import get_logger
@@ -30,7 +29,7 @@ def get_target_template(target_url: str) -> Template:
     return mem_template
 
 
-def load_external_needs(app: Sphinx, env: BuildEnvironment, _docname: str) -> None:
+def load_external_needs(app: Sphinx, env: BuildEnvironment, docname: str) -> None:
     """Load needs from configured external sources."""
     needs_config = NeedsSphinxConfig(app.config)
     for source in needs_config.external_needs:
@@ -159,10 +158,14 @@ def load_external_needs(app: Sphinx, env: BuildEnvironment, _docname: str) -> No
                     # delete the already existing external need from api need
                     del_need(app, ext_need_id)
                 else:
-                    raise NeedsDuplicatedId(
-                        f'During external needs handling, an identical ID was detected: {ext_need_id} \
-                            from needs_external_needs url: {source["base_url"]}'
+                    log.warning(
+                        f'During external needs handling, an identical ID was detected: {ext_need_id} '
+                        f'from needs_external_needs url: {source["base_url"]} [needs.duplicate_id]',
+                        type="needs",
+                        subtype="duplicate_id",
+                        location=docname if docname else None,
                     )
+                    return None
 
             add_external_need(app, **need_params)
 
diff --git a/sphinx_needs/layout.py b/sphinx_needs/layout.py
index f4bbc5a9..f766f96d 100644
--- a/sphinx_needs/layout.py
+++ b/sphinx_needs/layout.py
@@ -28,9 +28,9 @@ from sphinx.environment.collectors.asset import DownloadFileCollector, ImageColl
 from sphinx.util.logging import getLogger
 
 from sphinx_needs.config import NeedsSphinxConfig
-from sphinx_needs.data import NeedsInfoType, SphinxNeedsData
+from sphinx_needs.data import NeedsCoreFields, NeedsInfoType, SphinxNeedsData
 from sphinx_needs.debug import measure_time
-from sphinx_needs.utils import INTERNALS, match_string_link
+from sphinx_needs.utils import match_string_link
 
 LOGGER = getLogger(__name__)
 
@@ -734,7 +734,11 @@ class LayoutHandler:
         :param show_empty: If true, also need data with no value will be printed. Mostly useful for debugging.
         :return: docutils nodes
         """
-        default_excludes = list(INTERNALS)
+        default_excludes = [
+            name
+            for name, props in NeedsCoreFields.items()
+            if not props.get("show_in_layout")
+        ]
 
         if exclude is None or not isinstance(exclude, list):
             if defaults:
diff --git a/sphinx_needs/needs.py b/sphinx_needs/needs.py
index 21192597..b8501902 100644
--- a/sphinx_needs/needs.py
+++ b/sphinx_needs/needs.py
@@ -20,7 +20,7 @@ from sphinx_needs.builder import (
     build_needumls_pumls,
 )
 from sphinx_needs.config import NEEDS_CONFIG, LinkOptionsType, NeedsSphinxConfig
-from sphinx_needs.data import SphinxNeedsData, merge_data
+from sphinx_needs.data import NeedsCoreFields, SphinxNeedsData, merge_data
 from sphinx_needs.defaults import (
     LAYOUTS,
     NEED_DEFAULT_OPTIONS,
@@ -103,7 +103,7 @@ from sphinx_needs.roles.need_part import NeedPart, process_need_part
 from sphinx_needs.roles.need_ref import NeedRef, process_need_ref
 from sphinx_needs.services.github import GithubService
 from sphinx_needs.services.open_needs import OpenNeedsService
-from sphinx_needs.utils import INTERNALS, NEEDS_FUNCTIONS, node_match
+from sphinx_needs.utils import NEEDS_FUNCTIONS, node_match
 from sphinx_needs.warnings import process_warnings
 
 __version__ = VERSION = "2.1.0"
@@ -371,12 +371,18 @@ def load_config(app: Sphinx, *_args: Any) -> None:
                 type="needs",
                 subtype="config",
             )
-        NEEDS_CONFIG.extra_options[option] = directives.unchanged
+        NEEDS_CONFIG.add_extra_option(
+            option, "Added by needs_extra_options config", override=True
+        )
 
     # ensure options for ``needgantt`` functionality are added to the extra options
     for option in (needs_config.duration_option, needs_config.completion_option):
         if option not in NEEDS_CONFIG.extra_options:
-            NEEDS_CONFIG.extra_options[option] = directives.unchanged_required
+            NEEDS_CONFIG.add_extra_option(
+                option,
+                "Added for needgantt functionality",
+                validator=directives.unchanged_required,
+            )
 
     # Get extra links and create a dictionary of needed options.
     extra_links_raw = needs_config.extra_links
@@ -388,8 +394,12 @@ def load_config(app: Sphinx, *_args: Any) -> None:
     title_from_content = needs_config.title_from_content
 
     # Update NeedDirective to use customized options
-    NeedDirective.option_spec.update(NEEDS_CONFIG.extra_options)
-    NeedserviceDirective.option_spec.update(NEEDS_CONFIG.extra_options)
+    NeedDirective.option_spec.update(
+        {k: v.validator for k, v in NEEDS_CONFIG.extra_options.items()}
+    )
+    NeedserviceDirective.option_spec.update(
+        {k: v.validator for k, v in NEEDS_CONFIG.extra_options.items()}
+    )
 
     # Update NeedDirective to use customized links
     NeedDirective.option_spec.update(extra_links)
@@ -433,8 +443,8 @@ def load_config(app: Sphinx, *_args: Any) -> None:
     for key, value in NEEDS_CONFIG.extra_options.items():
         NeedextendDirective.option_spec.update(
             {
-                key: value,
-                f"+{key}": value,
+                key: value.validator,
+                f"+{key}": value.validator,
                 f"-{key}": directives.flag,
             }
         )
@@ -585,7 +595,7 @@ def check_configuration(_app: Sphinx, config: Config) -> None:
             )
 
     # Check for usage of internal names
-    for internal in INTERNALS:
+    for internal in NeedsCoreFields:
         if internal in extra_options:
             raise NeedsConfigException(
                 f'Extra option "{internal}" already used internally. '
diff --git a/sphinx_needs/needsfile.py b/sphinx_needs/needsfile.py
index 0c9e961c..c5c2960e 100644
--- a/sphinx_needs/needsfile.py
+++ b/sphinx_needs/needsfile.py
@@ -16,7 +16,7 @@ from jsonschema import Draft7Validator
 from sphinx.config import Config
 
 from sphinx_needs.config import NeedsSphinxConfig
-from sphinx_needs.data import NeedsFilterType, NeedsInfoType
+from sphinx_needs.data import NeedsCoreFields, NeedsFilterType, NeedsInfoType
 from sphinx_needs.logging import get_logger
 
 log = get_logger(__name__)
@@ -24,20 +24,7 @@ log = get_logger(__name__)
 
 class NeedsList:
     JSON_KEY_EXCLUSIONS_NEEDS = {
-        "links_back",
-        "type_color",
-        "hide",
-        "type_prefix",
-        "lineno",
-        "lineno_content",
-        "collapse",
-        "type_style",
-        "content",
-        "content_node",
-        # id_parent, id_parent are added on calls to `prepare_need_list`
-        # but are only relevant to parts
-        "id_parent",
-        "id_complete",
+        name for name, params in NeedsCoreFields.items() if params.get("exclude_json")
     }
 
     JSON_KEY_EXCLUSIONS_FILTERS = {
diff --git a/sphinx_needs/services/manager.py b/sphinx_needs/services/manager.py
index b95962a2..d993824f 100644
--- a/sphinx_needs/services/manager.py
+++ b/sphinx_needs/services/manager.py
@@ -2,7 +2,6 @@ from __future__ import annotations
 
 from typing import Any
 
-from docutils.parsers.rst import directives
 from sphinx.application import Sphinx
 
 from sphinx_needs.api.configuration import NEEDS_CONFIG
@@ -32,10 +31,12 @@ class ServiceManager:
         for option in klass.options:
             if option not in NEEDS_CONFIG.extra_options:
                 self.log.debug(f'Register option "{option}" for service "{name}"')
-                NEEDS_CONFIG.extra_options[option] = directives.unchanged
+                NEEDS_CONFIG.add_extra_option(option, f"Added by service {name}")
                 # Register new option directly in Service directive, as its class options got already
                 # calculated
-                NeedserviceDirective.option_spec[option] = directives.unchanged
+                NeedserviceDirective.option_spec[option] = NEEDS_CONFIG.extra_options[
+                    option
+                ].validator
 
         # Init service with custom config
         self.services[name] = klass(self.app, name, config, **kwargs)
diff --git a/sphinx_needs/utils.py b/sphinx_needs/utils.py
index 3ce30797..cdf4d2fa 100644
--- a/sphinx_needs/utils.py
+++ b/sphinx_needs/utils.py
@@ -39,50 +39,6 @@ class NeedFunctionsType(TypedDict):
 
 NEEDS_FUNCTIONS: dict[str, NeedFunctionsType] = {}
 
-# List of internal need option names. They should not be used by or presented to user.
-INTERNALS = (
-    "docname",
-    "doctype",
-    "lineno",
-    "refid",
-    "content",
-    "lineno_content",
-    "pre_content",
-    "post_content",
-    "collapse",
-    "parts",
-    "id_parent",
-    "id_complete",
-    "title",
-    "full_title",
-    "is_part",
-    "is_need",
-    "type_prefix",
-    "type_color",
-    "type_style",
-    "type",
-    "type_name",
-    "id",
-    "hide",
-    "sections",
-    "section_name",
-    "content_node",
-    "content_id",
-    # "parent_needs",
-    "parent_need",
-    # "child_needs",
-    "is_external",
-    "external_css",
-    "is_modified",
-    "modifications",
-    "constraints",
-    "constraints_passed",
-    "constraints_results",
-    "arch",
-    "target_id",
-    "has_dead_links",
-    "has_forbidden_dead_links",
-)
 
 MONTH_NAMES = [
     "January",


Test patch:
diff --git a/tests/doc_test/parallel_doc/index.rst b/tests/doc_test/parallel_doc/index.rst
index 4c318ed9..8b29080b 100644
--- a/tests/doc_test/parallel_doc/index.rst
+++ b/tests/doc_test/parallel_doc/index.rst
@@ -7,6 +7,7 @@ PARALLEL TEST DOCUMENT
    page_2
    page_3
    page_4
+   page_5
 
 .. spec:: Command line interface
     :id: SP_TOO_001
diff --git a/tests/doc_test/parallel_doc/page_5.rst b/tests/doc_test/parallel_doc/page_5.rst
new file mode 100644
index 00000000..602d47c9
--- /dev/null
+++ b/tests/doc_test/parallel_doc/page_5.rst
@@ -0,0 +1,5 @@
+Page 5
+======
+
+.. story:: duplicate
+   :id: STORY_PAGE_1
diff --git a/tests/test_broken_doc.py b/tests/test_broken_doc.py
index d203541e..1eddf7f6 100644
--- a/tests/test_broken_doc.py
+++ b/tests/test_broken_doc.py
@@ -1,18 +1,26 @@
-import pytest
+import os
 
-from sphinx_needs.api.need import NeedsDuplicatedId
+import pytest
+from sphinx.testing.util import SphinxTestApp
+from sphinx.util.console import strip_colors
 
 
 @pytest.mark.parametrize(
     "test_app",
-    [{"buildername": "html", "srcdir": "doc_test/broken_doc"}],
+    [{"buildername": "html", "srcdir": "doc_test/broken_doc", "no_plantuml": True}],
     indirect=True,
 )
-def test_doc_build_html(test_app):
-    with pytest.raises(NeedsDuplicatedId):
-        app = test_app
-
-        app.build()
-        html = (app.outdir / "index.html").read_text()
-        assert "<h1>BROKEN DOCUMENT" in html
-        assert "SP_TOO_001" in html
+def test_doc_build_html(test_app: SphinxTestApp):
+    test_app.build()
+    warnings = (
+        strip_colors(test_app._warning.getvalue())
+        .replace(str(test_app.srcdir) + os.path.sep, "<srcdir>/")
+        .strip()
+    )
+    assert (
+        warnings
+        == "<srcdir>/index.rst:11: WARNING: A need with ID SP_TOO_001 already exists, title: 'Command line interface'. [needs.duplicate_id]"
+    )
+    html = (test_app.outdir / "index.html").read_text()
+    assert "<h1>BROKEN DOCUMENT" in html
+    assert "SP_TOO_001" in html
diff --git a/tests/test_data.py b/tests/test_data.py
new file mode 100644
index 00000000..e7f38c0d
--- /dev/null
+++ b/tests/test_data.py
@@ -0,0 +1,30 @@
+import ast
+import inspect
+
+from sphinx_needs.data import NeedsCoreFields, NeedsInfoType
+
+
+def test_consistent():
+    """
+    Ideally NeedsCoreFields and NeedsInfoType would be merged, so there is no duplication,
+    but I'm not sure this is possible (to encode both the static and dynamic data required).
+    So at least here, we check that they are consistent with each other.
+    """
+    assert set(NeedsCoreFields).issubset(set(NeedsInfoType.__annotations__))
+
+    source = inspect.getsource(NeedsInfoType)
+    klass = ast.parse(source).body[0]
+    descriptions = {}
+    for i, node in enumerate(klass.body):
+        if (
+            isinstance(node, ast.AnnAssign)
+            and len(klass.body) > i + 1
+            and isinstance(klass.body[i + 1], ast.Expr)
+        ):
+            desc = " ".join(
+                [li.strip() for li in klass.body[i + 1].value.value.splitlines()]
+            )
+            descriptions[node.target.id] = desc.strip()
+    for field, desc in descriptions.items():
+        if field in NeedsCoreFields:
+            assert NeedsCoreFields[field]["description"] == desc, field
diff --git a/tests/test_parallel_execution.py b/tests/test_parallel_execution.py
index 44164ffc..9f2bf042 100644
--- a/tests/test_parallel_execution.py
+++ b/tests/test_parallel_execution.py
@@ -1,31 +1,38 @@
+import os
 from pathlib import Path
 
 import pytest
+from sphinx.util.console import strip_colors
 
 
 @pytest.mark.parametrize(
     "test_app",
-    [{"buildername": "html", "srcdir": "doc_test/parallel_doc", "parallel": 4}],
+    [
+        {
+            "buildername": "html",
+            "srcdir": "doc_test/parallel_doc",
+            "parallel": 4,
+            "no_plantuml": True,
+        }
+    ],
     indirect=True,
 )
 def test_doc_build_html(test_app):
     app = test_app
     app.build()
-    html = Path(app.outdir, "index.html").read_text()
-    assert app.statuscode == 0
-    assert "<h1>PARALLEL TEST DOCUMENT" in html
-    assert "SP_TOO_001" in html
-
+    warnings = (
+        strip_colors(app._warning.getvalue())
+        .replace(str(app.srcdir) + os.path.sep, "<srcdir>/")
+        .strip()
+    )
+    assert (
+        warnings
+        == "<srcdir>/page_5.rst:4: WARNING: A need with ID STORY_PAGE_1 already exists, title: 'duplicate'. [needs.duplicate_id]"
+    )
 
-@pytest.mark.parametrize(
-    "test_app",
-    [{"buildername": "html", "srcdir": "doc_test/parallel_doc", "parallel": 4}],
-    indirect=True,
-)
-def test_doc_parallel_build_html(test_app):
-    app = test_app
-    app.build()
-    assert app.statuscode == 0
+    index_html = Path(app.outdir, "index.html").read_text()
+    assert "<h1>PARALLEL TEST DOCUMENT" in index_html
+    assert "SP_TOO_001" in index_html
 
     page3_html = Path(app.outdir, "page_3.html").read_text()
     assert "Page 3" in page3_html