Repository: sphinx-doc/sphinx

Issue:
Enum values inside `typing.Literal` render weirdly
### Describe the bug

While `typing.Literal` is picked up properly by autodoc when it just contains a string, when the contained value is an enum variant, the output is very messy.

For some reason, it shows `~typing.Literal`, rather than just `Literal`, and the enum variant seems to just be the output of  `repr(SomeEnum.X)`, which includes the value of that enum, which isn't something we'd generally need to see. It also doesn't produce a clickable reference to the specific variant (or at least to the enum class).

While using `repr` works well when the Literal just contains strings, it looks very weird for enum variants, and clearly seems to be a bug.

Here's how the documentation would look from the minimal reproducable example below:

![image](https://github.com/sphinx-doc/sphinx/assets/20902250/e266ac72-9436-44cd-b586-4025519104d4)

<details>
<summary>
And here's another screenshot of how this might look from an actual project, to demonstrate how "cluttery" this can get:
</summary>

![image](https://github.com/sphinx-doc/sphinx/assets/20902250/29f51050-37d9-433a-af96-ae9724ad6c8b)
</details>

### How to Reproduce

```bash
python -m venv .venv
emulate bash -c '. .venv/bin/activate'
pip install sphinx
mkdir src
cat > src/example.py <<EOF
from typing import Literal
from enum import Enum

def works_great(x: Literal["foo"]) -> None:
    """Does stuff"""

class SomeEnum(str, Enum):
    X = "x"
    Y = "y"
    Z = "z"

def problematic_func(x: Literal[SomeEnum.X]) -> None:
    """Also does stuff"""
EOF
cat > src/conf.py <<EOF
import os, sys

sys.path.insert(0, os.path.abspath("."))

extensions = ["sphinx.ext.autodoc"]
EOF
cat > src/index.rst <<EOF
API reference
=============

.. automodule:: example
    :members:
    :undoc-members:
EOF
sphinx-build -b html src _build
xdg-open _build/index.html
```

### Environment Information

```text
Platform:              linux; (Linux-6.3.7-arch1-1-x86_64-with-glibc2.37)
Python version:        3.11.3 (main, Jun  5 2023, 09:32:32) [GCC 13.1.1 20230429])
Python implementation: CPython
Sphinx version:        7.0.1
Docutils version:      0.20.1
Jinja2 version:        3.1.2
Pygments version:      2.15.1
```


### Sphinx extensions

```python
["sphinx.ext.autodoc"]
```


### Additional context

_No response_

Discussion:
picnixz: I'm converting into a draft because I don't know which whould be the best course of action. 

I'll present the different possibilities on the issue (let's discuss there).

Patch:
diff --git a/CHANGES b/CHANGES
index 8645ac664..aa8cefcbc 100644
--- a/CHANGES
+++ b/CHANGES
@@ -77,6 +77,9 @@ Bugs fixed
   Patch by Bénédikt Tran.
 * #10930: Highlight all search terms on the search results page.
   Patch by Dmitry Shachnev.
+* #11473: Type annotations containing :py:data:`~typing.Literal` enumeration
+  values now render correctly.
+  Patch by Bénédikt Tran.
 
 Testing
 -------
diff --git a/sphinx/util/typing.py b/sphinx/util/typing.py
index 6619384a1..171420df5 100644
--- a/sphinx/util/typing.py
+++ b/sphinx/util/typing.py
@@ -7,11 +7,14 @@
 from collections.abc import Sequence
 from struct import Struct
 from types import TracebackType
-from typing import Any, Callable, ForwardRef, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Callable, ForwardRef, TypeVar, Union
 
 from docutils import nodes
 from docutils.parsers.rst.states import Inliner
 
+if TYPE_CHECKING:
+    import enum
+
 try:
     from types import UnionType  # type: ignore[attr-defined] # python 3.10 or above
 except ImportError:
@@ -186,7 +189,14 @@ def restify(cls: type | None, mode: str = 'fully-qualified-except-typing') -> st
                 args = ', '.join(restify(a, mode) for a in cls.__args__[:-1])
                 text += fr"\ [[{args}], {restify(cls.__args__[-1], mode)}]"
             elif cls.__module__ == 'typing' and getattr(origin, '_name', None) == 'Literal':
-                text += r"\ [%s]" % ', '.join(repr(a) for a in cls.__args__)
+                literal_args = []
+                for a in cls.__args__:
+                    if inspect.isenumattribute(a):
+                        literal_args.append(_format_literal_enum_arg(a, mode=mode))
+                    else:
+                        literal_args.append(repr(a))
+                text += r"\ [%s]" % ', '.join(literal_args)
+                del literal_args
             elif cls.__args__:
                 text += r"\ [%s]" % ", ".join(restify(a, mode) for a in cls.__args__)
 
@@ -338,7 +348,21 @@ def stringify_annotation(
             returns = stringify_annotation(annotation_args[-1], mode)
             return f'{module_prefix}Callable[[{args}], {returns}]'
         elif qualname == 'Literal':
-            args = ', '.join(repr(a) for a in annotation_args)
+            from sphinx.util.inspect import isenumattribute  # lazy loading
+
+            def format_literal_arg(arg):
+                if isenumattribute(arg):
+                    enumcls = arg.__class__
+
+                    if mode == 'smart':
+                        # MyEnum.member
+                        return f'{enumcls.__qualname__}.{arg.name}'
+
+                    # module.MyEnum.member
+                    return f'{enumcls.__module__}.{enumcls.__qualname__}.{arg.name}'
+                return repr(arg)
+
+            args = ', '.join(map(format_literal_arg, annotation_args))
             return f'{module_prefix}Literal[{args}]'
         elif str(annotation).startswith('typing.Annotated'):  # for py39+
             return stringify_annotation(annotation_args[0], mode)
@@ -352,6 +376,14 @@ def stringify_annotation(
     return module_prefix + qualname
 
 
+def _format_literal_enum_arg(arg: enum.Enum, /, *, mode: str) -> str:
+    enum_cls = arg.__class__
+    if mode == 'smart' or enum_cls.__module__ == 'typing':
+        return f':py:attr:`~{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}`'
+    else:
+        return f':py:attr:`{enum_cls.__module__}.{enum_cls.__qualname__}.{arg.name}`'
+
+
 # deprecated name -> (object to return, canonical path or empty string)
 _DEPRECATED_OBJECTS = {
     'stringify': (stringify_annotation, 'sphinx.util.typing.stringify_annotation'),


Test patch:
diff --git a/tests/roots/test-ext-autodoc/target/literal.py b/tests/roots/test-ext-autodoc/target/literal.py
new file mode 100644
index 000000000..4340e5103
--- /dev/null
+++ b/tests/roots/test-ext-autodoc/target/literal.py
@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Literal, TypeVar
+
+
+class MyEnum(Enum):
+    a = 1
+
+
+T = TypeVar('T', bound=Literal[1234])
+"""docstring"""
+
+
+U = TypeVar('U', bound=Literal[MyEnum.a])
+"""docstring"""
+
+
+def bar(x: Literal[1234]):
+    """docstring"""
+
+
+def foo(x: Literal[MyEnum.a]):
+    """docstring"""
diff --git a/tests/test_ext_autodoc.py b/tests/test_ext_autodoc.py
index efb5c0395..c2f0d0bfc 100644
--- a/tests/test_ext_autodoc.py
+++ b/tests/test_ext_autodoc.py
@@ -2467,3 +2467,59 @@ def test_canonical(app):
         '      docstring',
         '',
     ]
+
+
+@pytest.mark.sphinx('html', testroot='ext-autodoc')
+def test_literal_render(app):
+    def bounded_typevar_rst(name, bound):
+        return [
+            '',
+            f'.. py:class:: {name}',
+            '   :module: target.literal',
+            '',
+            '   docstring',
+            '',
+            f'   alias of TypeVar({name!r}, bound={bound})',
+            '',
+        ]
+
+    def function_rst(name, sig):
+        return [
+            '',
+            f'.. py:function:: {name}({sig})',
+            '   :module: target.literal',
+            '',
+            '   docstring',
+            '',
+        ]
+
+    # autodoc_typehints_format can take 'short' or 'fully-qualified' values
+    # and this will be interpreted as 'smart' or 'fully-qualified-except-typing' by restify()
+    # and 'smart' or 'fully-qualified' by stringify_annotation().
+
+    options = {'members': None, 'exclude-members': 'MyEnum'}
+    app.config.autodoc_typehints_format = 'short'
+    actual = do_autodoc(app, 'module', 'target.literal', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.literal',
+        '',
+        *bounded_typevar_rst('T', r'\ :py:obj:`~typing.Literal`\ [1234]'),
+        *bounded_typevar_rst('U', r'\ :py:obj:`~typing.Literal`\ [:py:attr:`~target.literal.MyEnum.a`]'),
+        *function_rst('bar', 'x: ~typing.Literal[1234]'),
+        *function_rst('foo', 'x: ~typing.Literal[MyEnum.a]'),
+    ]
+
+    # restify() assumes that 'fully-qualified' is 'fully-qualified-except-typing'
+    # because it is more likely that a user wants to suppress 'typing.*'
+    app.config.autodoc_typehints_format = 'fully-qualified'
+    actual = do_autodoc(app, 'module', 'target.literal', options)
+    assert list(actual) == [
+        '',
+        '.. py:module:: target.literal',
+        '',
+        *bounded_typevar_rst('T', r'\ :py:obj:`~typing.Literal`\ [1234]'),
+        *bounded_typevar_rst('U', r'\ :py:obj:`~typing.Literal`\ [:py:attr:`target.literal.MyEnum.a`]'),
+        *function_rst('bar', 'x: typing.Literal[1234]'),
+        *function_rst('foo', 'x: typing.Literal[target.literal.MyEnum.a]'),
+    ]
diff --git a/tests/test_util_typing.py b/tests/test_util_typing.py
index a0baf4ffd..d79852e8b 100644
--- a/tests/test_util_typing.py
+++ b/tests/test_util_typing.py
@@ -1,6 +1,7 @@
 """Tests util.typing functions."""
 
 import sys
+from enum import Enum
 from numbers import Integral
 from struct import Struct
 from types import TracebackType
@@ -31,6 +32,10 @@ class MyClass2(MyClass1):
     __qualname__ = '<MyClass2>'
 
 
+class MyEnum(Enum):
+    a = 1
+
+
 T = TypeVar('T')
 MyInt = NewType('MyInt', int)
 
@@ -194,6 +199,9 @@ def test_restify_type_Literal():
     from typing import Literal  # type: ignore[attr-defined]
     assert restify(Literal[1, "2", "\r"]) == ":py:obj:`~typing.Literal`\\ [1, '2', '\\r']"
 
+    assert restify(Literal[MyEnum.a], 'fully-qualified-except-typing') == ':py:obj:`~typing.Literal`\\ [:py:attr:`tests.test_util_typing.MyEnum.a`]'
+    assert restify(Literal[MyEnum.a], 'smart') == ':py:obj:`~typing.Literal`\\ [:py:attr:`~tests.test_util_typing.MyEnum.a`]'
+
 
 def test_restify_pep_585():
     assert restify(list[str]) == ":py:class:`list`\\ [:py:class:`str`]"  # type: ignore[attr-defined]
@@ -478,6 +486,10 @@ def test_stringify_type_Literal():
     assert stringify_annotation(Literal[1, "2", "\r"], "fully-qualified") == "typing.Literal[1, '2', '\\r']"
     assert stringify_annotation(Literal[1, "2", "\r"], "smart") == "~typing.Literal[1, '2', '\\r']"
 
+    assert stringify_annotation(Literal[MyEnum.a], 'fully-qualified-except-typing') == 'Literal[tests.test_util_typing.MyEnum.a]'
+    assert stringify_annotation(Literal[MyEnum.a], 'fully-qualified') == 'typing.Literal[tests.test_util_typing.MyEnum.a]'
+    assert stringify_annotation(Literal[MyEnum.a], 'smart') == '~typing.Literal[MyEnum.a]'
+
 
 @pytest.mark.skipif(sys.version_info[:2] <= (3, 9), reason='python 3.10+ is required.')
 def test_stringify_type_union_operator():