Repository: lebrice/SimpleParsing

Issue:
Missing descriptions with multiple inheritance
**Describe the bug**

Some argument descriptions are missing when using multiple inheritance.

**To Reproduce**
```python
from simple_parsing import ArgumentParser
from dataclasses import dataclass

@dataclass
class Foo:
   bar: int = 123  #: The bar property

@dataclass
class Baz:
   bat: int = 123  #: The bat property

@dataclass
class FooBaz(Foo, Baz):
   foobaz: int = 123  #: The foobaz property

if __name__ == "__main__":
   parser = ArgumentParser()
   parser.add_arguments(FooBaz, "foobaz")
   args = parser.parse_args()
   foobaz: FooBaz = args.foobaz
   print(foobaz)
```

**Expected behavior**
The help string for "bat" contains "The bat property". 

**Actual behavior**
The help string for "bat" is empty.

```console
usage: foo.py [-h] [--bat int] [--bar int] [--foobaz int]

optional arguments:
  -h, --help    show this help message and exit

FooBaz ['foobaz']:
  FooBaz(bat: int = 123, bar: int = 123, foobaz: int = 123)

  --bat int     (default: 123)
  --bar int     : The bar property (default: 123)
  --foobaz int  : The foobaz property (default: 123)
```

**Desktop (please complete the following information):**
 - Version 0.0.20
 - Python version: 3.8.9

**Additional context**
None.


Discussion:

Patch:
diff --git a/simple_parsing/docstring.py b/simple_parsing/docstring.py
index 81fb1c2..81105c1 100644
--- a/simple_parsing/docstring.py
+++ b/simple_parsing/docstring.py
@@ -1,10 +1,11 @@
 """Utility for retrieveing the docstring of a dataclass's attributes
 @author: Fabrice Normandin
 """
+from __future__ import annotations
+
 import inspect
 from dataclasses import dataclass
 from logging import getLogger
-from typing import List, Optional, Type
 
 logger = getLogger(__name__)
 
@@ -18,7 +19,9 @@ class AttributeDocString:
     docstring_below: str = ""
 
 
-def get_attribute_docstring(some_dataclass: Type, field_name: str) -> AttributeDocString:
+def get_attribute_docstring(
+    dataclass: type, field_name: str, accumulate_from_bases: bool = True
+) -> AttributeDocString:
     """Returns the docstrings of a dataclass field.
     NOTE: a docstring can either be:
     - An inline comment, starting with <#>
@@ -26,58 +29,96 @@ def get_attribute_docstring(some_dataclass: Type, field_name: str) -> AttributeD
     - A docstring on the following line, starting with either <\"\"\"> or <'''>
 
     Arguments:
-        some_dataclass {type} -- a dataclass
-        field_name {str} -- the name of the field.
-
+        some_dataclass: a dataclass
+        field_name: the name of the field.
+        accumulate_from_bases: Whether to accumulate the docstring components by looking through the
+            base classes. When set to `False`, whenever one of the classes has a definition for the
+            field, it is directly returned. Otherwise, we accumulate the parts of the dodc
     Returns:
         AttributeDocString -- an object holding the three possible comments
     """
-    try:
-        source = inspect.getsource(some_dataclass)
-    except (TypeError, OSError) as e:
-        logger.debug(f"Couldn't find the attribute docstring: {e}")
+    created_docstring: AttributeDocString | None = None
+
+    mro = inspect.getmro(dataclass)
+    assert mro[0] is dataclass
+    assert mro[-1] is object
+    mro = mro[:-1]
+    for base_class in mro:
+        attribute_docstring = _get_attribute_docstring(base_class, field_name)
+        if not attribute_docstring:
+            continue
+        if not created_docstring:
+            created_docstring = attribute_docstring
+            if not accumulate_from_bases:
+                # We found a definition for that field in that class, so return it directly.
+                return created_docstring
+        else:
+            # Update the fields.
+            created_docstring.comment_above = (
+                created_docstring.comment_above or attribute_docstring.comment_above
+            )
+            created_docstring.comment_inline = (
+                created_docstring.comment_inline or attribute_docstring.comment_inline
+            )
+            created_docstring.docstring_below = (
+                created_docstring.docstring_below or attribute_docstring.docstring_below
+            )
+    if not created_docstring:
+        logger.debug(
+            RuntimeWarning(
+                f"Couldn't find the definition for field '{field_name}' within the dataclass "
+                f"{dataclass} or any of its base classes {','.join(t.__name__ for t in mro[1:])}."
+            )
+        )
         return AttributeDocString()
+    return created_docstring
 
-    code_lines: List[str] = source.splitlines()
-    # the first line is the class definition, we skip it.
+
+def _get_attribute_docstring(dataclass: type, field_name: str) -> AttributeDocString | None:
+    """Gets the AttributeDocString of the given field in the given dataclass.
+    Doesn't inspect base classes.
+    """
+    try:
+        source = inspect.getsource(dataclass)
+    except (TypeError, OSError) as e:
+        logger.debug(
+            UserWarning(
+                f"Couldn't retrieve the source code of class {dataclass} "
+                f"(in order to retrieve the docstring of field {field_name}): {e}"
+            )
+        )
+        return None
+    # NOTE: We want to skip the docstring lines.
+    # NOTE: Currently, we just remove the __doc__ from the source. It's perhaps a bit crude,
+    # but it works.
+    if dataclass.__doc__ and dataclass.__doc__ in source:
+        source = source.replace(dataclass.__doc__, "\n", 1)
+        # note: does this remove the whitespace though?
+
+    code_lines: list[str] = source.splitlines()
+    # the first line is the class definition (OR the decorator!), we skip it.
     start_line_index = 1
     # starting at the second line, there might be the docstring for the class.
     # We want to skip over that until we reach an attribute definition.
     while start_line_index < len(code_lines):
-        if _contains_attribute_definition(code_lines[start_line_index]):
+        if _contains_field_definition(code_lines[start_line_index]):
             break
         start_line_index += 1
 
-    lines_with_attribute_defs = [
-        (index, line)
-        for index, line in enumerate(code_lines)
-        if _contains_attribute_definition(line)
+    lines_with_field_defs = [
+        (index, line) for index, line in enumerate(code_lines) if _contains_field_definition(line)
     ]
-    for i, line in lines_with_attribute_defs:
-        parts: List[str] = line.split(":", maxsplit=1)
-        if parts[0].strip() == field_name:
+    for i, line in lines_with_field_defs:
+        if _line_contains_definition_for(line, field_name):
             # we found the line with the definition of this field.
             comment_above = _get_comment_ending_at_line(code_lines, i - 1)
             comment_inline = _get_inline_comment_at_line(code_lines, i)
             docstring_below = _get_docstring_starting_at_line(code_lines, i + 1)
-            complete_docstring = AttributeDocString(comment_above, comment_inline, docstring_below)
-            return complete_docstring
-
-    # we didn't find the attribute.
-    mro = inspect.getmro(some_dataclass)
-    if len(mro) == 1:
-        raise RuntimeWarning(
-            f"Couldn't find the given attribute name {field_name}' within the " "given class."
-        )
-    base_class = mro[1]
-    try:
-        return get_attribute_docstring(base_class, field_name)
-    except OSError as e:
-        logger.warning(UserWarning(f"Couldn't find the docstring: {e}"))
-        return AttributeDocString()
+            return AttributeDocString(comment_above, comment_inline, docstring_below)
+    return None
 
 
-def _contains_attribute_definition(line_str: str) -> bool:
+def _contains_field_definition(line: str) -> bool:
     """Returns whether or not a line contains a an dataclass field definition.
 
     Arguments:
@@ -85,18 +126,57 @@ def _contains_attribute_definition(line_str: str) -> bool:
 
     Returns:
         bool -- True if there is an attribute definition in the line.
+
+    >>> _contains_field_definition("a: int = 0")
+    True
+    >>> _contains_field_definition("a: int")
+    True
+    >>> _contains_field_definition("a: int # comment")
+    True
+    >>> _contains_field_definition("a: int = 0 # comment")
+    True
+    >>> _contains_field_definition("class FooBaz(Foo, Baz):")
+    False
+    >>> _contains_field_definition("a = 4")
+    False
+    >>> _contains_field_definition("fooooooooobar.append(123)")
+    False
+    >>> _contains_field_definition("{a: int}")
+    False
+    >>> _contains_field_definition("        foobaz: int = 123  #: The foobaz property")
+    True
+    >>> _contains_field_definition("a #:= 3")
+    False
     """
-    parts = line_str.split("#", maxsplit=1)
-    before_comment = parts[0].strip()
+    # Get rid of any comments first.
+    line, _, _ = line.partition("#")
 
-    before_first_equal = before_comment.split("=", maxsplit=1)[0]
-    parts = before_first_equal.split(":")
-    if len(parts) != 2:
-        # For now, I don't think it's possible to have a type annotation contain :
+    if ":" not in line:
         return False
-    attr_name = parts[0]
-    attr_type = parts[1]
-    return not attr_name.isspace() and not attr_type.isspace()
+
+    if "=" in line:
+        attribute_and_type, _, _ = line.partition("=")
+    else:
+        attribute_and_type = line
+
+    field_name, _, type = attribute_and_type.partition(":")
+    field_name = field_name.strip()
+    if ":" in type:
+        # weird annotation or dictionary?
+        return False
+    if not field_name:
+        # Empty attribute name?
+        return False
+    return field_name.isidentifier()
+
+
+def _line_contains_definition_for(line: str, field_name: str) -> bool:
+    line = line.strip()
+    if not _contains_field_definition(line):
+        return False
+    attribute, _, type_and_value_assignment = line.partition(":")
+    attribute = attribute.strip()  # remove any whitespace after the attribute name.
+    return attribute.isidentifier() and attribute == field_name
 
 
 def _is_empty(line_str: str) -> bool:
@@ -107,7 +187,7 @@ def _is_comment(line_str: str) -> bool:
     return line_str.strip().startswith("#")
 
 
-def _get_comment_at_line(code_lines: List[str], line: int) -> str:
+def _get_comment_at_line(code_lines: list[str], line: int) -> str:
     """Gets the comment at line `line` in `code_lines`.
 
     Arguments:
@@ -117,7 +197,7 @@ def _get_comment_at_line(code_lines: List[str], line: int) -> str:
         str -- the comment at the given line. empty string if not present.
     """
     line_str = code_lines[line]
-    assert not _contains_attribute_definition(line_str)
+    assert not _contains_field_definition(line_str)
     if "#" not in line_str:
         return ""
     parts = line_str.split("#", maxsplit=1)
@@ -125,7 +205,7 @@ def _get_comment_at_line(code_lines: List[str], line: int) -> str:
     return comment
 
 
-def _get_inline_comment_at_line(code_lines: List[str], line: int) -> str:
+def _get_inline_comment_at_line(code_lines: list[str], line: int) -> str:
     """Gets the inline comment at line `line`.
 
     Arguments:
@@ -135,7 +215,7 @@ def _get_inline_comment_at_line(code_lines: List[str], line: int) -> str:
         str -- the inline comment at the given line, else an empty string.
     """
     assert 0 <= line < len(code_lines)
-    assert _contains_attribute_definition(code_lines[line])
+    assert _contains_field_definition(code_lines[line])
     line_str = code_lines[line]
     parts = line_str.split("#", maxsplit=1)
     if len(parts) != 2:
@@ -144,18 +224,14 @@ def _get_inline_comment_at_line(code_lines: List[str], line: int) -> str:
     return comment
 
 
-def _get_comment_ending_at_line(code_lines: List[str], line: int) -> str:
+def _get_comment_ending_at_line(code_lines: list[str], line: int) -> str:
     start_line = line
     end_line = line
-    # print(f"Get comment ending at line {line}")
-    # for i, l in enumerate(code_lines):
-    #     print(f"line {i}: {l}")
-
     # move up the code, one line at a time, while we don't hit the start,
     # an attribute definition, or the end of a docstring.
     while start_line > 0:
         line_str = code_lines[start_line]
-        if _contains_attribute_definition(line_str):
+        if _contains_field_definition(line_str):
             break  # previous line is an assignment
         if '"""' in line_str or "'''" in line_str:
             break  # previous line has a docstring
@@ -167,15 +243,15 @@ def _get_comment_ending_at_line(code_lines: List[str], line: int) -> str:
         # print(f"line {i}: {code_lines[i]}")
         if _is_empty(code_lines[i]):
             continue
-        assert not _contains_attribute_definition(code_lines[i])
+        assert not _contains_field_definition(code_lines[i])
         comment = _get_comment_at_line(code_lines, i)
         lines.append(comment)
-    return "\n".join(lines)
+    return "\n".join(lines).strip()
 
 
-def _get_docstring_starting_at_line(code_lines: List[str], line: int) -> str:
+def _get_docstring_starting_at_line(code_lines: list[str], line: int) -> str:
     i = line
-    token: Optional[str] = None
+    token: str | None = None
     triple_single = "'''"
     triple_double = '"""'
     # print("finding docstring starting from line", line)
@@ -185,9 +261,9 @@ def _get_docstring_starting_at_line(code_lines: List[str], line: int) -> str:
     if line >= len(code_lines):
         return ""
     # the list of lines making up the docstring.
-    docstring_contents: List[str] = []
+    docstring_contents: list[str] = []
 
-    while i <= len(code_lines):
+    while i < len(code_lines):
         line_str = code_lines[i]
         # print(f"(docstring) line {line}: {line_str}")
 
@@ -197,7 +273,7 @@ def _get_docstring_starting_at_line(code_lines: List[str], line: int) -> str:
                 i += 1
                 continue
 
-            elif _contains_attribute_definition(line_str) or _is_comment(line_str):
+            elif _contains_field_definition(line_str) or _is_comment(line_str):
                 # we haven't reached the start of a docstring yet (since token
                 # is None), and we reached a line with an attribute definition,
                 # or a comment, hence the docstring is empty.


Test patch:
diff --git a/simple_parsing/helpers/hparams/hyperparameters_test.py b/simple_parsing/helpers/hparams/hyperparameters_test.py
index 23ac082..06d7c9c 100644
--- a/simple_parsing/helpers/hparams/hyperparameters_test.py
+++ b/simple_parsing/helpers/hparams/hyperparameters_test.py
@@ -265,6 +265,6 @@ def test_field_types():
     assert C.get_priors()["f"].discrete is True
 
     if numpy_installed:
-        assert all(c.f.dtype == np.int for c in cs)
+        assert all(c.f.dtype == int for c in cs)
     else:
         assert all(all(isinstance(v, int) for v in c.f) for c in cs)
diff --git a/test/test_conflicts.py b/test/test_conflicts.py
index c2059ce..d1e5cab 100644
--- a/test/test_conflicts.py
+++ b/test/test_conflicts.py
@@ -74,10 +74,10 @@ def test_parent_child_conflict():
         batch_size: int = 32
 
     @dataclass
-    class Parent(TestSetup):
+    class Parent2(TestSetup):
         batch_size: int = 48
         child: HParams = HParams()
 
-    p: Parent = Parent.setup()
+    p: Parent2 = Parent2.setup()
     assert p.child.batch_size == 32
     assert p.batch_size == 48
diff --git a/test/test_docstrings.py b/test/test_docstrings.py
index e91ca5f..a4d5fa6 100644
--- a/test/test_docstrings.py
+++ b/test/test_docstrings.py
@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from typing import List
 
 from simple_parsing import field
-from simple_parsing.docstring import get_attribute_docstring
+from simple_parsing.docstring import AttributeDocString, get_attribute_docstring
 
 from .testutils import TestSetup
 
@@ -84,14 +84,95 @@ def test_docstring_parsing_works_on_extended():
 
 def test_docstring_works_with_field_function():
     @dataclass
-    class Foo(TestSetup):
+    class UniqueFoo(TestSetup):
         """Some class Foo"""
 
         # A sequence of tasks.
         task_sequence: List[str] = field(choices=["train", "test", "ood"])  # side
         """Below"""
 
-    docstring = get_attribute_docstring(Foo, "task_sequence")
+    docstring = get_attribute_docstring(UniqueFoo, "task_sequence")
     assert docstring.comment_above == "A sequence of tasks."
     assert docstring.comment_inline == "side"
     assert docstring.docstring_below == "Below"
+
+
+def test_docstrings_with_multiple_inheritance():
+    """Test to reproduce issue 162: https://github.com/lebrice/SimpleParsing/issues/162"""
+
+    @dataclass
+    class Fooz:
+        bar: int = 123  #: The bar property
+
+    @dataclass
+    class Baz:
+        bat: int = 123  #: The bat property
+
+    @dataclass
+    class FooBaz(Fooz, Baz):
+        foobaz: int = 123  #: The foobaz property
+
+    assert get_attribute_docstring(FooBaz, "bar") == AttributeDocString(
+        comment_inline=": The bar property"
+    )
+    assert get_attribute_docstring(FooBaz, "bat") == AttributeDocString(
+        comment_inline=": The bat property"
+    )
+    assert get_attribute_docstring(FooBaz, "foobaz") == AttributeDocString(
+        comment_inline=": The foobaz property"
+    )
+
+
+def test_weird_docstring_with_field_like():
+    @dataclass
+    class FooA:
+        """
+        @dataclass
+        class weird:
+            bar: int = 123  # WRONG DOCSTRING
+        """
+
+        bar: int = 123  # The bar property
+
+    assert get_attribute_docstring(FooA, "bar") == AttributeDocString(
+        comment_inline="The bar property"
+    )
+
+
+def test_docstring_builds_upon_bases():
+    @dataclass
+    class Base2(TestSetup):
+        """
+        # WRONG ABOVE
+        bar: int = 333 # WRONG INLINE
+        '''WRONG DOCSTRING'''
+        """
+
+        bar: int = 123  # inline
+        """field docstring from base class"""
+
+    @dataclass
+    class FooB(Base2):
+        # Above
+        bar: int = 123  # The bar property
+
+    assert get_attribute_docstring(FooB, "bar") == AttributeDocString(
+        comment_inline="The bar property",
+        comment_above="Above",
+        docstring_below="field docstring from base class",
+    )
+
+    assert "field docstring from base class" in FooB.get_help_text()
+
+
+def test_getdocstring_bug():
+    @dataclass
+    class HParams:
+        batch_size: int = 32
+
+    @dataclass
+    class Parent(TestSetup):
+        batch_size: int = 48
+        child: HParams = HParams()
+
+    assert get_attribute_docstring(Parent, "child") == AttributeDocString()