Repository: linw1995/data_extractor

Issue:
Can't locate SytanxError.text from source code.
```python3
class Parameter(Item): name = Field(XPathExtractor("./span[@class='name']"))
```

```python3
class User(Item):
    uid = Field(JSONExtractor("id")); name = Field(JSONExtractor("name"))
```

```python3
Parameter = type(
    "Parameter", (Item,), {"name": Field(XPathExtractor("./span[@class='name']"))}
)
```

this line raises `AssertError` when above code executed.

https://github.com/linw1995/data_extractor/blob/99a4a7f9ca7908d9e15399a8cfb96a50af7a1f01/data_extractor/abc.py#L45



Discussion:

Patch:
diff --git a/data_extractor/abc.py b/data_extractor/abc.py
index f26a4a2..865a7c3 100644
--- a/data_extractor/abc.py
+++ b/data_extractor/abc.py
@@ -26,7 +26,10 @@ class ComplexExtractorMeta(type):
 
         for key, attr in attr_dict.items():
             if isinstance(type(attr), ComplexExtractorMeta):
-                if key in __init_args:
+                if key in __init_args[1:]:
+                    # Item's attribute overwrites the 'Item.__init__' parameters except first parameter.
+                    args = []  # type: List[Any]
+                    exc_args = None
                     frame = inspect.currentframe()
                     assert (
                         frame is not None
@@ -36,28 +39,54 @@ class ComplexExtractorMeta(type):
 
                         filename = outer_frame.f_code.co_filename
                         firstlineno = outer_frame.f_lineno
-                        lines, _ = inspect.findsource(outer_frame)
+                        firstline_idx = firstlineno - 1
+                        lines = None
+                        try:
+                            lines, _ = inspect.findsource(outer_frame)
+                        except OSError:
+                            # can't get the source code from python repl
+                            pass
+
+                        if lines is not None:
+                            start_index = inspect.indentsize(lines[firstline_idx])
+                            for lineno, line in enumerate(
+                                lines[firstline_idx + 1 :], start=firstlineno + 1
+                            ):
+                                # iterate line in the code block body
+                                cur_index = inspect.indentsize(line)
+                                if cur_index <= start_index:
+                                    # reach end of the code block, use code block firstlineno as SyntaxError.lineno
+                                    line = lines[firstline_idx]
+                                    lineno = firstlineno
+                                    break
+
+                                if line.lstrip().startswith(key):
+                                    # find the line as SyntaxError.text
+                                    break
+
+                            else:
+                                # reach EOF, use code block firstlineno as SyntaxError.lineno
+                                line = lines[firstline_idx]
+                                lineno = firstlineno
+
+                            offset = inspect.indentsize(line)
+                            line = line.strip()
+                            exc_args = (filename, lineno, offset, line)
+                        else:
+                            line = f"{key}={attr!r}"
 
-                        for lineno, line in enumerate(lines[firstlineno:], start=1):
-                            if line.strip().startswith(key):
-                                break
-                        else:  # pragma: no cover
-                            assert False, "This line is never executed."
-
-                        lineno += firstlineno
-                        index = inspect.indentsize(line)
                     finally:
                         del outer_frame
                         del frame
 
-                    line = line.strip()
-                    raise SyntaxError(
-                        (
-                            f"{line!r} overwriten the parameter {key!r} of '{name}.__init__' method. "
-                            f"Please using the optional parameter name={key!r} in {attr!r} to avoid overwriting parameter name."
-                        ),
-                        (filename, lineno, index, line),
+                    args.append(
+                        f"{line!r} overwriten the parameter {key!r} of '{name}.__init__' method. "
+                        f"Please using the optional parameter name={key!r} in {attr!r} to avoid overwriting parameter name."
                     )
+                    if exc_args is not None:
+                        args.append(exc_args)
+
+                    raise SyntaxError(*args)
 
                 field_names.append(key)
 


Test patch:
diff --git a/tests/test_item.py b/tests/test_item.py
index 13fe5a7..801bd6e 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -1,5 +1,6 @@
 # Standard Library
 import inspect
+import linecache
 
 from pathlib import Path
 
@@ -316,7 +317,7 @@ def test_misplacing():
         Field(extractor=ComplexExtractor(extractor=JSONExtractor("users[*]")))
 
 
-def test_field_name_overwrite_item_parameter():
+def test_field_name_overwrite_item_parameter_common():
     with pytest.raises(SyntaxError) as catch:
 
         class User(Item):
@@ -330,6 +331,99 @@ def test_field_name_overwrite_item_parameter():
     assert exc.text == 'name = Field(JSONExtractor("name"))'
 
 
+def test_field_name_overwrite_item_parameter_oneline():
+    with pytest.raises(SyntaxError) as catch:
+        # fmt: off
+        class Parameter(Item): name = Field(XPathExtractor("./span[@class='name']"))  # noqa
+        # fmt: on
+
+    exc = catch.value
+    assert exc.filename == __file__
+    assert exc.lineno == inspect.currentframe().f_lineno - 5
+    assert exc.offset == 8
+    assert (
+        exc.text
+        == "class Parameter(Item): name = Field(XPathExtractor(\"./span[@class='name']\"))  # noqa"
+    )
+
+
+def test_field_name_overwrite_item_parameter_type_creation():
+    with pytest.raises(SyntaxError) as catch:
+        # fmt: off
+        type("Parameter", (Item,), {"name": Field(XPathExtractor("./span[@class='name']"))})
+        # fmt: on
+
+    exc = catch.value
+    assert exc.filename == __file__
+    assert exc.lineno == inspect.currentframe().f_lineno - 5
+    assert exc.offset == 8
+    assert (
+        exc.text
+        == 'type("Parameter", (Item,), {"name": Field(XPathExtractor("./span[@class=\'name\']"))})'
+    )
+
+
+source_codes = [
+    'type("Parameter", (Item,), {"name": Field(XPathExtractor("./span[@class=\'name\']"))})',
+    "class Parameter(Item): name = Field(XPathExtractor(\"./span[@class='name']\"))  # noqa",
+    """class User(Item):
+    uid = Field(JSONExtractor("id")); name = Field(JSONExtractor("name"))
+    """,
+    """
+class User(Item):
+    uid = Field(JSONExtractor("id"))
+    name = Field(JSONExtractor("name"))
+    """,
+]
+
+
+@pytest.mark.parametrize("source_code", source_codes)
+def test_field_name_overwrite_item_parameter_in_repl(source_code):
+    with pytest.raises(SyntaxError) as catch:
+        exec(source_code)
+
+    exc = catch.value
+    assert exc.filename is None
+    assert exc.lineno is None
+    assert exc.offset is None
+    assert exc.text is None
+
+
+@pytest.mark.parametrize("source_code", source_codes[:-1])
+def test_field_name_overwrite_item_parameter_oneline_in_script(source_code, tmp_path):
+    tmp_file = tmp_path / "foo.py"
+    tmp_file.write_text(source_code)
+    tmp_file = str(tmp_file)
+    linecache.updatecache(tmp_file)
+
+    with pytest.raises(SyntaxError) as catch:
+        exec(compile(source_code, tmp_file, "exec"))
+
+    exc = catch.value
+    assert exc.filename == tmp_file
+    assert exc.lineno == 1
+    assert exc.offset == 0
+    assert exc.text == source_code.split("\n")[0].strip()
+
+
+def test_field_name_overwrite_item_parameter_common_in_script(tmp_path):
+    source_code = source_codes[-1]
+
+    tmp_file = tmp_path / "foo.py"
+    tmp_file.write_text(source_code)
+    tmp_file = str(tmp_file)
+    linecache.updatecache(tmp_file)
+
+    with pytest.raises(SyntaxError) as catch:
+        exec(compile(source_code, tmp_file, "exec"))
+
+    exc = catch.value
+    assert exc.filename == tmp_file
+    assert exc.lineno == 4
+    assert exc.offset == 4
+    assert exc.text == 'name = Field(JSONExtractor("name"))'
+
+
 def test_avoid_field_name_overwriting_item_parameter(json0):
     data = json0