Repository: explosion/spaCy

Issue:
Allow setting of custom attributes during retokenization
## Feature description
<!-- Please describe the feature: Which area of the library is it related to? What specific solution would you like? -->

Perhaps I am just missing something, but I don't seem to be able to set custom attributes on a merged token during retokenization (using the new 2.1.x Retokenizer class).

    with doc.retokenize() as retokenizer:
        retokenizer.merge(span, attrs={ ... })

I can set any of the attributes listed in the Token documentation, but I am unable to do that for custom attributes. If there is a special syntax for setting custom attributes already, it should be documented.



Discussion:
Yes, custom attributes aren't supported in the `attrs`, but it's an interesting idea.

The only potential problem I see here is that it can lead to confusing and inconsistent results: Not all custom attributes are writable – if the attribute isn't registered with a default value or doesn't have a setter, setting them on retokenization won't work. But then again, this _is_ pretty logical and I'm not sure people would really be so confused by that.

We could support it via the `_` property, like this:

```python
attrs = {"POS": "PROPN", "_": {"foo": "bar"}}
```

Resolving patch:
diff --git a/spacy/errors.py b/spacy/errors.py
--- a/spacy/errors.py
+++ b/spacy/errors.py
@@ -327,6 +327,17 @@ class Errors(object):
             "performance.")
     E117 = ("The newly split tokens must match the text of the original token. "
             "New orths: {new}. Old text: {old}.")
+    E118 = ("The custom extension attribute '{attr}' is not registered on the "
+            "Token object so it can't be set during retokenization. To "
+            "register an attribute, use the Token.set_extension classmethod.")
+    E119 = ("Can't set custom extension attribute '{attr}' during retokenization "
+            "because it's not writable. This usually means it was registered "
+            "with a getter function (and no setter) or as a method extension, "
+            "so the value is computed dynamically. To overwrite a custom "
+            "attribute manually, it should be registered with a default value "
+            "or with a getter AND setter.")
+    E120 = ("Can't set custom extension attributes during retokenization. "
+            "Expected dict mapping attribute names to values, but got: {value}")
 
 
 @add_codes
diff --git a/spacy/tokens/underscore.py b/spacy/tokens/underscore.py
--- a/spacy/tokens/underscore.py
+++ b/spacy/tokens/underscore.py
@@ -75,3 +75,18 @@ def get_ext_args(**kwargs):
     if method is not None and not hasattr(method, "__call__"):
         raise ValueError(Errors.E091.format(name="method", value=repr(method)))
     return (default, method, getter, setter)
+
+
+def is_writable_attr(ext):
+    """Check if an extension attribute is writable.
+    ext (tuple): The (default, getter, setter, method) tuple available  via
+        {Doc,Span,Token}.get_extension.
+    RETURNS (bool): Whether the attribute is writable.
+    """
+    default, method, getter, setter = ext
+    # Extension is writable if it has a setter (getter + setter), if it has a
+    # default value (or, if its default value is none, none of the other values
+    # should be set).
+    if setter is not None or default is not None or all(e is None for e in ext):
+        return True
+    return False


Test patch:
diff --git a/spacy/tests/doc/test_retokenize_merge.py b/spacy/tests/doc/test_retokenize_merge.py
--- a/spacy/tests/doc/test_retokenize_merge.py
+++ b/spacy/tests/doc/test_retokenize_merge.py
@@ -4,7 +4,7 @@
 import pytest
 from spacy.attrs import LEMMA
 from spacy.vocab import Vocab
-from spacy.tokens import Doc
+from spacy.tokens import Doc, Token
 
 from ..util import get_doc
 
@@ -259,3 +259,36 @@ def test_doc_retokenize_spans_subtree_size_check(en_tokenizer):
         attrs = {"lemma": "none", "ent_type": "none"}
         retokenizer.merge(doc[0:2], attrs=attrs)
     assert len(list(sent1.root.subtree)) == init_len - 1
+
+
+def test_doc_retokenize_merge_extension_attrs(en_vocab):
+    Token.set_extension("a", default=False, force=True)
+    Token.set_extension("b", default="nothing", force=True)
+    doc = Doc(en_vocab, words=["hello", "world", "!"])
+    # Test regular merging
+    with doc.retokenize() as retokenizer:
+        attrs = {"lemma": "hello world", "_": {"a": True, "b": "1"}}
+        retokenizer.merge(doc[0:2], attrs=attrs)
+    assert doc[0].lemma_ == "hello world"
+    assert doc[0]._.a == True
+    assert doc[0]._.b == "1"
+    # Test bulk merging
+    doc = Doc(en_vocab, words=["hello", "world", "!", "!"])
+    with doc.retokenize() as retokenizer:
+        retokenizer.merge(doc[0:2], attrs={"_": {"a": True, "b": "1"}})
+        retokenizer.merge(doc[2:4], attrs={"_": {"a": None, "b": "2"}})
+    assert doc[0]._.a == True
+    assert doc[0]._.b == "1"
+    assert doc[1]._.a == None
+    assert doc[1]._.b == "2"
+
+
+@pytest.mark.parametrize("underscore_attrs", [{"a": "x"}, {"b": "x"}, {"c": "x"}, [1]])
+def test_doc_retokenize_merge_extension_attrs_invalid(en_vocab, underscore_attrs):
+    Token.set_extension("a", getter=lambda x: x, force=True)
+    Token.set_extension("b", method=lambda x: x, force=True)
+    doc = Doc(en_vocab, words=["hello", "world", "!"])
+    attrs = {"_": underscore_attrs}
+    with pytest.raises(ValueError):
+        with doc.retokenize() as retokenizer:
+            retokenizer.merge(doc[0:2], attrs=attrs)
diff --git a/spacy/tests/doc/test_retokenize_split.py b/spacy/tests/doc/test_retokenize_split.py
--- a/spacy/tests/doc/test_retokenize_split.py
+++ b/spacy/tests/doc/test_retokenize_split.py
@@ -3,7 +3,7 @@
 
 import pytest
 from spacy.vocab import Vocab
-from spacy.tokens import Doc
+from spacy.tokens import Doc, Token
 
 from ..util import get_doc
 
@@ -125,3 +125,43 @@ def test_doc_retokenize_split_orths_mismatch(en_vocab):
     with pytest.raises(ValueError):
         with doc.retokenize() as retokenizer:
             retokenizer.split(doc[0], ["L", "A"], [(doc[0], 0), (doc[0], 0)])
+
+
+def test_doc_retokenize_split_extension_attrs(en_vocab):
+    Token.set_extension("a", default=False, force=True)
+    Token.set_extension("b", default="nothing", force=True)
+    doc = Doc(en_vocab, words=["LosAngeles", "start"])
+    with doc.retokenize() as retokenizer:
+        heads = [(doc[0], 1), doc[1]]
+        underscore = [{"a": True, "b": "1"}, {"b": "2"}]
+        attrs = {"lemma": ["los", "angeles"], "_": underscore}
+        retokenizer.split(doc[0], ["Los", "Angeles"], heads, attrs=attrs)
+    assert doc[0].lemma_ == "los"
+    assert doc[0]._.a == True
+    assert doc[0]._.b == "1"
+    assert doc[1].lemma_ == "angeles"
+    assert doc[1]._.a == False
+    assert doc[1]._.b == "2"
+
+
+@pytest.mark.parametrize(
+    "underscore_attrs",
+    [
+        [{"a": "x"}, {}],  # Overwriting getter without setter
+        [{"b": "x"}, {}],  # Overwriting method
+        [{"c": "x"}, {}],  # Overwriting nonexistent attribute
+        [{"a": "x"}, {"x": "x"}],  # Combination
+        [{"a": "x", "x": "x"}, {"x": "x"}],  # Combination
+        {"x": "x"},  # Not a list of dicts
+    ],
+)
+def test_doc_retokenize_split_extension_attrs_invalid(en_vocab, underscore_attrs):
+    Token.set_extension("x", default=False, force=True)
+    Token.set_extension("a", getter=lambda x: x, force=True)
+    Token.set_extension("b", method=lambda x: x, force=True)
+    doc = Doc(en_vocab, words=["LosAngeles", "start"])
+    attrs = {"_": underscore_attrs}
+    with pytest.raises(ValueError):
+        with doc.retokenize() as retokenizer:
+            heads = [(doc[0], 1), doc[1]]
+            retokenizer.split(doc[0], ["Los", "Angeles"], heads, attrs=attrs)
diff --git a/spacy/tests/regression/test_issue1971.py b/spacy/tests/regression/test_issue1971.py
--- a/spacy/tests/regression/test_issue1971.py
+++ b/spacy/tests/regression/test_issue1971.py
@@ -36,8 +36,8 @@ def test_issue_1971_2(en_vocab):
 
 def test_issue_1971_3(en_vocab):
     """Test that pattern matches correctly for multiple extension attributes."""
-    Token.set_extension("a", default=1)
-    Token.set_extension("b", default=2)
+    Token.set_extension("a", default=1, force=True)
+    Token.set_extension("b", default=2, force=True)
     doc = Doc(en_vocab, words=["hello", "world"])
     matcher = Matcher(en_vocab)
     matcher.add("A", None, [{"_": {"a": 1}}])
@@ -51,8 +51,8 @@ def test_issue_1971_4(en_vocab):
     """Test that pattern matches correctly with multiple extension attribute
     values on a single token.
     """
-    Token.set_extension("ext_a", default="str_a")
-    Token.set_extension("ext_b", default="str_b")
+    Token.set_extension("ext_a", default="str_a", force=True)
+    Token.set_extension("ext_b", default="str_b", force=True)
     matcher = Matcher(en_vocab)
     doc = Doc(en_vocab, words=["this", "is", "text"])
     pattern = [{"_": {"ext_a": "str_a", "ext_b": "str_b"}}] * 3