Repository: explosion/curated-tokenizers (language: python)

Issue:
Add support for `copy`/`deepcopy` + `SentencePieceProcessor` deserialization bugfix
<!--- Provide a general summary of your changes in the title. -->

## Description

<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
`deepcopy` support is required for the `replace_listeners` functionality in `spacy-curated-transformers`.

### Types of change

<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
Enhancement/bugfix

## Checklist

<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->

- [x] I confirm that I have the right to submit this contribution under the project's MIT license.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.



Discussion / hints:

Fix patch:
diff --git a/curated_tokenizers/_bbpe.pyx b/curated_tokenizers/_bbpe.pyx
index dcd0a14..98a9c70 100644
--- a/curated_tokenizers/_bbpe.pyx
+++ b/curated_tokenizers/_bbpe.pyx
@@ -66,6 +66,14 @@ cdef class ByteBPEProcessor:
             c_merges.push_back(pair[string, string](p1.encode('utf-8'), p2.encode('utf-8')))
         self._merges = make_shared[Merges](c_merges)
 
+    def __copy__(self):
+        return ByteBPEProcessor(vocab=self.vocab, merges=self.merges)
+
+    def __deepcopy__(self, memo):
+        # We don't need a deepcopy of the vocab and merges dicts as their
+        # contents will be copied into a backing store in the c'tor.
+        return ByteBPEProcessor(vocab=self.vocab, merges=self.merges)
+
     @staticmethod
     def load_from_files(*, vocab: Path, merges: Path) -> ByteBPEProcessor:
         """Construct a processor from the given vocabulary and merges files."""
diff --git a/curated_tokenizers/_spp.pyx b/curated_tokenizers/_spp.pyx
index 449a7ef..ad1cba9 100644
--- a/curated_tokenizers/_spp.pyx
+++ b/curated_tokenizers/_spp.pyx
@@ -12,6 +12,14 @@ cdef class SentencePieceProcessor:
         _check_status(deref(self.spp).status())
         return deref(self.spp).GetPieceSize()
 
+    def __copy__(self):
+        return SentencePieceProcessor.from_protobuf(self.to_protobuf())
+
+    def __deepcopy__(self, memo):
+        result = SentencePieceProcessor.from_protobuf(self.to_protobuf())
+        memo[id(self)] = result
+        return result
+
     @staticmethod
     def from_file(str filename):
         """
@@ -34,6 +42,9 @@ cdef class SentencePieceProcessor:
                 protocol buffer.
         """
         cdef SentencePieceProcessor processor = SentencePieceProcessor.__new__(SentencePieceProcessor)
+        if len(protobuf) == 0:
+            # SentencePiece returns an empty protobuf for uninitialized models.
+            return processor
         cdef string_view protobuf_view = string_view(protobuf, len(protobuf))
         _check_status(deref(processor.spp).LoadFromSerializedProto(protobuf_view))
         return processor
diff --git a/curated_tokenizers/_wordpiece.pyx b/curated_tokenizers/_wordpiece.pyx
index 1976aa1..204ff1c 100644
--- a/curated_tokenizers/_wordpiece.pyx
+++ b/curated_tokenizers/_wordpiece.pyx
@@ -18,6 +18,13 @@ cdef class WordPieceProcessor:
             byte_array = piece[2:].encode('utf8') if not is_initial else piece.encode('utf8')
             self._pieces.add_piece(byte_array, is_initial)
 
+    def __copy__(self):
+        # This is essentially a deepcopy, but there's no better way to do it.
+        return WordPieceProcessor(self.to_list())
+
+    def __deepcopy__(self, memo):
+        return WordPieceProcessor(self.to_list())
+
     def encode(self, token: str) -> Tuple[List[int], List[str]]:
         """
         Encode a token using word pieces. The piece identifiers are


Test patch:
diff --git a/curated_tokenizers/tests/test_sp.py b/curated_tokenizers/tests/test_sp.py
index db2f8db..f1af6f5 100644
--- a/curated_tokenizers/tests/test_sp.py
+++ b/curated_tokenizers/tests/test_sp.py
@@ -22,6 +22,10 @@ def test_load_proto(test_dir):
     serialized_data = spp.to_protobuf()
     assert serialized_data == data
 
+    # Zero-length buffer.
+    spp = SentencePieceProcessor.from_protobuf(bytes())
+    assert spp.to_protobuf() == bytes()
+
 
 def test_load_unknown_file():
     with pytest.raises(OSError, match=r"No such file"):