Repository: deepjavalibrary/djl
Pull request #1399 (2021-12-02 06:19:12)

add sentencepiece load from bytes method
## Description ##

Add SentencePiece load from byte array method.

codecov-commenter: # [Codecov](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) Report
> Merging [#1399](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) (ea9c4c2) into [master](https://codecov.io/gh/deepjavalibrary/djl/commit/bb5073f7ffc33f1883621fc1783dbb01e70b418f?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) (bb5073f) will **decrease** coverage by `1.57%`.
> The diff coverage is `50.42%`.

[![Impacted file tree graph](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/graphs/tree.svg?width=650&height=150&src=pr&token=mDKVNwFXRV&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary)](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary)

```diff
@@             Coverage Diff              @@
##             master    #1399      +/-   ##
============================================
- Coverage     72.08%   70.51%   -1.58%     
- Complexity     5126     5237     +111     
============================================
  Files           473      487      +14     
  Lines         21970    23042    +1072     
  Branches       2351     2516     +165     
============================================
+ Hits          15838    16247     +409     
- Misses         4925     5529     +604     
- Partials       1207     1266      +59     
```


| [Impacted Files](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) | Coverage Δ | |
|---|---|---|
| [api/src/main/java/ai/djl/modality/cv/Image.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L0ltYWdlLmphdmE=) | `69.23% <ø> (-4.11%)` | :arrow_down: |
| [...i/djl/modality/cv/translator/BigGANTranslator.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvQmlnR0FOVHJhbnNsYXRvci5qYXZh) | `21.42% <ø> (-5.24%)` | :arrow_down: |
| [...odality/cv/translator/BigGANTranslatorFactory.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvQmlnR0FOVHJhbnNsYXRvckZhY3RvcnkuamF2YQ==) | `33.33% <0.00%> (+8.33%)` | :arrow_up: |
| [...nslator/InstanceSegmentationTranslatorFactory.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvSW5zdGFuY2VTZWdtZW50YXRpb25UcmFuc2xhdG9yRmFjdG9yeS5qYXZh) | `14.28% <0.00%> (-3.90%)` | :arrow_down: |
| [.../modality/cv/translator/YoloTranslatorFactory.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvWW9sb1RyYW5zbGF0b3JGYWN0b3J5LmphdmE=) | `8.33% <0.00%> (-1.67%)` | :arrow_down: |
| [...i/djl/modality/cv/translator/YoloV5Translator.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvWW9sb1Y1VHJhbnNsYXRvci5qYXZh) | `5.69% <0.00%> (ø)` | |
| [...odality/cv/translator/YoloV5TranslatorFactory.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvWW9sb1Y1VHJhbnNsYXRvckZhY3RvcnkuamF2YQ==) | `8.33% <0.00%> (-1.67%)` | :arrow_down: |
| [...modality/nlp/embedding/TrainableWordEmbedding.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L25scC9lbWJlZGRpbmcvVHJhaW5hYmxlV29yZEVtYmVkZGluZy5qYXZh) | `45.00% <0.00%> (-3.34%)` | :arrow_down: |
| [...pi/src/main/java/ai/djl/ndarray/BytesSupplier.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL25kYXJyYXkvQnl0ZXNTdXBwbGllci5qYXZh) | `54.54% <0.00%> (-12.13%)` | :arrow_down: |
| [...i/src/main/java/ai/djl/ndarray/NDArrayAdapter.java](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL25kYXJyYXkvTkRBcnJheUFkYXB0ZXIuamF2YQ==) | `15.23% <0.00%> (-0.83%)` | :arrow_down: |
| ... and [143 more](https://codecov.io/gh/deepjavalibrary/djl/pull/1399/diff?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary). Last update [11d9974...ea9c4c2](https://codecov.io/gh/deepjavalibrary/djl/pull/1399?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).


Patch:
diff --git a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java
index d2c6913c7..44d05936e 100644
--- a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java
+++ b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpProcessor.java
@@ -44,6 +44,10 @@ final class SpProcessor extends NativeResource<Long> {
         SentencePieceLibrary.LIB.loadModel(getHandle(), path);
     }
 
+    void loadModelFromBytes(byte[] serializedProto) {
+        SentencePieceLibrary.LIB.loadModelFromBytes(getHandle(), serializedProto);
+    }
+
     /**
      * Tokenize a sentence into array of tokens.
      *
diff --git a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java
index d11c2a903..613fcced4 100644
--- a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java
+++ b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/SpTokenizer.java
@@ -50,6 +50,16 @@ public class SpTokenizer implements Tokenizer, AutoCloseable {
         loadModel(modelPath, prefix);
     }
 
+    /**
+     * Creates a SentencePiece Tokenizer from byte array.
+     *
+     * @param serializedModel the serialized model
+     */
+    public SpTokenizer(byte[] serializedModel) {
+        this.processor = SpProcessor.newInstance();
+        processor.loadModelFromBytes(serializedModel);
+    }
+
     /** {@inheritDoc} */
     @Override
     public List<String> tokenize(String sentence) {
diff --git a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/jni/SentencePieceLibrary.java b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/jni/SentencePieceLibrary.java
index 2bf7b09bc..35cbd4d3f 100644
--- a/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/jni/SentencePieceLibrary.java
+++ b/extensions/sentencepiece/src/main/java/ai/djl/sentencepiece/jni/SentencePieceLibrary.java
@@ -24,6 +24,8 @@ public final class SentencePieceLibrary {
 
     public native void loadModel(long handle, String filePath);
 
+    public native void loadModelFromBytes(long handle, byte[] bytes);
+
     public native void deleteSentencePieceProcessor(long handle);
 
     public native String[] tokenize(long handle, String text);
diff --git a/extensions/sentencepiece/src/main/native/ai_djl_sentencepiece_jni_SentencePieceLibrary.cc b/extensions/sentencepiece/src/main/native/ai_djl_sentencepiece_jni_SentencePieceLibrary.cc
index 0a3352978..cf6d1c73c 100644
--- a/extensions/sentencepiece/src/main/native/ai_djl_sentencepiece_jni_SentencePieceLibrary.cc
+++ b/extensions/sentencepiece/src/main/native/ai_djl_sentencepiece_jni_SentencePieceLibrary.cc
@@ -37,6 +37,16 @@ JNIEXPORT void JNICALL Java_ai_djl_sentencepiece_jni_SentencePieceLibrary_loadMo
   CheckStatus(env, processor_ptr->Load(path_string));
 }
 
+JNIEXPORT void JNICALL Java_ai_djl_sentencepiece_jni_SentencePieceLibrary_loadModelFromBytes(
+        JNIEnv* env, jobject jthis, jlong jhandle, jbyteArray jserialized) {
+    auto* processor_ptr = reinterpret_cast<sentencepiece::SentencePieceProcessor*>(jhandle);
+    int length = env->GetArrayLength(jserialized);
+    std::vector<char> buff(length, 0);
+    env->GetByteArrayRegion(jserialized, 0, length, reinterpret_cast<jbyte*>(buff.data()));
+    std::string serialized(buff.data(), buff.size());
+    CheckStatus(env, processor_ptr->LoadFromSerializedProto(serialized));
+}
+
 JNIEXPORT void JNICALL Java_ai_djl_sentencepiece_jni_SentencePieceLibrary_deleteSentencePieceProcessor(
     JNIEnv* env, jobject jthis, jlong jhandle) {
   auto* processor_ptr = reinterpret_cast<sentencepiece::SentencePieceProcessor*>(jhandle);


Test patch:
diff --git a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
index 1b2f97f22..e14360390 100644
--- a/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
+++ b/extensions/sentencepiece/src/test/java/ai/djl/sentencepiece/SpTokenizerTest.java
@@ -37,6 +37,22 @@ public class SpTokenizerTest {
         }
     }
 
+    @Test
+    public void testLoadFromBytes() throws IOException {
+        TestRequirements.notWindows();
+
+        Path modelPath = Paths.get("build/test/models/sententpiece_test_model.model");
+        byte[] bytes = Files.readAllBytes(modelPath);
+        try (SpTokenizer tokenizer = new SpTokenizer(bytes)) {
+            String original = "Hello World";
+            List<String> tokens = tokenizer.tokenize(original);
+            List<String> expected = Arrays.asList("▁He", "ll", "o", "▁", "W", "or", "l", "d");
+            Assert.assertEquals(tokens, expected);
+            String recovered = tokenizer.buildSentence(tokens);
+            Assert.assertEquals(original, recovered);
+        }
+    }
+
     @Test
     public void testTokenize() throws IOException {
         TestRequirements.notWindows();