Repository: deepjavalibrary/djl
Pull request #2857 (2023-11-16 19:27:00)

[tokenizer] Not returns overflow tokens by default
Fixes: #2846

codecov-commenter: ## [Codecov](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) Report
Attention: `1363 lines` in your changes are missing coverage. Please review.
> Comparison is base [(`bb5073f`)](https://app.codecov.io/gh/deepjavalibrary/djl/commit/bb5073f7ffc33f1883621fc1783dbb01e70b418f?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) 72.08% compared to head [(`e356f6e`)](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) 72.32%.
> Report is 912 commits behind head on master.

| [Files](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) | Patch % | Lines |
|---|---|---|
| [...va/ai/djl/modality/nlp/generate/TextGenerator.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L25scC9nZW5lcmF0ZS9UZXh0R2VuZXJhdG9yLmphdmE=) | 2.81% | [276 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [.../java/ai/djl/modality/nlp/generate/SeqBatcher.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L25scC9nZW5lcmF0ZS9TZXFCYXRjaGVyLmphdmE=) | 0.75% | [132 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [...ity/nlp/generate/ContrastiveSeqBatchScheduler.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L25scC9nZW5lcmF0ZS9Db250cmFzdGl2ZVNlcUJhdGNoU2NoZWR1bGVyLmphdmE=) | 2.97% | [98 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [...i/djl/modality/nlp/generate/SeqBatchScheduler.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L25scC9nZW5lcmF0ZS9TZXFCYXRjaFNjaGVkdWxlci5qYXZh) | 9.83% | [55 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [.../java/ai/djl/modality/cv/BufferedImageFactory.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L0J1ZmZlcmVkSW1hZ2VGYWN0b3J5LmphdmE=) | 40.96% | [47 Missing and 2 partials :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [...a/ai/djl/modality/nlp/generate/StepGeneration.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L25scC9nZW5lcmF0ZS9TdGVwR2VuZXJhdGlvbi5qYXZh) | 2.04% | [48 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [api/src/main/java/ai/djl/ndarray/NDArray.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL25kYXJyYXkvTkRBcnJheS5qYXZh) | 43.42% | [39 Missing and 4 partials :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [...n/java/ai/djl/modality/cv/output/CategoryMask.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L291dHB1dC9DYXRlZ29yeU1hc2suamF2YQ==) | 22.00% | [39 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [...i/src/main/java/ai/djl/ndarray/NDArrayAdapter.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL25kYXJyYXkvTkRBcnJheUFkYXB0ZXIuamF2YQ==) | 71.21% | [31 Missing and 7 partials :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| [.../cv/translator/SemanticSegmentationTranslator.java](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary#diff-YXBpL3NyYy9tYWluL2phdmEvYWkvZGpsL21vZGFsaXR5L2N2L3RyYW5zbGF0b3IvU2VtYW50aWNTZWdtZW50YXRpb25UcmFuc2xhdG9yLmphdmE=) | 37.50% | [35 Missing :warning: ](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) |
| ... and [76 more](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) | |

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff              @@
##             master    #2857      +/-   ##
============================================
+ Coverage     72.08%   72.32%   +0.23%     
- Complexity     5126     7165    +2039     
============================================
  Files           473      708     +235     
  Lines         21970    31915    +9945     
  Branches       2351     3314     +963     
============================================
+ Hits          15838    23082    +7244     
- Misses         4925     7254    +2329     
- Partials       1207     1579     +372     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/deepjavalibrary/djl/pull/2857?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).


Patch:
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
index fa601e155..de26f1b06 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
@@ -44,6 +44,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
     private static final Logger logger = LoggerFactory.getLogger(HuggingFaceTokenizer.class);
 
     private boolean addSpecialTokens;
+    private boolean withOverflowingTokens;
     private TruncationStrategy truncation;
     private PaddingStrategy padding;
     private int maxLength;
@@ -64,6 +65,8 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
         if (options != null) {
             val = options.getOrDefault("addSpecialTokens", "true");
             addSpecialTokens = Boolean.parseBoolean(val);
+            val = options.getOrDefault("withOverflowingTokens", "false");
+            withOverflowingTokens = Boolean.parseBoolean(val);
             modelMaxLength = ArgumentsUtil.intValue(options, "modelMaxLength", 512);
             if (options.containsKey("truncation")) {
                 truncation = TruncationStrategy.fromValue(options.get("truncation"));
@@ -203,11 +206,12 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param text the input sentence
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input sentence
      */
-    public Encoding encode(String text, boolean addSpecialTokens) {
+    public Encoding encode(String text, boolean addSpecialTokens, boolean withOverflowingTokens) {
         long encoding = TokenizersLibrary.LIB.encode(getHandle(), text, addSpecialTokens);
-        return toEncoding(encoding);
+        return toEncoding(encoding, withOverflowingTokens);
     }
 
     /**
@@ -217,7 +221,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input sentence
      */
     public Encoding encode(String text) {
-        return encode(text, addSpecialTokens);
+        return encode(text, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -227,12 +231,14 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param textPair the second input sentence
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input sentence
      */
-    public Encoding encode(String text, String textPair, boolean addSpecialTokens) {
+    public Encoding encode(
+            String text, String textPair, boolean addSpecialTokens, boolean withOverflowingTokens) {
         long encoding =
                 TokenizersLibrary.LIB.encodeDual(getHandle(), text, textPair, addSpecialTokens);
-        return toEncoding(encoding);
+        return toEncoding(encoding, withOverflowingTokens);
     }
 
     /**
@@ -243,7 +249,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input sentence
      */
     public Encoding encode(String text, String textPair) {
-        return encode(text, textPair, addSpecialTokens);
+        return encode(text, textPair, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -252,11 +258,13 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param inputs the input sentences
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input sentences
      */
-    public Encoding encode(List<String> inputs, boolean addSpecialTokens) {
+    public Encoding encode(
+            List<String> inputs, boolean addSpecialTokens, boolean withOverflowingTokens) {
         String[] array = inputs.toArray(Utils.EMPTY_ARRAY);
-        return encode(array, addSpecialTokens);
+        return encode(array, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -266,7 +274,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input sentences
      */
     public Encoding encode(List<String> inputs) {
-        return encode(inputs, addSpecialTokens);
+        return encode(inputs, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -275,11 +283,13 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param inputs the input sentences
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input sentences
      */
-    public Encoding encode(String[] inputs, boolean addSpecialTokens) {
+    public Encoding encode(
+            String[] inputs, boolean addSpecialTokens, boolean withOverflowingTokens) {
         long encoding = TokenizersLibrary.LIB.encodeList(getHandle(), inputs, addSpecialTokens);
-        return toEncoding(encoding);
+        return toEncoding(encoding, withOverflowingTokens);
     }
 
     /**
@@ -289,7 +299,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input sentences
      */
     public Encoding encode(String[] inputs) {
-        return encode(inputs, addSpecialTokens);
+        return encode(inputs, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -298,11 +308,13 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param inputs the batch of input sentence
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input sentence in batch
      */
-    public Encoding[] batchEncode(List<String> inputs, boolean addSpecialTokens) {
+    public Encoding[] batchEncode(
+            List<String> inputs, boolean addSpecialTokens, boolean withOverflowingTokens) {
         String[] array = inputs.toArray(Utils.EMPTY_ARRAY);
-        return batchEncode(array, addSpecialTokens);
+        return batchEncode(array, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -312,7 +324,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input sentence in batch
      */
     public Encoding[] batchEncode(List<String> inputs) {
-        return batchEncode(inputs, addSpecialTokens);
+        return batchEncode(inputs, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -321,13 +333,15 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param inputs the batch of input sentence
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input sentence in batch
      */
-    public Encoding[] batchEncode(String[] inputs, boolean addSpecialTokens) {
+    public Encoding[] batchEncode(
+            String[] inputs, boolean addSpecialTokens, boolean withOverflowingTokens) {
         long[] encodings = TokenizersLibrary.LIB.batchEncode(getHandle(), inputs, addSpecialTokens);
         Encoding[] ret = new Encoding[encodings.length];
         for (int i = 0; i < encodings.length; ++i) {
-            ret[i] = toEncoding(encodings[i]);
+            ret[i] = toEncoding(encodings[i], withOverflowingTokens);
         }
         return ret;
     }
@@ -339,7 +353,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input sentence in batch
      */
     public Encoding[] batchEncode(String[] inputs) {
-        return batchEncode(inputs, addSpecialTokens);
+        return batchEncode(inputs, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -348,9 +362,13 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @param inputs the batch of input text pair
      * @param addSpecialTokens whether to encode the sequence with special tokens relative to their
      *     model
+     * @param withOverflowingTokens whether to return overflowing tokens
      * @return the {@code Encoding} of the input text pair in batch
      */
-    public Encoding[] batchEncode(PairList<String, String> inputs, boolean addSpecialTokens) {
+    public Encoding[] batchEncode(
+            PairList<String, String> inputs,
+            boolean addSpecialTokens,
+            boolean withOverflowingTokens) {
         String[] text = inputs.keyArray(Utils.EMPTY_ARRAY);
         String[] textPair = inputs.valueArray(Utils.EMPTY_ARRAY);
         long[] encodings =
@@ -358,7 +376,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
                         getHandle(), text, textPair, addSpecialTokens);
         Encoding[] ret = new Encoding[encodings.length];
         for (int i = 0; i < encodings.length; ++i) {
-            ret[i] = toEncoding(encodings[i]);
+            ret[i] = toEncoding(encodings[i], withOverflowingTokens);
         }
         return ret;
     }
@@ -370,7 +388,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
      * @return the {@code Encoding} of the input text pair in batch
      */
     public Encoding[] batchEncode(PairList<String, String> inputs) {
-        return batchEncode(inputs, addSpecialTokens);
+        return batchEncode(inputs, addSpecialTokens, withOverflowingTokens);
     }
 
     /**
@@ -503,7 +521,7 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
         }
     }
 
-    private Encoding toEncoding(long encoding) {
+    private Encoding toEncoding(long encoding, boolean withOverflowingTokens) {
         long[] ids = TokenizersLibrary.LIB.getTokenIds(encoding);
         long[] typeIds = TokenizersLibrary.LIB.getTypeIds(encoding);
         String[] tokens = TokenizersLibrary.LIB.getTokens(encoding);
@@ -511,11 +529,17 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
         long[] attentionMask = TokenizersLibrary.LIB.getAttentionMask(encoding);
         long[] specialTokenMask = TokenizersLibrary.LIB.getSpecialTokenMask(encoding);
         CharSpan[] charSpans = TokenizersLibrary.LIB.getTokenCharSpans(encoding);
-        long[] overflowingHandles = TokenizersLibrary.LIB.getOverflowing(encoding);
 
-        Encoding[] overflowing = new Encoding[overflowingHandles.length];
-        for (int i = 0; i < overflowingHandles.length; ++i) {
-            overflowing[i] = toEncoding(overflowingHandles[i]);
+        Encoding[] overflowing;
+        if (withOverflowingTokens) {
+            long[] overflowingHandles = TokenizersLibrary.LIB.getOverflowing(encoding);
+
+            overflowing = new Encoding[overflowingHandles.length];
+            for (int i = 0; i < overflowingHandles.length; ++i) {
+                overflowing[i] = toEncoding(overflowingHandles[i], true);
+            }
+        } else {
+            overflowing = new Encoding[0];
         }
 
         TokenizersLibrary.LIB.deleteEncoding(encoding);
@@ -651,6 +675,17 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
             return this;
         }
 
+        /**
+         * Sets if add special tokens.
+         *
+         * @param withOverflowingTokens true to return overflowing tokens
+         * @return this builder
+         */
+        public Builder optWithOverflowingTokens(boolean withOverflowingTokens) {
+            options.put("withOverflowingTokens", String.valueOf(withOverflowingTokens));
+            return this;
+        }
+
         /**
          * Enables or Disables default truncation behavior for the tokenizer.
          *
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java
index 43b120cac..9a4ccba42 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java
@@ -37,7 +37,7 @@ public class FillMaskBatchTranslator implements NoBatchifyTranslator<String[], C
         this.maskToken = maskToken;
         this.topK = topK;
         this.batchifier = batchifier;
-        Encoding encoding = tokenizer.encode(maskToken, false);
+        Encoding encoding = tokenizer.encode(maskToken, false, false);
         maskTokenId = encoding.getIds()[0];
     }
 
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java
index e99c32c9d..6401b4d0a 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java
@@ -44,7 +44,7 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
         this.maskToken = maskToken;
         this.topK = topK;
         this.batchifier = batchifier;
-        Encoding encoding = tokenizer.encode(maskToken, false);
+        Encoding encoding = tokenizer.encode(maskToken, false, false);
         maskTokenId = encoding.getIds()[0];
     }
 


Test patch:
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
index 2bc30d4bd..8b5d6d575 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -300,6 +300,7 @@ public class HuggingFaceTokenizerTest {
                 HuggingFaceTokenizer.builder()
                         .optTokenizerName("bert-base-cased")
                         .optAddSpecialTokens(false)
+                        .optWithOverflowingTokens(true)
                         .optTruncation(true)
                         .optMaxLength(3)
                         .optStride(1)
@@ -322,6 +323,7 @@ public class HuggingFaceTokenizerTest {
                 HuggingFaceTokenizer.builder()
                         .optTokenizerName("bert-base-cased")
                         .optAddSpecialTokens(false)
+                        .optWithOverflowingTokens(true)
                         .optTruncation(true)
                         .optMaxLength(8)
                         .optStride(2)
@@ -458,13 +460,13 @@ public class HuggingFaceTokenizerTest {
             Assert.assertEquals(outputs, outputsWithSpecialTokens);
 
             // encode with special tokens, decode with special tokens
-            encodings = tokenizer.batchEncode(inputs, true);
+            encodings = tokenizer.batchEncode(inputs, true, false);
             batchIds = Arrays.stream(encodings).map(Encoding::getIds).toArray(long[][]::new);
             outputs = tokenizer.batchDecode(batchIds, false);
             Assert.assertEquals(outputs, outputsWithSpecialTokens);
 
             // encode without special tokens, decode without special tokens
-            encodings = tokenizer.batchEncode(inputs, false);
+            encodings = tokenizer.batchEncode(inputs, false, false);
             batchIds = Arrays.stream(encodings).map(Encoding::getIds).toArray(long[][]::new);
             outputs = tokenizer.batchDecode(batchIds, true);
             Assert.assertEquals(outputs, outputsWithoutSpecialTokens);