Repository: deepjavalibrary/djl
Pull request #3040 (2024-03-21 03:57:48)

Updates dependencies version to latest
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here


codecov-commenter: ## [Codecov](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3040?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 72.22%. Comparing base [(`e3a8e4c`)](https://app.codecov.io/gh/deepjavalibrary/djl/commit/e3a8e4cd794cae96afa5dd2b4cbf390dbd7e22da?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) to head [(`a9d1bcc`)](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3040?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).


:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff              @@
##             master    #3040      +/-   ##
============================================
- Coverage     72.29%   72.22%   -0.08%     
+ Complexity     7292     7288       -4     
============================================
  Files           722      722              
  Lines         32513    32513              
  Branches       3396     3396              
============================================
- Hits          23505    23482      -23     
- Misses         7387     7412      +25     
+ Partials       1621     1619       -2     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3040?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).


Patch:
diff --git a/api/src/main/java/ai/djl/ndarray/NDSerializer.java b/api/src/main/java/ai/djl/ndarray/NDSerializer.java
index f42be7f08..26dde5d92 100644
--- a/api/src/main/java/ai/djl/ndarray/NDSerializer.java
+++ b/api/src/main/java/ai/djl/ndarray/NDSerializer.java
@@ -83,6 +83,16 @@ final class NDSerializer {
         Shape shape = array.getShape();
         dos.write(shape.getEncoded());
 
+        if (array.getDataType() == DataType.STRING) {
+            String[] data = array.toStringArray();
+            dos.writeInt(data.length);
+            for (String str : data) {
+                dos.writeUTF(str);
+            }
+            dos.flush();
+            return;
+        }
+
         ByteBuffer bb = array.toByteBuffer();
         dos.write(bb.order() == ByteOrder.BIG_ENDIAN ? '>' : '<');
         int length = bb.remaining();
@@ -167,6 +177,17 @@ final class NDSerializer {
         // Shape
         Shape shape = Shape.decode(bb);
 
+        if (dataType == DataType.STRING) {
+            int size = bb.getInt();
+            String[] data = new String[size];
+            for (int i = 0; i < size; ++i) {
+                data[i] = readUTF(bb);
+            }
+            NDArray array = manager.create(data, StandardCharsets.UTF_8, shape);
+            array.setName(name);
+            return array;
+        }
+
         // Data
         ByteOrder order;
         if (version > 2) {
diff --git a/engines/pytorch/pytorch-engine/src/main/java/ai/djl/pytorch/engine/PtNDArray.java b/engines/pytorch/pytorch-engine/src/main/java/ai/djl/pytorch/engine/PtNDArray.java
index 499f51eba..551a16d03 100644
--- a/engines/pytorch/pytorch-engine/src/main/java/ai/djl/pytorch/engine/PtNDArray.java
+++ b/engines/pytorch/pytorch-engine/src/main/java/ai/djl/pytorch/engine/PtNDArray.java
@@ -100,6 +100,7 @@ public class PtNDArray extends NativeResource<Long> implements NDArray {
         super(-1L);
         this.manager = manager;
         this.strs = strs;
+        this.sparseFormat = SparseFormat.DENSE;
         this.shape = shape;
         this.dataType = DataType.STRING;
         NDScope.register(this);
@@ -225,6 +226,10 @@ public class PtNDArray extends NativeResource<Long> implements NDArray {
     /** {@inheritDoc} */
     @Override
     public ByteBuffer toByteBuffer() {
+        if (getDataType() == DataType.STRING) {
+            throw new UnsupportedOperationException(
+                    "toByteBuffer is not supported for String tensor.");
+        }
         return JniUtils.getByteBuffer(this);
     }
 
@@ -429,6 +434,9 @@ public class PtNDArray extends NativeResource<Long> implements NDArray {
         if (getDataType() != other.getDataType()) {
             return false;
         }
+        if (getDataType() == DataType.STRING) {
+            return Arrays.equals(toStringArray(), other.toStringArray());
+        }
         return JniUtils.contentEqual(this, manager.from(other));
     }
 
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java
index 9a4ccba42..ee4e9cf96 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskBatchTranslator.java
@@ -29,13 +29,19 @@ public class FillMaskBatchTranslator implements NoBatchifyTranslator<String[], C
     private String maskToken;
     private long maskTokenId;
     private int topK;
+    private boolean includeTokenTypes;
     private Batchifier batchifier;
 
     FillMaskBatchTranslator(
-            HuggingFaceTokenizer tokenizer, String maskToken, int topK, Batchifier batchifier) {
+            HuggingFaceTokenizer tokenizer,
+            String maskToken,
+            int topK,
+            boolean includeTokenTypes,
+            Batchifier batchifier) {
         this.tokenizer = tokenizer;
         this.maskToken = maskToken;
         this.topK = topK;
+        this.includeTokenTypes = includeTokenTypes;
         this.batchifier = batchifier;
         Encoding encoding = tokenizer.encode(maskToken, false, false);
         maskTokenId = encoding.getIds()[0];
@@ -52,7 +58,7 @@ public class FillMaskBatchTranslator implements NoBatchifyTranslator<String[], C
         for (int i = 0; i < encodings.length; ++i) {
             long[] indices = encodings[i].getIds();
             maskIndices[i] = FillMaskTranslator.getMaskIndex(indices, maskToken, maskTokenId);
-            batch[i] = encodings[i].toNDList(manager, false);
+            batch[i] = encodings[i].toNDList(manager, includeTokenTypes);
         }
         return batchifier.batchify(batch);
     }
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java
index 6401b4d0a..87c819816 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/FillMaskTranslator.java
@@ -36,13 +36,19 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
     private String maskToken;
     private long maskTokenId;
     private int topK;
+    private boolean includeTokenTypes;
     private Batchifier batchifier;
 
     FillMaskTranslator(
-            HuggingFaceTokenizer tokenizer, String maskToken, int topK, Batchifier batchifier) {
+            HuggingFaceTokenizer tokenizer,
+            String maskToken,
+            int topK,
+            boolean includeTokenTypes,
+            Batchifier batchifier) {
         this.tokenizer = tokenizer;
         this.maskToken = maskToken;
         this.topK = topK;
+        this.includeTokenTypes = includeTokenTypes;
         this.batchifier = batchifier;
         Encoding encoding = tokenizer.encode(maskToken, false, false);
         maskTokenId = encoding.getIds()[0];
@@ -61,7 +67,7 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
         long[] indices = encoding.getIds();
         int maskIndex = getMaskIndex(indices, maskToken, maskTokenId);
         ctx.setAttachment("maskIndex", maskIndex);
-        return encoding.toNDList(ctx.getNDManager(), false);
+        return encoding.toNDList(ctx.getNDManager(), includeTokenTypes);
     }
 
     /** {@inheritDoc} */
@@ -75,7 +81,8 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
     @Override
     public FillMaskBatchTranslator toBatchTranslator(Batchifier batchifier) {
         tokenizer.enableBatch();
-        return new FillMaskBatchTranslator(tokenizer, maskToken, topK, batchifier);
+        return new FillMaskBatchTranslator(
+                tokenizer, maskToken, topK, includeTokenTypes, batchifier);
     }
 
     static int getMaskIndex(long[] indices, String maskToken, long maskTokenId)
@@ -139,6 +146,7 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
         private HuggingFaceTokenizer tokenizer;
         private String maskedToken = "[MASK]";
         private int topK = 5;
+        private boolean includeTokenTypes;
         private Batchifier batchifier = Batchifier.STACK;
 
         Builder(HuggingFaceTokenizer tokenizer) {
@@ -167,6 +175,17 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
             return this;
         }
 
+        /**
+         * Sets if include token types for the {@link Translator}.
+         *
+         * @param includeTokenTypes true to include token types
+         * @return this builder
+         */
+        public Builder optIncludeTokenTypes(boolean includeTokenTypes) {
+            this.includeTokenTypes = includeTokenTypes;
+            return this;
+        }
+
         /**
          * Sets the {@link Batchifier} for the {@link Translator}.
          *
@@ -186,6 +205,7 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
         public void configure(Map<String, ?> arguments) {
             optMaskToken(ArgumentsUtil.stringValue(arguments, "maskToken", "[MASK]"));
             optTopK(ArgumentsUtil.intValue(arguments, "topK", 5));
+            optIncludeTokenTypes(ArgumentsUtil.booleanValue(arguments, "includeTokenTypes"));
             String batchifierStr = ArgumentsUtil.stringValue(arguments, "batchifier", "stack");
             optBatchifier(Batchifier.fromString(batchifierStr));
         }
@@ -197,7 +217,8 @@ public class FillMaskTranslator implements Translator<String, Classifications> {
          * @throws IOException if I/O error occurs
          */
         public FillMaskTranslator build() throws IOException {
-            return new FillMaskTranslator(tokenizer, maskedToken, topK, batchifier);
+            return new FillMaskTranslator(
+                    tokenizer, maskedToken, topK, includeTokenTypes, batchifier);
         }
     }
 }
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationBatchTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationBatchTranslator.java
index 6c9beda25..72be9a8aa 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationBatchTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationBatchTranslator.java
@@ -32,11 +32,14 @@ public class TextClassificationBatchTranslator
         implements NoBatchifyTranslator<String[], Classifications[]> {
 
     private HuggingFaceTokenizer tokenizer;
+    private boolean includeTokenTypes;
     private Batchifier batchifier;
     private PretrainedConfig config;
 
-    TextClassificationBatchTranslator(HuggingFaceTokenizer tokenizer, Batchifier batchifier) {
+    TextClassificationBatchTranslator(
+            HuggingFaceTokenizer tokenizer, boolean includeTokenTypes, Batchifier batchifier) {
         this.tokenizer = tokenizer;
+        this.includeTokenTypes = includeTokenTypes;
         this.batchifier = batchifier;
     }
 
@@ -56,7 +59,7 @@ public class TextClassificationBatchTranslator
         Encoding[] encodings = tokenizer.batchEncode(inputs);
         NDList[] batch = new NDList[encodings.length];
         for (int i = 0; i < encodings.length; ++i) {
-            batch[i] = encodings[i].toNDList(manager, false);
+            batch[i] = encodings[i].toNDList(manager, includeTokenTypes);
         }
         return batchifier.batchify(batch);
     }
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationTranslator.java
index d624b69d7..defa0f90b 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TextClassificationTranslator.java
@@ -35,11 +35,14 @@ import java.util.Map;
 public class TextClassificationTranslator implements Translator<String, Classifications> {
 
     private HuggingFaceTokenizer tokenizer;
+    private boolean includeTokenTypes;
     private Batchifier batchifier;
     private PretrainedConfig config;
 
-    TextClassificationTranslator(HuggingFaceTokenizer tokenizer, Batchifier batchifier) {
+    TextClassificationTranslator(
+            HuggingFaceTokenizer tokenizer, boolean includeTokenTypes, Batchifier batchifier) {
         this.tokenizer = tokenizer;
+        this.includeTokenTypes = includeTokenTypes;
         this.batchifier = batchifier;
     }
 
@@ -63,7 +66,7 @@ public class TextClassificationTranslator implements Translator<String, Classifi
     @Override
     public NDList processInput(TranslatorContext ctx, String input) {
         Encoding encoding = tokenizer.encode(input);
-        return encoding.toNDList(ctx.getNDManager(), false);
+        return encoding.toNDList(ctx.getNDManager(), includeTokenTypes);
     }
 
     /** {@inheritDoc} */
@@ -76,7 +79,7 @@ public class TextClassificationTranslator implements Translator<String, Classifi
     @Override
     public TextClassificationBatchTranslator toBatchTranslator(Batchifier batchifier) {
         tokenizer.enableBatch();
-        return new TextClassificationBatchTranslator(tokenizer, batchifier);
+        return new TextClassificationBatchTranslator(tokenizer, includeTokenTypes, batchifier);
     }
 
     static Classifications toClassifications(PretrainedConfig config, NDList list) {
@@ -127,12 +130,24 @@ public class TextClassificationTranslator implements Translator<String, Classifi
     public static final class Builder {
 
         private HuggingFaceTokenizer tokenizer;
+        private boolean includeTokenTypes;
         private Batchifier batchifier = Batchifier.STACK;
 
         Builder(HuggingFaceTokenizer tokenizer) {
             this.tokenizer = tokenizer;
         }
 
+        /**
+         * Sets if include token types for the {@link Translator}.
+         *
+         * @param includeTokenTypes true to include token types
+         * @return this builder
+         */
+        public Builder optIncludeTokenTypes(boolean includeTokenTypes) {
+            this.includeTokenTypes = includeTokenTypes;
+            return this;
+        }
+
         /**
          * Sets the {@link Batchifier} for the {@link Translator}.
          *
@@ -150,6 +165,7 @@ public class TextClassificationTranslator implements Translator<String, Classifi
          * @param arguments the model arguments
          */
         public void configure(Map<String, ?> arguments) {
+            optIncludeTokenTypes(ArgumentsUtil.booleanValue(arguments, "includeTokenTypes"));
             String batchifierStr = ArgumentsUtil.stringValue(arguments, "batchifier", "stack");
             optBatchifier(Batchifier.fromString(batchifierStr));
         }
@@ -161,7 +177,7 @@ public class TextClassificationTranslator implements Translator<String, Classifi
          * @throws IOException if I/O error occurs
          */
         public TextClassificationTranslator build() throws IOException {
-            return new TextClassificationTranslator(tokenizer, batchifier);
+            return new TextClassificationTranslator(tokenizer, includeTokenTypes, batchifier);
         }
     }
 }
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationBatchTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationBatchTranslator.java
index 2ae45438c..b3b4ac0d3 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationBatchTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationBatchTranslator.java
@@ -32,11 +32,14 @@ public class TokenClassificationBatchTranslator
         implements NoBatchifyTranslator<String[], NamedEntity[][]> {
 
     private HuggingFaceTokenizer tokenizer;
+    private boolean includeTokenTypes;
     private Batchifier batchifier;
     private PretrainedConfig config;
 
-    TokenClassificationBatchTranslator(HuggingFaceTokenizer tokenizer, Batchifier batchifier) {
+    TokenClassificationBatchTranslator(
+            HuggingFaceTokenizer tokenizer, boolean includeTokenTypes, Batchifier batchifier) {
         this.tokenizer = tokenizer;
+        this.includeTokenTypes = includeTokenTypes;
         this.batchifier = batchifier;
     }
 
@@ -58,7 +61,7 @@ public class TokenClassificationBatchTranslator
         ctx.setAttachment("encodings", encodings);
         NDList[] batch = new NDList[encodings.length];
         for (int i = 0; i < encodings.length; ++i) {
-            batch[i] = encodings[i].toNDList(manager, false);
+            batch[i] = encodings[i].toNDList(manager, includeTokenTypes);
         }
         return batchifier.batchify(batch);
     }
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationTranslator.java
index b1106c244..5e846be14 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationTranslator.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/TokenClassificationTranslator.java
@@ -36,11 +36,14 @@ import java.util.Map;
 public class TokenClassificationTranslator implements Translator<String, NamedEntity[]> {
 
     private HuggingFaceTokenizer tokenizer;
+    private boolean includeTokenTypes;
     private Batchifier batchifier;
     private PretrainedConfig config;
 
-    TokenClassificationTranslator(HuggingFaceTokenizer tokenizer, Batchifier batchifier) {
+    TokenClassificationTranslator(
+            HuggingFaceTokenizer tokenizer, boolean includeTokenTypes, Batchifier batchifier) {
         this.tokenizer = tokenizer;
+        this.includeTokenTypes = includeTokenTypes;
         this.batchifier = batchifier;
     }
 
@@ -65,7 +68,7 @@ public class TokenClassificationTranslator implements Translator<String, NamedEn
     public NDList processInput(TranslatorContext ctx, String input) {
         Encoding encoding = tokenizer.encode(input);
         ctx.setAttachment("encoding", encoding);
-        return encoding.toNDList(ctx.getNDManager(), false);
+        return encoding.toNDList(ctx.getNDManager(), includeTokenTypes);
     }
 
     /** {@inheritDoc} */
@@ -79,7 +82,7 @@ public class TokenClassificationTranslator implements Translator<String, NamedEn
     @Override
     public TokenClassificationBatchTranslator toBatchTranslator(Batchifier batchifier) {
         tokenizer.enableBatch();
-        return new TokenClassificationBatchTranslator(tokenizer, batchifier);
+        return new TokenClassificationBatchTranslator(tokenizer, includeTokenTypes, batchifier);
     }
 
     /**
@@ -139,12 +142,24 @@ public class TokenClassificationTranslator implements Translator<String, NamedEn
     public static final class Builder {
 
         private HuggingFaceTokenizer tokenizer;
+        private boolean includeTokenTypes;
         private Batchifier batchifier = Batchifier.STACK;
 
         Builder(HuggingFaceTokenizer tokenizer) {
             this.tokenizer = tokenizer;
         }
 
+        /**
+         * Sets if include token types for the {@link Translator}.
+         *
+         * @param includeTokenTypes true to include token types
+         * @return this builder
+         */
+        public Builder optIncludeTokenTypes(boolean includeTokenTypes) {
+            this.includeTokenTypes = includeTokenTypes;
+            return this;
+        }
+
         /**
          * Sets the {@link Batchifier} for the {@link Translator}.
          *
@@ -162,6 +177,7 @@ public class TokenClassificationTranslator implements Translator<String, NamedEn
          * @param arguments the model arguments
          */
         public void configure(Map<String, ?> arguments) {
+            optIncludeTokenTypes(ArgumentsUtil.booleanValue(arguments, "includeTokenTypes"));
             String batchifierStr = ArgumentsUtil.stringValue(arguments, "batchifier", "stack");
             optBatchifier(Batchifier.fromString(batchifierStr));
         }
@@ -173,7 +189,7 @@ public class TokenClassificationTranslator implements Translator<String, NamedEn
          * @throws IOException if I/O error occurs
          */
         public TokenClassificationTranslator build() throws IOException {
-            return new TokenClassificationTranslator(tokenizer, batchifier);
+            return new TokenClassificationTranslator(tokenizer, includeTokenTypes, batchifier);
         }
     }
 }
diff --git a/gradle.properties b/gradle.properties
index 3fe84bd6e..40ca3de54 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -28,19 +28,19 @@ lightgbm_version=3.2.110
 rapis_version=22.12.0
 
 commons_cli_version=1.6.0
-commons_compress_version=1.26.0
+commons_compress_version=1.26.1
 commons_csv_version=1.10.0
-commons_logging_version=1.2
+commons_logging_version=1.3.0
 gson_version=2.10.1
 jna_version=5.13.0
 slf4j_version=1.7.36
-log4j_slf4j_version=2.22.1
-awssdk_version=2.22.12
+log4j_slf4j_version=2.23.1
+awssdk_version=2.25.13
 hadoop_version=3.3.6
 javacpp_version=1.5.9
 javacv_version=1.5.9
 ffmpeg_version=6.0-1.5.9
-protobuf_version=3.23.3
+protobuf_version=3.25.3
 tablesaw_version=0.43.1
 spark_version=3.3.2
 openpnp_opencv_version=4.7.0-0


Test patch:
diff --git a/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java b/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java
index e89f22442..98ba89688 100644
--- a/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java
+++ b/api/src/test/java/ai/djl/ndarray/NDSerializerTest.java
@@ -97,6 +97,23 @@ public class NDSerializerTest {
         }
     }
 
+    @Test
+    public void testStringTensor() {
+        try (NDManager manager = NDManager.newBaseManager("PyTorch")) {
+            NDArray array = manager.create("hello");
+            byte[] buf = array.encode();
+            NDArray decoded = NDArray.decode(manager, buf);
+            Assert.assertTrue(decoded.getShape().isScalar());
+
+            array = manager.create(new String[] {"hello", "world"});
+            buf = array.encode();
+            decoded = NDArray.decode(manager, buf);
+            Assert.assertEquals(decoded.getShape(), array.getShape());
+            Assert.assertEquals(decoded.toStringArray()[1], "world");
+            Assert.assertEquals(decoded, array);
+        }
+    }
+
     private static byte[] encode(NDArray array) throws IOException {
         try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
             NDSerializer.encodeAsNumpy(array, bos);