Repository: deepjavalibrary/djl
Pull request #3572 (2024-12-30 20:21:10)

[tokenizer] Updates huggingface tokenizer to 0.21.0
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here


codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3572?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 58.90%. Comparing base [(`c0d3a37`)](https://app.codecov.io/gh/deepjavalibrary/djl/commit/c0d3a37dc74922f49bb9c8325965c0eba007dfad?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) to head [(`50f1c86`)](https://app.codecov.io/gh/deepjavalibrary/djl/commit/50f1c8664af541921f8951354d5607d83a20cad6?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).
> Report is 2 commits behind head on master.

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.
> :exclamation:  There is a different number of reports uploaded between BASE (c0d3a37) and HEAD (50f1c86). Click for more details.
> 
> <details><summary>HEAD has 1 upload less than BASE</summary>
>
>| Flag | BASE (c0d3a37) | HEAD (50f1c86) |
>|------|------|------|
>||2|1|
></details>

<details><summary>Additional details and impacted files</summary>


```diff
@@              Coverage Diff              @@
##             master    #3572       +/-   ##
=============================================
- Coverage     72.43%   58.90%   -13.53%     
+ Complexity     7421     5862     -1559     
=============================================
  Files           674      674               
  Lines         33087    33087               
  Branches       3526     3526               
=============================================
- Hits          23966    19490     -4476     
- Misses         7490    12117     +4627     
+ Partials       1631     1480      -151     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3572?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).


Patch:
diff --git a/android/gradle.properties b/android/gradle.properties
index 78b3fa76c..56211c900 100644
--- a/android/gradle.properties
+++ b/android/gradle.properties
@@ -17,6 +17,6 @@ org.gradle.jvmargs=-Xmx1536m
 android.useAndroidX=true
 # Automatically convert third-party libraries to use AndroidX
 android.enableJetifier=true
-djl_version=0.31.0
+djl_version=0.32.0
 pytorch_version=2.5.1
-tokenizers_version=0.20.3
+tokenizers_version=0.21.0
diff --git a/api/src/main/java/ai/djl/modality/cv/transform/Resize.java b/api/src/main/java/ai/djl/modality/cv/transform/Resize.java
index f2bddc8f5..1947b6973 100644
--- a/api/src/main/java/ai/djl/modality/cv/transform/Resize.java
+++ b/api/src/main/java/ai/djl/modality/cv/transform/Resize.java
@@ -57,6 +57,24 @@ public class Resize implements Transform {
         this.interpolation = interpolation;
     }
 
+    /**
+     * Returns the width.
+     *
+     * @return the width
+     */
+    public int getWidth() {
+        return width;
+    }
+
+    /**
+     * Returns the height.
+     *
+     * @return the height
+     */
+    public int getHeight() {
+        return height;
+    }
+
     /** {@inheritDoc} */
     @Override
     public NDArray transform(NDArray array) {
diff --git a/api/src/main/java/ai/djl/modality/cv/translator/BaseImageTranslator.java b/api/src/main/java/ai/djl/modality/cv/translator/BaseImageTranslator.java
index 3a66df732..675c4fb2f 100644
--- a/api/src/main/java/ai/djl/modality/cv/translator/BaseImageTranslator.java
+++ b/api/src/main/java/ai/djl/modality/cv/translator/BaseImageTranslator.java
@@ -116,6 +116,26 @@ public abstract class BaseImageTranslator<T> implements Translator<Image, T> {
          */
         public T setPipeline(Pipeline pipeline) {
             this.pipeline = pipeline;
+            for (Transform transform : pipeline.getTransforms()) {
+                if (transform instanceof Resize) {
+                    Resize resize = (Resize) transform;
+                    width = resize.getWidth();
+                    height = resize.getHeight();
+                }
+            }
+            return self();
+        }
+
+        /**
+         * Sets the image size.
+         *
+         * @param width the image width
+         * @param height the image height
+         * @return this builder
+         */
+        public T setImageSize(int width, int height) {
+            this.width = width;
+            this.height = height;
             return self();
         }
 
@@ -129,6 +149,12 @@ public abstract class BaseImageTranslator<T> implements Translator<Image, T> {
             if (pipeline == null) {
                 pipeline = new Pipeline();
             }
+            if (transform instanceof Resize) {
+                Resize resize = (Resize) transform;
+                width = resize.getWidth();
+                height = resize.getHeight();
+            }
+
             pipeline.add(transform);
             return self();
         }
@@ -167,13 +193,14 @@ public abstract class BaseImageTranslator<T> implements Translator<Image, T> {
             } else if (!"false".equals(resize)) {
                 String[] tokens = resize.split("\\s*,\\s*");
                 if (tokens.length > 1) {
-                    addTransform(
-                            new Resize(
-                                    (int) Double.parseDouble(tokens[0]),
-                                    (int) Double.parseDouble(tokens[1])));
+                    setImageSize(
+                            (int) Double.parseDouble(tokens[0]),
+                            (int) Double.parseDouble(tokens[1]));
                 } else {
-                    addTransform(new Resize((int) Double.parseDouble(tokens[0])));
+                    int size = (int) Double.parseDouble(tokens[0]);
+                    setImageSize(size, size);
                 }
+                addTransform(new Resize(width, height));
             }
             if (ArgumentsUtil.booleanValue(arguments, "centerCrop", false)) {
                 addTransform(new CenterCrop(width, height));
diff --git a/api/src/main/java/ai/djl/translate/Pipeline.java b/api/src/main/java/ai/djl/translate/Pipeline.java
index d31fcf315..3de467fdd 100644
--- a/api/src/main/java/ai/djl/translate/Pipeline.java
+++ b/api/src/main/java/ai/djl/translate/Pipeline.java
@@ -17,6 +17,7 @@ import ai.djl.ndarray.NDList;
 import ai.djl.util.Pair;
 import ai.djl.util.PairList;
 
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -176,6 +177,15 @@ public class Pipeline {
         return new NDList(arrays);
     }
 
+    /**
+     * Returns the list of transforms.
+     *
+     * @return the list of transforms
+     */
+    public List<Transform> getTransforms() {
+        return transforms.values();
+    }
+
     private static final class IndexKey {
         private String key;
         private int index;
diff --git a/extensions/tokenizers/rust/Cargo.toml b/extensions/tokenizers/rust/Cargo.toml
index ebfa6a93f..f33a58bdc 100644
--- a/extensions/tokenizers/rust/Cargo.toml
+++ b/extensions/tokenizers/rust/Cargo.toml
@@ -13,7 +13,7 @@ candle-flash-attn = { version = "*", optional = true }
 candle-cublaslt = { git = "https://github.com/huggingface/candle-cublaslt", rev = "cf789b7dd6d4abb19b03b9556442f94f0588b4a0", optional = true }
 candle-layer-norm = { git = "https://github.com/xyang16/candle-layer-norm", rev = "e574de6a7f88bafbede8edf9ee43170c6a8ce51a", optional = true }
 candle-rotary = { git = "https://github.com/huggingface/candle-rotary", rev = "0a718a0856569a92f3112e64f10d07e4447822e8", optional = true }
-tokenizers = { version = "0.20.3" }
+tokenizers = { version = "0.21.0" }
 half = "2.4.0"
 tracing = "0.1.40"
 safetensors = "0.4.3"
@@ -34,7 +34,7 @@ openssl-src = { git = "https://github.com/alexcrichton/openssl-src-rs", rev = "c
 openssl = { version = "0.10", features = ["vendored"] }
 
 [target.'cfg(not(target_os = "android"))'.dependencies]
-tokenizers = { version = "0.20.0", features = ["http"] }
+tokenizers = { version = "0.21.0", features = ["http"] }
 
 [lib]
 crate-type = ["cdylib"]
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 9a0221074..ddd35ca4e 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -14,7 +14,7 @@ lightgbm = "3.2.110"
 tensorrt = "8.4.1"
 fasttext = "0.9.2"
 sentencepiece = "0.2.0"
-tokenizers = "0.20.3"
+tokenizers = "0.21.0"
 
 slf4j = "2.0.16"
 log4j_slf4j = "2.24.1"


Test patch:
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
index 44103f3a8..d44e426f7 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -39,7 +39,7 @@ public class HuggingFaceTokenizerTest {
     public void testVersion() {
         try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
             String djlVersion = Engine.getDjlVersion();
-            Assert.assertEquals(tokenizer.getVersion(), "0.20.3-" + djlVersion);
+            Assert.assertEquals(tokenizer.getVersion(), "0.21.0-" + djlVersion);
         }
     }