Repository: deepjavalibrary/djl
Pull request #1871 (2022-08-04 20:03:05)

[android] Adds formatJava task for android project
Change-Id: If15efb1ad2a744b1bfadb7bf89a09a265a81082b

## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/android/build.gradle b/android/build.gradle
index d469c4627..cc13cf3c0 100644
--- a/android/build.gradle
+++ b/android/build.gradle
@@ -23,6 +23,8 @@ allprojects {
             url "https://oss.sonatype.org/content/repositories/snapshots/"
         }
     }
+
+    apply from: file("${rootProject.projectDir}/../tools/gradle/android-formatter.gradle")
 }
 
 task clean(type: Delete) {
diff --git a/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java b/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java
index aeca1d997..87e2f6485 100644
--- a/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java
+++ b/android/core/src/main/java/ai/djl/android/core/BitmapImageFactory.java
@@ -19,14 +19,6 @@ import android.graphics.Color;
 import android.graphics.Paint;
 import android.graphics.Rect;
 
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import java.util.List;
-
 import ai.djl.modality.cv.Image;
 import ai.djl.modality.cv.ImageFactory;
 import ai.djl.modality.cv.output.BoundingBox;
@@ -41,9 +33,15 @@ import ai.djl.ndarray.types.DataType;
 import ai.djl.ndarray.types.Shape;
 import ai.djl.util.RandomUtils;
 
-/**
- * {@code BitmapImageFactory} is the Android implementation of {@link ImageFactory}.
- */
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.util.List;
+
+/** {@code BitmapImageFactory} is the Android implementation of {@link ImageFactory}. */
 public class BitmapImageFactory extends ImageFactory {
 
     /** {@inheritDoc} */
@@ -59,7 +57,7 @@ public class BitmapImageFactory extends ImageFactory {
     /** {@inheritDoc} */
     @Override
     public Image fromUrl(URL url) throws IOException {
-            return fromInputStream(url.openStream());
+        return fromInputStream(url.openStream());
     }
 
     /** {@inheritDoc} */
@@ -122,7 +120,6 @@ public class BitmapImageFactory extends ImageFactory {
         return new BitMapWrapper(bitmap);
     }
 
-
     static class BitMapWrapper implements Image {
         private Bitmap bitmap;
 
@@ -195,7 +192,7 @@ public class BitmapImageFactory extends ImageFactory {
         @Override
         public void save(OutputStream os, String type) throws IOException {
             if (!bitmap.compress(Bitmap.CompressFormat.valueOf(type.toUpperCase()), 100, os)) {
-                throw new IOException("Cannot save image file to output stream File type " +  type);
+                throw new IOException("Cannot save image file to output stream File type " + type);
             }
         }
 
@@ -210,7 +207,7 @@ public class BitmapImageFactory extends ImageFactory {
         @Override
         public void drawBoundingBoxes(DetectedObjects detections) {
             Bitmap mutableBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true);
-            Canvas canvas =  new Canvas(mutableBitmap);
+            Canvas canvas = new Canvas(mutableBitmap);
             // set the paint configure
             int stroke = 2;
             Paint paint = new Paint();
@@ -235,7 +232,8 @@ public class BitmapImageFactory extends ImageFactory {
                         x,
                         y,
                         x + (int) (rectangle.getWidth() * imageWidth),
-                        y + (int) (rectangle.getHeight() * imageHeight), paint);
+                        y + (int) (rectangle.getHeight() * imageHeight),
+                        paint);
                 drawText(canvas, color, className, x, y, stroke, 4);
                 // If we have a mask instead of a plain rectangle, draw tha mask
                 if (box instanceof Mask) {
@@ -252,7 +250,7 @@ public class BitmapImageFactory extends ImageFactory {
         @Override
         public void drawJoints(Joints joints) {
             Bitmap mutableBitmap = bitmap.copy(Bitmap.Config.ARGB_8888, true);
-            Canvas canvas =  new Canvas(mutableBitmap);
+            Canvas canvas = new Canvas(mutableBitmap);
             // set the paint configure
             Paint paint = new Paint();
             paint.setStrokeWidth(2);
@@ -274,7 +272,8 @@ public class BitmapImageFactory extends ImageFactory {
         }
 
         private int randomColor() {
-            return Color.rgb(RandomUtils.nextInt(255), RandomUtils.nextInt(255), RandomUtils.nextInt(255));
+            return Color.rgb(
+                    RandomUtils.nextInt(255), RandomUtils.nextInt(255), RandomUtils.nextInt(255));
         }
 
         private int darker(int color) {
@@ -282,13 +281,11 @@ public class BitmapImageFactory extends ImageFactory {
             int r = Math.round(Color.red(color) * 0.8f);
             int g = Math.round(Color.green(color) * 0.8f);
             int b = Math.round(Color.blue(color) * 0.8f);
-            return Color.argb(a,
-                    Math.min(r, 255),
-                    Math.min(g, 255),
-                    Math.min(b, 255));
+            return Color.argb(a, Math.min(r, 255), Math.min(g, 255), Math.min(b, 255));
         }
 
-        private void drawText(Canvas canvas, int color, String text, int x, int y, int stroke, int padding) {
+        private void drawText(
+                Canvas canvas, int color, String text, int x, int y, int stroke, int padding) {
             Paint paint = new Paint();
             Paint.FontMetrics metrics = paint.getFontMetrics();
             paint.setStyle(Paint.Style.FILL);
@@ -326,7 +323,9 @@ public class BitmapImageFactory extends ImageFactory {
                 y = 0;
             }
 
-            Bitmap maskedImage = Bitmap.createBitmap(probDist.length, probDist[0].length, Bitmap.Config.ARGB_8888);
+            Bitmap maskedImage =
+                    Bitmap.createBitmap(
+                            probDist.length, probDist[0].length, Bitmap.Config.ARGB_8888);
             for (int xCor = 0; xCor < probDist.length; xCor++) {
                 for (int yCor = 0; yCor < probDist[xCor].length; yCor++) {
                     float opacity = probDist[xCor][yCor];
@@ -339,8 +338,8 @@ public class BitmapImageFactory extends ImageFactory {
                     maskedImage.setPixel(xCor, yCor, darker(Color.argb(opacity, r, g, b)));
                 }
             }
-            Canvas canvas =  new Canvas(image);
-            canvas.drawBitmap(maskedImage, x, y,null);
+            Canvas canvas = new Canvas(image);
+            canvas.drawBitmap(maskedImage, x, y, null);
         }
     }
 }
diff --git a/extensions/tokenizers/rust/src/lib.rs b/extensions/tokenizers/rust/src/lib.rs
index 3b0d52a7e..17bfa79b8 100644
--- a/extensions/tokenizers/rust/src/lib.rs
+++ b/extensions/tokenizers/rust/src/lib.rs
@@ -18,6 +18,8 @@ use std::str::FromStr;
 use tk::tokenizer::{EncodeInput, Encoding};
 use tk::Tokenizer;
 use tk::{FromPretrainedParameters, Offsets};
+use tk::utils::truncation::{TruncationParams, TruncationStrategy};
+use tk::utils::padding::{PaddingParams, PaddingStrategy};
 
 use jni::objects::{JClass, JMethodID, JObject, JString, JValue, ReleaseMode};
 use jni::sys::{
@@ -405,6 +407,81 @@ pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_
     ret
 }
 
+#[no_mangle]
+pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_setPadding(
+    env: JNIEnv,
+    _: JObject,
+    handle: jlong,
+    max_length: jlong,
+    padding_strategy: JString,
+    pad_to_multiple_of: jlong,
+) {
+    let strategy: String = env
+        .get_string(padding_strategy)
+        .expect("Couldn't get java string!")
+        .into();
+    let len = max_length as usize;
+    let res_strategy = match strategy.as_ref() {
+        "batch_longest" => Ok(PaddingStrategy::BatchLongest),
+        "fixed" => Ok(PaddingStrategy::Fixed(len)),
+        _ => Err("strategy must be one of [batch_longest, fixed]"),
+    };
+
+    let mut params = PaddingParams::default();
+    params.strategy = res_strategy.unwrap();
+    params.pad_to_multiple_of = Some(pad_to_multiple_of as usize);
+    let tokenizer = cast_handle::<Tokenizer>(handle);
+    tokenizer.with_padding(Some(params));
+}
+
+#[no_mangle]
+pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_disablePadding(
+    _env: JNIEnv,
+    _: JObject,
+    handle: jlong,
+) {
+    let tokenizer = cast_handle::<Tokenizer>(handle);
+    tokenizer.with_padding(None);
+}
+
+#[no_mangle]
+pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_setTruncation(
+    env: JNIEnv,
+    _: JObject,
+    handle: jlong,
+    truncation_max_length: jlong,
+    truncation_strategy: JString,
+    truncation_stride: jlong,
+) {
+    let strategy: String = env
+        .get_string(truncation_strategy)
+        .expect("Couldn't get java string!")
+        .into();
+    let res_strategy = match strategy.as_ref() {
+       "longest_first" => Ok(TruncationStrategy::LongestFirst),
+       "only_first" => Ok(TruncationStrategy::OnlyFirst),
+       "only_second" => Ok(TruncationStrategy::OnlySecond),
+       _ => Err("strategy must be one of [longest_first, only_first, only_second]"),
+    };
+    let mut params = TruncationParams::default();
+    params.max_length = truncation_max_length as usize;
+    params.strategy = res_strategy.unwrap();
+    params.stride = truncation_stride as usize;
+
+    let tokenizer = cast_handle::<Tokenizer>(handle);
+    tokenizer.with_truncation(Some(params));
+}
+
+#[no_mangle]
+pub extern "system" fn Java_ai_djl_huggingface_tokenizers_jni_TokenizersLibrary_disableTruncation(
+    _env: JNIEnv,
+    _: JObject,
+    handle: jlong,
+) {
+    let tokenizer = cast_handle::<Tokenizer>(handle);
+    tokenizer.with_truncation(None);
+}
+
 fn to_handle<T: 'static>(val: T) -> jlong {
     let handle = Box::into_raw(Box::new(val)) as jlong;
     handle
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
index 123ba7c84..f80779960 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizer.java
@@ -302,6 +302,90 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
         return decode(ids, !addSpecialTokens);
     }
 
+    /**
+     * Sets the truncation and padding behavior for the tokenizer.
+     *
+     * @param truncationStrategy the {@code TruncationStrategy} to use
+     * @param paddingStrategy the {@code PaddingStrategy} to use
+     * @param maxLength the maximum length to pad/truncate sequences to
+     */
+    public void setTruncationAndPadding(
+            TruncationStrategy truncationStrategy,
+            PaddingStrategy paddingStrategy,
+            long maxLength) {
+        setTruncationAndPadding(truncationStrategy, paddingStrategy, maxLength, 0, 0);
+    }
+
+    /**
+     * Sets the truncation and padding behavior for the tokenizer.
+     *
+     * @param truncationStrategy the {@code TruncationStrategy} to use
+     * @param paddingStrategy the {@code PaddingStrategy} to use
+     * @param maxLength the maximum length to pad/truncate sequences to
+     * @param stride value to use when handling overflow
+     */
+    public void setTruncationAndPadding(
+            TruncationStrategy truncationStrategy,
+            PaddingStrategy paddingStrategy,
+            long maxLength,
+            long stride) {
+        setTruncationAndPadding(truncationStrategy, paddingStrategy, maxLength, stride, 0);
+    }
+
+    /**
+     * Sets the truncation and padding behavior for the tokenizer.
+     *
+     * @param truncationStrategy the {@code TruncationStrategy} to use
+     * @param paddingStrategy the {@code PaddingStrategy} to use
+     * @param maxLength the maximum length to pad/truncate sequences to
+     * @param stride value to use when handling overflow
+     * @param padToMultipleOf pad sequence length to multiple of value
+     */
+    public void setTruncationAndPadding(
+            TruncationStrategy truncationStrategy,
+            PaddingStrategy paddingStrategy,
+            long maxLength,
+            long stride,
+            long padToMultipleOf) {
+        setTruncation(truncationStrategy, maxLength, stride);
+        setPadding(paddingStrategy, maxLength, padToMultipleOf);
+    }
+
+    /**
+     * Sets the truncation behavior for the tokenizer.
+     *
+     * @param truncationStrategy the {@code TruncationStrategy} to use
+     * @param maxLength the maximum length to truncate sequences to
+     * @param stride value to use when handling overflow
+     */
+    public void setTruncation(TruncationStrategy truncationStrategy, long maxLength, long stride) {
+        if (truncationStrategy == TruncationStrategy.DO_NOT_TRUNCATE) {
+            TokenizersLibrary.LIB.disableTruncation(getHandle());
+        } else {
+            TokenizersLibrary.LIB.setTruncation(
+                    getHandle(), maxLength, truncationStrategy.toString().toLowerCase(), stride);
+        }
+    }
+
+    /**
+     * Sets the padding behavior for the tokenizer.
+     *
+     * @param paddingStrategy the {@code PaddingStrategy} to use
+     * @param maxLength the maximum length to pad sequences to
+     * @param padToMultipleOf pad sequence length to multiple of value
+     */
+    public void setPadding(PaddingStrategy paddingStrategy, long maxLength, long padToMultipleOf) {
+        if (paddingStrategy == PaddingStrategy.DO_NOT_PAD) {
+            TokenizersLibrary.LIB.disablePadding(getHandle());
+        } else {
+            TokenizersLibrary.LIB.setPadding(
+                    getHandle(),
+                    maxLength,
+                    paddingStrategy.toString().toLowerCase(),
+                    padToMultipleOf);
+        }
+    }
+
     private Encoding toEncoding(long encoding) {
         long[] ids = TokenizersLibrary.LIB.getTokenIds(encoding);
         long[] typeIds = TokenizersLibrary.LIB.getTypeIds(encoding);
@@ -315,4 +399,17 @@ public final class HuggingFaceTokenizer extends NativeResource<Long> implements
         return new Encoding(
                 ids, typeIds, tokens, wordIds, attentionMask, specialTokenMask, charSpans);
     }
+
+    enum TruncationStrategy {
+        DO_NOT_TRUNCATE,
+        LONGEST_FIRST,
+        ONLY_FIRST,
+        ONLY_SECOND
+    }
+
+    enum PaddingStrategy {
+        DO_NOT_PAD,
+        BATCH_LONGEST,
+        FIXED
+    }
 }
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/jni/TokenizersLibrary.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/jni/TokenizersLibrary.java
index b532886bf..b0e9095ac 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/jni/TokenizersLibrary.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/tokenizers/jni/TokenizersLibrary.java
@@ -52,4 +52,14 @@ public final class TokenizersLibrary {
     public native CharSpan[] getTokenCharSpans(long encoding);
 
     public native String decode(long tokenizer, long[] ids, boolean addSpecialTokens);
+
+    public native void disablePadding(long tokenizer);
+
+    public native void setPadding(
+            long tokenizer, long maxLength, String paddingStrategy, long padToMultipleOf);
+
+    public native void disableTruncation(long tokenizer);
+
+    public native void setTruncation(
+            long tokenizer, long maxLength, String truncationStrategy, long stride);
 }
diff --git a/tools/gradle/android-formatter.gradle b/tools/gradle/android-formatter.gradle
new file mode 100644
index 000000000..7ffe40638
--- /dev/null
+++ b/tools/gradle/android-formatter.gradle
@@ -0,0 +1,40 @@
+buildscript {
+    repositories {
+        mavenCentral()
+        maven {
+            url "https://plugins.gradle.org/m2/"
+        }
+    }
+    dependencies {
+        classpath 'com.google.googlejavaformat:google-java-format:1.15.0'
+    }
+}
+
+apply plugin: JavaFormatterPlugin
+
+import com.google.googlejavaformat.java.Formatter
+import com.google.googlejavaformat.java.ImportOrderer
+import com.google.googlejavaformat.java.JavaFormatterOptions
+import com.google.googlejavaformat.java.Main
+import com.google.googlejavaformat.java.RemoveUnusedImports
+
+class JavaFormatterPlugin implements Plugin<Project> {
+    void apply(Project project) {
+        project.task('formatJava') {
+            doLast {
+                if (project.getRootProject() == project) {
+                    return
+                }
+                Main formatter = new Main(new PrintWriter(System.out, true), new PrintWriter(System.err, true), System.in)
+                FileTree dirs = project.fileTree("${project.getProjectDir()}/src").matching {
+                    include "**/*.java"
+                }
+                for (file in dirs) {
+                    if (formatter.format("-a", "-i", file.getAbsolutePath()) != 0) {
+                        throw new GradleException("Format java failed: " + file.getAbsolutePath())
+                    }
+                }
+            }
+        }
+    }
+}


Test patch:
diff --git a/android/core/src/androidTest/java/ai/djl/android/core/BitmapWrapperTest.java b/android/core/src/androidTest/java/ai/djl/android/core/BitmapWrapperTest.java
index d54f8dcd9..7b8f005a8 100644
--- a/android/core/src/androidTest/java/ai/djl/android/core/BitmapWrapperTest.java
+++ b/android/core/src/androidTest/java/ai/djl/android/core/BitmapWrapperTest.java
@@ -12,14 +12,13 @@
  */
 package ai.djl.android.core;
 
-import android.content.Context;
+import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation;
 
-import androidx.test.ext.junit.runners.AndroidJUnit4;
+import static org.junit.Assert.assertEquals;
 
-import org.junit.Test;
-import org.junit.runner.RunWith;
+import android.content.Context;
 
-import java.io.IOException;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
 
 import ai.djl.modality.cv.Image;
 import ai.djl.modality.cv.ImageFactory;
@@ -28,8 +27,10 @@ import ai.djl.ndarray.NDManager;
 import ai.djl.ndarray.types.DataType;
 import ai.djl.ndarray.types.Shape;
 
-import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation;
-import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.IOException;
 
 /**
  * Instrumented test, which will execute on an Android device.
@@ -51,7 +52,9 @@ public class BitmapWrapperTest {
     public void testImageToNDArray() throws IOException {
         try (NDManager manager = NDManager.newBaseManager()) {
             ImageFactory factory = ImageFactory.getInstance();
-            Image img = factory.fromUrl("https://github.com/deepjavalibrary/djl/raw/master/examples/src/test/resources/dog_bike_car.jpg");
+            Image img =
+                    factory.fromUrl(
+                            "https://github.com/deepjavalibrary/djl/raw/master/examples/src/test/resources/dog_bike_car.jpg");
             NDArray array = img.toNDArray(manager);
             assertEquals(new Shape(img.getHeight(), img.getWidth(), 3), array.getShape());
         }
@@ -61,7 +64,9 @@ public class BitmapWrapperTest {
     public void testImageGetSubImage() throws IOException {
         try (NDManager manager = NDManager.newBaseManager()) {
             ImageFactory factory = ImageFactory.getInstance();
-            Image img = factory.fromUrl("https://github.com/deepjavalibrary/djl/raw/master/examples/src/test/resources/dog_bike_car.jpg");
+            Image img =
+                    factory.fromUrl(
+                            "https://github.com/deepjavalibrary/djl/raw/master/examples/src/test/resources/dog_bike_car.jpg");
             NDArray array = img.toNDArray(manager);
             Image subImg = img.getSubImage(10, 20, 30, 40);
             NDArray subArray = subImg.toNDArray(manager);
@@ -73,7 +78,9 @@ public class BitmapWrapperTest {
     public void testImageDuplicate() throws IOException {
         try (NDManager manager = NDManager.newBaseManager()) {
             ImageFactory factory = ImageFactory.getInstance();
-            Image img = factory.fromUrl("https://github.com/deepjavalibrary/djl/raw/master/examples/src/test/resources/dog_bike_car.jpg");
+            Image img =
+                    factory.fromUrl(
+                            "https://github.com/deepjavalibrary/djl/raw/master/examples/src/test/resources/dog_bike_car.jpg");
             NDArray array = img.toNDArray(manager);
             NDArray duplicate = img.duplicate().toNDArray(manager);
             assertEquals(array, duplicate);
@@ -86,9 +93,8 @@ public class BitmapWrapperTest {
             NDArray array = manager.arange(0.0f, 12.0f).reshape(3, 2, 2);
             ImageFactory factory = ImageFactory.getInstance();
             Image image = factory.fromNDArray(array);
-            NDArray converted = image.toNDArray(manager)
-                    .transpose(2, 0, 1)
-                    .toType(DataType.FLOAT32, true);
+            NDArray converted =
+                    image.toNDArray(manager).transpose(2, 0, 1).toType(DataType.FLOAT32, true);
             assertEquals(array, converted);
 
             array = manager.arange(0, 12, 1, DataType.UINT8).reshape(3, 2, 2);
diff --git a/android/core/src/androidTest/java/ai/djl/android/core/ModelLoadingTest.java b/android/core/src/androidTest/java/ai/djl/android/core/ModelLoadingTest.java
index 73277e972..a15b13a53 100644
--- a/android/core/src/androidTest/java/ai/djl/android/core/ModelLoadingTest.java
+++ b/android/core/src/androidTest/java/ai/djl/android/core/ModelLoadingTest.java
@@ -12,11 +12,6 @@
  */
 package ai.djl.android.core;
 
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-
 import ai.djl.ModelException;
 import ai.djl.inference.Predictor;
 import ai.djl.modality.Classifications;
@@ -30,6 +25,11 @@ import ai.djl.repository.zoo.ModelZoo;
 import ai.djl.repository.zoo.ZooModel;
 import ai.djl.translate.TranslateException;
 
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.IOException;
+
 /* Integration test to check if model and inference runs fine */
 public class ModelLoadingTest {
 
@@ -38,21 +38,25 @@ public class ModelLoadingTest {
 
         String modelUrl = "https://resources.djl.ai/demo/pytorch/traced_resnet18.zip";
         ImageClassificationTranslator.builder()
-            .addTransform(new Resize(224, 224))
-            .addTransform(new ToTensor())
-            .optApplySoftmax(true)
-            .build();
-        Criteria<Image, Classifications> criteria = Criteria.builder()
-            .setTypes(Image.class, Classifications.class)
-            .optModelUrls(modelUrl)
-            .optTranslator(ImageClassificationTranslator.builder()
                 .addTransform(new Resize(224, 224))
                 .addTransform(new ToTensor())
-                .optApplySoftmax(true).build())
-            .build();
-        Image image = ImageFactory.getInstance().fromUrl("https://resources.djl.ai/images/kitten.jpg");
+                .optApplySoftmax(true)
+                .build();
+        Criteria<Image, Classifications> criteria =
+                Criteria.builder()
+                        .setTypes(Image.class, Classifications.class)
+                        .optModelUrls(modelUrl)
+                        .optTranslator(
+                                ImageClassificationTranslator.builder()
+                                        .addTransform(new Resize(224, 224))
+                                        .addTransform(new ToTensor())
+                                        .optApplySoftmax(true)
+                                        .build())
+                        .build();
+        Image image =
+                ImageFactory.getInstance().fromUrl("https://resources.djl.ai/images/kitten.jpg");
         try (ZooModel<Image, Classifications> model = ModelZoo.loadModel(criteria);
-             Predictor<Image, Classifications> predictor = model.newPredictor()) {
+                Predictor<Image, Classifications> predictor = model.newPredictor()) {
             Classifications result = predictor.predict(image);
             Assert.assertEquals("n02124075 Egyptian cat", result.best().getClassName());
         }
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
index 591df6b22..22cf0782d 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/HuggingFaceTokenizerTest.java
@@ -155,4 +155,77 @@ public class HuggingFaceTokenizerTest {
             }
         }
     }
+
+    @Test
+    public void testTruncationAndPadding() {
+        TestRequirements.notArm();
+
+        try (HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("bert-base-cased")) {
+            List<String> inputs =
+                    Arrays.asList(
+                            "Hello, y'all! How are you?",
+                            "Today is a sunny day. Good weather I'd say",
+                            "I am happy");
+            tokenizer.setTruncationAndPadding(
+                    HuggingFaceTokenizer.TruncationStrategy.LONGEST_FIRST,
+                    HuggingFaceTokenizer.PaddingStrategy.BATCH_LONGEST,
+                    10);
+            List<long[]> expectedIds =
+                    Arrays.asList(
+                            new long[] {101, 8667, 117, 194, 112, 1155, 106, 1731, 1132, 102},
+                            new long[] {101, 3570, 1110, 170, 21162, 1285, 119, 2750, 4250, 102},
+                            new long[] {101, 146, 1821, 2816, 102, 0, 0, 0, 0, 0});
+            List<long[]> expectedAttentionMasks =
+                    Arrays.asList(
+                            new long[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+                            new long[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+                            new long[] {1, 1, 1, 1, 1, 0, 0, 0, 0, 0});
+            Encoding[] encodings = tokenizer.batchEncode(inputs);
+            for (int i = 0; i < encodings.length; ++i) {
+                Assert.assertEquals(encodings[i].getIds(), expectedIds.get(i));
+                Assert.assertEquals(encodings[i].getAttentionMask(), expectedAttentionMasks.get(i));
+            }
+
+            tokenizer.setTruncationAndPadding(
+                    HuggingFaceTokenizer.TruncationStrategy.LONGEST_FIRST,
+                    HuggingFaceTokenizer.PaddingStrategy.FIXED,
+                    12);
+            expectedIds =
+                    Arrays.asList(
+                            new long[] {
+                                101, 8667, 117, 194, 112, 1155, 106, 1731, 1132, 1128, 136, 102
+                            },
+                            new long[] {
+                                101, 3570, 1110, 170, 21162, 1285, 119, 2750, 4250, 146, 112, 102
+                            },
+                            new long[] {101, 146, 1821, 2816, 102, 0, 0, 0, 0, 0, 0, 0});
+            expectedAttentionMasks =
+                    Arrays.asList(
+                            new long[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+                            new long[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+                            new long[] {1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0});
+            encodings = tokenizer.batchEncode(inputs);
+            for (int i = 0; i < encodings.length; ++i) {
+                Assert.assertEquals(encodings[i].getIds(), expectedIds.get(i));
+                Assert.assertEquals(encodings[i].getAttentionMask(), expectedAttentionMasks.get(i));
+            }
+
+            tokenizer.setTruncationAndPadding(
+                    HuggingFaceTokenizer.TruncationStrategy.ONLY_FIRST,
+                    HuggingFaceTokenizer.PaddingStrategy.BATCH_LONGEST,
+                    8);
+            Encoding encoding = tokenizer.encode("Hello there my friend", "How are you");
+            long[] expectedId = new long[] {101, 8667, 1175, 102, 1731, 1132, 1128, 102};
+            Assert.assertEquals(encoding.getIds(), expectedId);
+
+            tokenizer.setTruncationAndPadding(
+                    HuggingFaceTokenizer.TruncationStrategy.ONLY_SECOND,
+                    HuggingFaceTokenizer.PaddingStrategy.DO_NOT_PAD,
+                    8);
+
+            encoding = tokenizer.encode("Hello there my friend", "How are you");
+            expectedId = new long[] {101, 8667, 1175, 1139, 1910, 102, 1731, 102};
+            Assert.assertEquals(encoding.getIds(), expectedId);
+        }
+    }
 }