Repository: deepjavalibrary/djl
Pull request #3626 (2025-03-06 20:29:55)

[api] Add zero shot object detection support
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/api/src/main/java/ai/djl/Application.java b/api/src/main/java/ai/djl/Application.java
index 0cc7ef409..4c7d9e91a 100644
--- a/api/src/main/java/ai/djl/Application.java
+++ b/api/src/main/java/ai/djl/Application.java
@@ -53,9 +53,15 @@ public class Application {
             case "cv/image_classification":
             case "image_classification":
                 return CV.IMAGE_CLASSIFICATION;
+            case "cv/zero_shot_image_classification":
+            case "zero_shot_image_classification":
+                return CV.ZERO_SHOT_IMAGE_CLASSIFICATION;
             case "cv/object_detection":
             case "object_detection":
                 return CV.OBJECT_DETECTION;
+            case "cv/zero_shot_object_detection":
+            case "zero_shot_object_detection":
+                return CV.ZERO_SHOT_OBJECT_DETECTION;
             case "cv/semantic_segmentation":
             case "semantic_segmentation":
                 return CV.SEMANTIC_SEGMENTATION;
@@ -181,6 +187,15 @@ public class Application {
          */
         Application IMAGE_CLASSIFICATION = new Application("cv/image_classification");
 
+        /**
+         * An application where images are assigned a single class name.
+         *
+         * <p>The typical signature is Model&lt;{@link ai.djl.modality.cv.VisionLanguageInput},
+         * {@link ai.djl.modality.Classifications}&gt;.
+         */
+        Application ZERO_SHOT_IMAGE_CLASSIFICATION =
+                new Application("cv/zero_shot_image_classification");
+
         /**
          * An application that finds zero or more objects in an image, the object class (see image
          * classification), and their locations as a {@link ai.djl.modality.cv.output.BoundingBox}.
@@ -193,6 +208,18 @@ public class Application {
          */
         Application OBJECT_DETECTION = new Application("cv/object_detection");
 
+        /**
+         * An application that finds zero or more objects in an image, the object class (see image
+         * classification), and their locations as a {@link ai.djl.modality.cv.output.BoundingBox}.
+         *
+         * <p>The typical signature is Model&lt;{@link ai.djl.modality.cv.VisionLanguageInput},
+         * {@link ai.djl.modality.cv.output.DetectedObjects}&gt;.
+         *
+         * @see <a href="https://d2l.djl.ai/chapter_computer-vision/bounding-box.html">The D2L
+         *     chapter on object detection</a>
+         */
+        Application ZERO_SHOT_OBJECT_DETECTION = new Application("cv/zero_shot_object_detection");
+
         /** An application that classifies each pixel in an image into a category. */
         Application SEMANTIC_SEGMENTATION = new Application("cv/semantic_segmentation");
 
diff --git a/api/src/main/java/ai/djl/modality/cv/VisionLanguageInput.java b/api/src/main/java/ai/djl/modality/cv/VisionLanguageInput.java
new file mode 100644
index 000000000..8ed20a063
--- /dev/null
+++ b/api/src/main/java/ai/djl/modality/cv/VisionLanguageInput.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.modality.cv;
+
+import ai.djl.modality.Input;
+import ai.djl.translate.TranslateException;
+import ai.djl.util.JsonUtils;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.annotations.SerializedName;
+
+import java.io.IOException;
+
+/** The input container for a vision language model. */
+public class VisionLanguageInput {
+
+    private Image image;
+    private String text;
+
+    @SerializedName("candidate_labels")
+    private String[] candidates;
+
+    @SerializedName("hypothesis_template")
+    private String hypothesisTemplate;
+
+    /**
+     * Constructs a new {@code ImageTextInput} instance.
+     *
+     * @param image the image input
+     * @param text the prompt
+     */
+    public VisionLanguageInput(Image image, String text) {
+        this(image, text, null, null);
+    }
+
+    /**
+     * Constructs a new {@code ImageTextInput} instance.
+     *
+     * @param image the image input
+     * @param candidates the candidate labels
+     */
+    public VisionLanguageInput(Image image, String[] candidates) {
+        this(image, null, candidates, null);
+    }
+
+    /**
+     * Constructs a new {@code ImageTextInput} instance.
+     *
+     * @param image the image input
+     * @param text the prompt
+     * @param candidates the candidate labels
+     * @param hypothesisTemplate the hypothesis template
+     */
+    public VisionLanguageInput(
+            Image image, String text, String[] candidates, String hypothesisTemplate) {
+        this.image = image;
+        this.text = text;
+        this.candidates = candidates;
+        this.hypothesisTemplate = hypothesisTemplate;
+    }
+
+    /**
+     * Returns the {@code ImageTextInput} from the {@link Input}.
+     *
+     * @param input the input object
+     * @return the {@code ImageTextInput} from the {@link Input}
+     * @throws TranslateException if the input is invalid
+     * @throws IOException if failed to load image
+     */
+    public static VisionLanguageInput parseInput(Input input)
+            throws TranslateException, IOException {
+        String data = input.getData().getAsString();
+        try {
+            JsonObject obj = JsonUtils.GSON.fromJson(data, JsonObject.class);
+            JsonElement url = obj.get("image");
+            if (url == null) {
+                url = obj.get("image_url");
+            }
+            if (url == null) {
+                throw new TranslateException("Missing \"image\" parameter in input.");
+            }
+            Image img = ImageFactory.getInstance().fromUrl(url.getAsString());
+            String text = JsonUtils.GSON.fromJson(obj.get("text"), String.class);
+            String[] candidates =
+                    JsonUtils.GSON.fromJson(obj.get("candidate_labels"), String[].class);
+            String hypothesisTemplate =
+                    JsonUtils.GSON.fromJson(obj.get("hypothesis_template"), String.class);
+            return new VisionLanguageInput(img, text, candidates, hypothesisTemplate);
+        } catch (JsonParseException e) {
+            throw new TranslateException("Input is not a valid json.", e);
+        }
+    }
+
+    /**
+     * Returns the image input.
+     *
+     * @return the image input
+     */
+    public Image getImage() {
+        return image;
+    }
+
+    /**
+     * Sets the image input.
+     *
+     * @param image the image input
+     */
+    public void setImage(Image image) {
+        this.image = image;
+    }
+
+    /**
+     * Returns the prompt text.
+     *
+     * @return the prompt text
+     */
+    public String getText() {
+        return text;
+    }
+
+    /**
+     * Sets the prompt text.
+     *
+     * @param text the prompt text
+     */
+    public void setText(String text) {
+        this.text = text;
+    }
+
+    /**
+     * Returns the candidate labels.
+     *
+     * @return the candidate labels
+     */
+    public String[] getCandidates() {
+        return candidates;
+    }
+
+    /**
+     * Sets the candidate labels.
+     *
+     * @param candidates the candidate labels
+     */
+    public void setCandidates(String[] candidates) {
+        this.candidates = candidates;
+    }
+
+    /**
+     * Returns the hypothesis template.
+     *
+     * @return the hypothesis template
+     */
+    public String getHypothesisTemplate() {
+        return hypothesisTemplate == null ? "This is a photo of {}." : hypothesisTemplate;
+    }
+
+    /**
+     * Sets the hypothesis template.
+     *
+     * @param hypothesisTemplate the hypothesis template
+     */
+    public void setHypothesisTemplate(String hypothesisTemplate) {
+        this.hypothesisTemplate = hypothesisTemplate;
+    }
+}
diff --git a/api/src/main/java/ai/djl/modality/cv/translator/ZeroShotObjectDetectionServingTranslator.java b/api/src/main/java/ai/djl/modality/cv/translator/ZeroShotObjectDetectionServingTranslator.java
new file mode 100644
index 000000000..f507e28a9
--- /dev/null
+++ b/api/src/main/java/ai/djl/modality/cv/translator/ZeroShotObjectDetectionServingTranslator.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.modality.cv.translator;
+
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.modality.cv.VisionLanguageInput;
+import ai.djl.modality.cv.output.DetectedObjects;
+import ai.djl.ndarray.BytesSupplier;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.Batchifier;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslateException;
+import ai.djl.translate.Translator;
+import ai.djl.translate.TranslatorContext;
+
+/** A {@link Translator} that can serve ZeroShotObjectDetection model. */
+public class ZeroShotObjectDetectionServingTranslator
+        implements NoBatchifyTranslator<Input, Output> {
+
+    private Translator<VisionLanguageInput, DetectedObjects> translator;
+
+    /**
+     * Constructs a {@code ZeroShotObjectDetectionServingTranslator} instance.
+     *
+     * @param translator a {@code Translator} processes token classification input
+     */
+    public ZeroShotObjectDetectionServingTranslator(
+            Translator<VisionLanguageInput, DetectedObjects> translator) {
+        this.translator = translator;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void prepare(TranslatorContext ctx) throws Exception {
+        translator.prepare(ctx);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, Input input) throws Exception {
+        if (input.getContent().isEmpty()) {
+            throw new TranslateException("Input data is empty.");
+        }
+
+        VisionLanguageInput prompt = VisionLanguageInput.parseInput(input);
+        NDList ret = translator.processInput(ctx, prompt);
+        Batchifier batchifier = translator.getBatchifier();
+        if (batchifier != null) {
+            NDList[] batch = {ret};
+            return batchifier.batchify(batch);
+        }
+        return ret;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Output processOutput(TranslatorContext ctx, NDList list) throws Exception {
+        Output output = new Output();
+        output.addProperty("Content-Type", "application/json");
+        Batchifier batchifier = translator.getBatchifier();
+        if (batchifier != null) {
+            list = batchifier.unbatchify(list)[0];
+        }
+        output.add(BytesSupplier.wrapAsJson(translator.processOutput(ctx, list)));
+        return output;
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/cv/ZeroShotObjectDetection.java b/examples/src/main/java/ai/djl/examples/inference/cv/ZeroShotObjectDetection.java
new file mode 100644
index 000000000..3f0e6c8ce
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/cv/ZeroShotObjectDetection.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.cv;
+
+import ai.djl.Device;
+import ai.djl.ModelException;
+import ai.djl.huggingface.translator.ZeroShotObjectDetectionTranslatorFactory;
+import ai.djl.inference.Predictor;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.modality.cv.VisionLanguageInput;
+import ai.djl.modality.cv.output.DetectedObjects;
+import ai.djl.repository.zoo.Criteria;
+import ai.djl.repository.zoo.ZooModel;
+import ai.djl.training.util.ProgressBar;
+import ai.djl.translate.TranslateException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public final class ZeroShotObjectDetection {
+
+    private static final Logger logger = LoggerFactory.getLogger(ZeroShotObjectDetection.class);
+
+    private ZeroShotObjectDetection() {}
+
+    public static void main(String[] args) throws IOException, ModelException, TranslateException {
+        DetectedObjects classifications = predict();
+        logger.info("{}", classifications);
+    }
+
+    public static DetectedObjects predict() throws IOException, ModelException, TranslateException {
+        String url = "http://images.cocodataset.org/val2017/000000039769.jpg";
+        Image img = ImageFactory.getInstance().fromUrl(url);
+        VisionLanguageInput input = new VisionLanguageInput(img, new String[] {"a cat"});
+
+        // You can use src/main/python/trace_owlv2.py to trace the model
+        Criteria<VisionLanguageInput, DetectedObjects> criteria =
+                Criteria.builder()
+                        .setTypes(VisionLanguageInput.class, DetectedObjects.class)
+                        .optModelUrls("djl://ai.djl.huggingface.pytorch/google/owlv2-base-patch16")
+                        .optEngine("PyTorch")
+                        .optDevice(Device.cpu()) // Only support CPU
+                        .optTranslatorFactory(new ZeroShotObjectDetectionTranslatorFactory())
+                        .optProgress(new ProgressBar())
+                        .build();
+
+        try (ZooModel<VisionLanguageInput, DetectedObjects> model = criteria.loadModel();
+                Predictor<VisionLanguageInput, DetectedObjects> predictor = model.newPredictor()) {
+            DetectedObjects ret = predictor.predict(input);
+            img.drawBoundingBoxes(ret);
+
+            Path outputDir = Paths.get("build/output");
+            Files.createDirectories(outputDir);
+            Path imagePath = outputDir.resolve("zero_shot_object_detection.png");
+            // OpenJDK can't save jpg with alpha channel
+            img.save(Files.newOutputStream(imagePath), "png");
+            return ret;
+        }
+    }
+}
diff --git a/examples/src/main/python/trace_owlv2.py b/examples/src/main/python/trace_owlv2.py
new file mode 100644
index 000000000..bde5de33e
--- /dev/null
+++ b/examples/src/main/python/trace_owlv2.py
@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+import os
+from typing import Any
+from collections import OrderedDict
+
+import requests
+import torch
+from PIL import Image
+from torch import nn
+from transformers import pipeline
+from transformers.modeling_outputs import BaseModelOutput
+
+
+class ModelWrapper(nn.Module):
+
+    def __init__(self, model) -> None:
+        super().__init__()
+        self.model = model
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor,
+        pixel_values: torch.Tensor,
+    ) -> dict[Any, torch.Tensor]:
+        output = self.model(input_ids=input_ids,
+                            attention_mask=attention_mask,
+                            pixel_values=pixel_values)
+        # filter non-Tensor
+        ret = OrderedDict()
+        for k, v in output.items():
+            if isinstance(v, torch.Tensor):
+                ret[k] = v
+        return ret
+
+
+def jit_trace():
+    model_id = "google/owlv2-base-patch16"
+    pipe = pipeline(model=model_id, framework="pt", device="cpu")
+
+    image_url = "http://images.cocodataset.org/val2017/000000039769.jpg"
+    image = Image.open(requests.get(image_url, stream=True).raw)
+    texts = ["a cat"]
+
+    encoding = pipe.tokenizer(texts, padding=True, return_tensors='pt')
+    input_ids = encoding["input_ids"]
+    attention_mask = encoding["attention_mask"]
+
+    image_features = pipe.image_processor(images=image, return_tensors='pt')
+    pixel_values = image_features["pixel_values"]
+
+    traced_model = torch.jit.trace(ModelWrapper(pipe.model),
+                                   (input_ids, attention_mask, pixel_values),
+                                   strict=False)
+
+    model_name = model_id.split("/")[1]
+    os.makedirs(model_name, exist_ok=True)
+
+    pipe.tokenizer.save_pretrained(model_name)
+    for path in os.listdir(model_name):
+        if path != "tokenizer.json" and path != "tokenizer_config.json":
+            os.remove(os.path.join(model_name, path))
+
+    torch.jit.save(traced_model, f"{model_name}/{model_name}.pt")
+
+    serving_file = os.path.join(model_name, "serving.properties")
+    arguments = {
+        "engine":
+        "PyTorch",
+        "option.modelName":
+        model_name,
+        "option.mapLocation":
+        "true",
+        "width":
+        "960",
+        "height":
+        "960",
+        "pad":
+        "128",
+        "resize":
+        "true",
+        "toTensor":
+        "true",
+        "normalize":
+        "0.48145466,0.4578275,0.40821073,0.26862954,0.26130258,0.27577711",
+        "translatorFactory":
+        "ai.djl.huggingface.translator.ZeroShotObjectDetectionTranslatorFactory",
+    }
+
+    with open(serving_file, 'w') as f:
+        for k, v in arguments.items():
+            f.write(f"{k}={v}\n")
+
+
+if __name__ == '__main__':
+    jit_trace()
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/ZeroShotObjectDetectionTranslator.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/ZeroShotObjectDetectionTranslator.java
new file mode 100644
index 000000000..add4d42c5
--- /dev/null
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/ZeroShotObjectDetectionTranslator.java
@@ -0,0 +1,236 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.huggingface.translator;
+
+import ai.djl.huggingface.tokenizers.Encoding;
+import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.VisionLanguageInput;
+import ai.djl.modality.cv.output.BoundingBox;
+import ai.djl.modality.cv.output.DetectedObjects;
+import ai.djl.modality.cv.output.Rectangle;
+import ai.djl.modality.cv.translator.BaseImageTranslator;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.ndarray.NDManager;
+import ai.djl.translate.ArgumentsUtil;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslateException;
+import ai.djl.translate.Translator;
+import ai.djl.translate.TranslatorContext;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/** The translator for Huggingface zero-shot-object-detection model. */
+public class ZeroShotObjectDetectionTranslator
+        implements NoBatchifyTranslator<VisionLanguageInput, DetectedObjects> {
+
+    private HuggingFaceTokenizer tokenizer;
+    private BaseImageTranslator<?> imageProcessor;
+    private boolean int32;
+    private float threshold;
+
+    ZeroShotObjectDetectionTranslator(
+            HuggingFaceTokenizer tokenizer,
+            BaseImageTranslator<?> imageProcessor,
+            boolean int32,
+            float threshold) {
+        this.tokenizer = tokenizer;
+        this.imageProcessor = imageProcessor;
+        this.int32 = int32;
+        this.threshold = threshold;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, VisionLanguageInput input)
+            throws TranslateException {
+        NDManager manager = ctx.getNDManager();
+
+        String[] candidates = input.getCandidates();
+        if (candidates == null || candidates.length == 0) {
+            throw new TranslateException("Missing candidates in input");
+        }
+
+        Encoding[] encodings = tokenizer.batchEncode(candidates);
+        NDList list = Encoding.toNDList(encodings, manager, false, int32);
+        Image img = input.getImage();
+        NDList imageFeatures = imageProcessor.processInput(ctx, img);
+        NDArray array = imageFeatures.get(0).expandDims(0);
+        list.add(array);
+
+        ctx.setAttachment("candidates", candidates);
+        return list;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public DetectedObjects processOutput(TranslatorContext ctx, NDList list)
+            throws TranslateException {
+        NDArray logits = list.get("logits");
+        NDArray boxes = list.get("pred_boxes");
+        NDArray labels = logits.argMax(-1);
+        NDArray scores = logits.max(new int[] {-1}).getNDArrayInternal().sigmoid();
+        NDArray selected = scores.gt(threshold);
+
+        scores = scores.get(selected);
+        labels = labels.get(selected);
+        boxes = boxes.get(selected);
+
+        float[] prob = scores.toFloatArray();
+        long[] labelsIndex = labels.toLongArray();
+        float[] box = boxes.toFloatArray();
+
+        String[] candidates = (String[]) ctx.getAttachment("candidates");
+
+        List<String> classes = new ArrayList<>(labelsIndex.length);
+        List<Double> probabilities = new ArrayList<>(labelsIndex.length);
+        List<BoundingBox> boundingBoxes = new ArrayList<>(labelsIndex.length);
+        int width = (Integer) ctx.getAttachment("width");
+        int height = (Integer) ctx.getAttachment("height");
+
+        for (int i = 0; i < labelsIndex.length; i++) {
+            classes.add(candidates[(int) labelsIndex[i]]);
+            int pos = i * 4;
+            float x = box[pos];
+            float y = box[pos + 1];
+            float w = box[pos + 2];
+            float h = box[pos + 3];
+            x = x - w / 2;
+            y = y - h / 2;
+            // remove padding stretch
+            if (width > height) {
+                y = y * width / height;
+                h = h * width / height;
+            } else if (width < height) {
+                x = x * height / width;
+                w = w * height / width;
+            }
+
+            BoundingBox bbox = new Rectangle(x, y, w, h);
+            boundingBoxes.add(bbox);
+            probabilities.add((double) prob[i]);
+        }
+        return new DetectedObjects(classes, probabilities, boundingBoxes);
+    }
+
+    /**
+     * Creates a builder to build a {@code ZeroShotObjectDetectionTranslator}.
+     *
+     * @param tokenizer the tokenizer
+     * @return a new builder
+     */
+    public static Builder builder(HuggingFaceTokenizer tokenizer) {
+        return new Builder(tokenizer);
+    }
+
+    /**
+     * Creates a builder to build a {@code ZeroShotObjectDetectionTranslator}.
+     *
+     * @param tokenizer the tokenizer
+     * @param arguments the models' arguments
+     * @return a new builder
+     */
+    public static Builder builder(HuggingFaceTokenizer tokenizer, Map<String, ?> arguments) {
+        Builder builder = builder(tokenizer);
+        builder.configure(arguments);
+
+        return builder;
+    }
+
+    /** The builder for zero-shot classification translator. */
+    public static final class Builder extends BaseImageTranslator.BaseBuilder<Builder> {
+
+        private HuggingFaceTokenizer tokenizer;
+        private boolean int32;
+        private float threshold = 0.2f;
+
+        Builder(HuggingFaceTokenizer tokenizer) {
+            this.tokenizer = tokenizer;
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        protected Builder self() {
+            return this;
+        }
+
+        /**
+         * Sets the threshold for prediction accuracy.
+         *
+         * <p>Predictions below the threshold will be dropped.
+         *
+         * @param threshold the threshold for the prediction accuracy
+         * @return this builder
+         */
+        public Builder optThreshold(float threshold) {
+            this.threshold = threshold;
+            return this;
+        }
+
+        /**
+         * Sets if use int32 datatype for the {@link Translator}.
+         *
+         * @param int32 true to include token types
+         * @return this builder
+         */
+        public Builder optInt32(boolean int32) {
+            this.int32 = int32;
+            return this;
+        }
+
+        /**
+         * Configures the builder with the model arguments.
+         *
+         * @param arguments the model arguments
+         */
+        public void configure(Map<String, ?> arguments) {
+            configPreProcess(arguments);
+            optInt32(ArgumentsUtil.booleanValue(arguments, "int32"));
+            optThreshold(ArgumentsUtil.floatValue(arguments, "threshold", 0.2f));
+        }
+
+        /**
+         * Builds the translator.
+         *
+         * @return the new translator
+         * @throws IOException if I/O error occurs
+         */
+        public ZeroShotObjectDetectionTranslator build() throws IOException {
+            ImageTranslator imageProcessor = new ImageTranslator(this);
+            return new ZeroShotObjectDetectionTranslator(
+                    tokenizer, imageProcessor, int32, threshold);
+        }
+    }
+
+    private static final class ImageTranslator extends BaseImageTranslator<Void> {
+
+        /**
+         * Constructs an {@code ImageTranslator} with the provided builder.
+         *
+         * @param builder the data to build with
+         */
+        public ImageTranslator(BaseBuilder<?> builder) {
+            super(builder);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public Void processOutput(TranslatorContext ctx, NDList list) {
+            return null;
+        }
+    }
+}
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/ZeroShotObjectDetectionTranslatorFactory.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/ZeroShotObjectDetectionTranslatorFactory.java
new file mode 100644
index 000000000..45d79ec3d
--- /dev/null
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/translator/ZeroShotObjectDetectionTranslatorFactory.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.huggingface.translator;
+
+import ai.djl.Model;
+import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.modality.cv.VisionLanguageInput;
+import ai.djl.modality.cv.output.DetectedObjects;
+import ai.djl.modality.cv.translator.ZeroShotObjectDetectionServingTranslator;
+import ai.djl.translate.TranslateException;
+import ai.djl.translate.Translator;
+import ai.djl.translate.TranslatorFactory;
+import ai.djl.util.Pair;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.reflect.Type;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A {@link TranslatorFactory} that creates a {@link ZeroShotObjectDetectionTranslatorFactory}
+ * instance.
+ */
+public class ZeroShotObjectDetectionTranslatorFactory implements TranslatorFactory, Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    private static final Set<Pair<Type, Type>> SUPPORTED_TYPES = new HashSet<>();
+
+    static {
+        SUPPORTED_TYPES.add(new Pair<>(VisionLanguageInput.class, DetectedObjects.class));
+        SUPPORTED_TYPES.add(new Pair<>(Input.class, Output.class));
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Set<Pair<Type, Type>> getSupportedTypes() {
+        return SUPPORTED_TYPES;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    @SuppressWarnings("unchecked")
+    public <I, O> Translator<I, O> newInstance(
+            Class<I> input, Class<O> output, Model model, Map<String, ?> arguments)
+            throws TranslateException {
+        Path modelPath = model.getModelPath();
+        try {
+            HuggingFaceTokenizer tokenizer =
+                    HuggingFaceTokenizer.builder(arguments)
+                            .optTokenizerPath(modelPath)
+                            .optManager(model.getNDManager())
+                            .build();
+            ZeroShotObjectDetectionTranslator translator =
+                    ZeroShotObjectDetectionTranslator.builder(tokenizer, arguments).build();
+            if (input == VisionLanguageInput.class && output == DetectedObjects.class) {
+                return (Translator<I, O>) translator;
+            } else if (input == Input.class && output == Output.class) {
+                return (Translator<I, O>) new ZeroShotObjectDetectionServingTranslator(translator);
+            }
+            throw new IllegalArgumentException("Unsupported input/output types.");
+        } catch (IOException e) {
+            throw new TranslateException("Failed to load tokenizer.", e);
+        }
+    }
+}
diff --git a/extensions/tokenizers/src/main/java/ai/djl/huggingface/zoo/HfModelZoo.java b/extensions/tokenizers/src/main/java/ai/djl/huggingface/zoo/HfModelZoo.java
index b8a0b76f8..7b844dd20 100644
--- a/extensions/tokenizers/src/main/java/ai/djl/huggingface/zoo/HfModelZoo.java
+++ b/extensions/tokenizers/src/main/java/ai/djl/huggingface/zoo/HfModelZoo.java
@@ -13,6 +13,7 @@
 package ai.djl.huggingface.zoo;
 
 import ai.djl.Application;
+import ai.djl.Application.CV;
 import ai.djl.Application.NLP;
 import ai.djl.engine.Engine;
 import ai.djl.repository.Repository;
@@ -68,6 +69,7 @@ public class HfModelZoo extends ModelZoo {
             synchronized (HfModelZoo.class) {
                 if (!initialized) {
                     Version version = new Version(Engine.getDjlVersion());
+                    addModels(CV.ZERO_SHOT_OBJECT_DETECTION, version);
                     addModels(NLP.FILL_MASK, version);
                     addModels(NLP.QUESTION_ANSWER, version);
                     addModels(NLP.TEXT_CLASSIFICATION, version);


Test patch:
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/ZeroShotObjectDetectionTranslatorTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/ZeroShotObjectDetectionTranslatorTest.java
new file mode 100644
index 000000000..23c1acbe2
--- /dev/null
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/ZeroShotObjectDetectionTranslatorTest.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.huggingface.tokenizers;
+
+import ai.djl.Model;
+import ai.djl.ModelException;
+import ai.djl.huggingface.translator.ZeroShotObjectDetectionTranslatorFactory;
+import ai.djl.inference.Predictor;
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.modality.cv.VisionLanguageInput;
+import ai.djl.modality.cv.output.DetectedObjects;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.ndarray.NDManager;
+import ai.djl.ndarray.index.NDIndex;
+import ai.djl.ndarray.types.Shape;
+import ai.djl.nn.Block;
+import ai.djl.nn.LambdaBlock;
+import ai.djl.repository.zoo.Criteria;
+import ai.djl.repository.zoo.ZooModel;
+import ai.djl.testing.Assertions;
+import ai.djl.translate.TranslateException;
+import ai.djl.util.JsonUtils;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ZeroShotObjectDetectionTranslatorTest {
+
+    @Test
+    public void testZeroShotObjectDetectionTranslator()
+            throws ModelException, IOException, TranslateException {
+        String url = "http://images.cocodataset.org/val2017/000000039769.jpg";
+        Image img = ImageFactory.getInstance().fromUrl(url);
+        VisionLanguageInput textInput = new VisionLanguageInput(img, new String[] {"a cat"});
+
+        Block block =
+                new LambdaBlock(
+                        a -> {
+                            NDManager manager = a.getManager();
+                            NDArray logits = manager.zeros(new Shape(1, 3600, 1));
+                            logits.setName("logits");
+                            logits.set(new NDIndex(0), -4);
+                            logits.set(new NDIndex(0, 0, 0), 2);
+                            NDArray boxes = manager.zeros(new Shape(1, 3600, 4));
+                            boxes.setName("pred_boxes");
+                            boxes.set(new NDIndex(0, 0, 0), 0.76164067);
+                            boxes.set(new NDIndex(0, 0, 1), 0.28735778);
+                            boxes.set(new NDIndex(0, 0, 2), 0.48600525);
+                            boxes.set(new NDIndex(0, 0, 3), 0.50610954);
+                            return new NDList(logits, boxes);
+                        },
+                        "model");
+
+        Path modelDir = Paths.get("build/model");
+        Files.createDirectories(modelDir);
+
+        Criteria<VisionLanguageInput, DetectedObjects> criteria =
+                Criteria.builder()
+                        .setTypes(VisionLanguageInput.class, DetectedObjects.class)
+                        .optModelPath(modelDir)
+                        .optBlock(block)
+                        .optEngine("PyTorch")
+                        .optArgument("width", 960)
+                        .optArgument("height", 960)
+                        .optArgument("pad", 128)
+                        .optArgument("resize", "true")
+                        .optArgument("threshold", "0.2")
+                        .optArgument("tokenizer", "bert-base-uncased")
+                        .optOption("hasParameter", "false")
+                        .optArgument("toTensor", true)
+                        .optTranslatorFactory(new ZeroShotObjectDetectionTranslatorFactory())
+                        .build();
+
+        try (ZooModel<VisionLanguageInput, DetectedObjects> model = criteria.loadModel();
+                Predictor<VisionLanguageInput, DetectedObjects> predictor = model.newPredictor()) {
+            DetectedObjects.DetectedObject res = predictor.predict(textInput).best();
+            Assertions.assertAlmostEquals(res.getProbability(), 0.88079);
+            Assert.assertEquals(res.getClassName(), "a cat");
+        }
+
+        Criteria<Input, Output> criteria2 =
+                Criteria.builder()
+                        .setTypes(Input.class, Output.class)
+                        .optModelPath(modelDir)
+                        .optBlock(block)
+                        .optEngine("PyTorch")
+                        .optArgument("tokenizer", "bert-base-uncased")
+                        .optOption("hasParameter", "false")
+                        .optTranslatorFactory(new ZeroShotObjectDetectionTranslatorFactory())
+                        .build();
+
+        try (ZooModel<Input, Output> model = criteria2.loadModel();
+                Predictor<Input, Output> predictor = model.newPredictor()) {
+            Input input = new Input();
+            Map<String, Object> map = new ConcurrentHashMap<>();
+            map.put("image_url", url);
+            map.put("candidate_labels", textInput.getCandidates());
+            input.add(JsonUtils.toJson(map));
+            Output out = predictor.predict(input);
+            DetectedObjects res = (DetectedObjects) out.getData().getAsObject();
+            DetectedObjects.DetectedObject detection = res.best();
+            Assertions.assertAlmostEquals(detection.getProbability(), 0.88079);
+            Assert.assertEquals(detection.getClassName(), "a cat");
+
+            map.remove("candidate_labels");
+            Input input1 = new Input();
+            input1.add(JsonUtils.toJson(map));
+            Assert.assertThrows(TranslateException.class, () -> predictor.predict(input1));
+        }
+
+        try (Model model = Model.newInstance("test")) {
+            model.setBlock(block);
+            Map<String, String> options = new HashMap<>();
+            options.put("hasParameter", "false");
+            model.load(modelDir, "test", options);
+
+            ZeroShotObjectDetectionTranslatorFactory factory =
+                    new ZeroShotObjectDetectionTranslatorFactory();
+            Map<String, String> arguments = new HashMap<>();
+
+            Assert.assertThrows(
+                    TranslateException.class,
+                    () -> factory.newInstance(String.class, Integer.class, model, arguments));
+
+            arguments.put("tokenizer", "bert-base-uncased");
+
+            Assert.assertThrows(
+                    IllegalArgumentException.class,
+                    () -> factory.newInstance(String.class, Integer.class, model, arguments));
+        }
+    }
+}