Repository: deepjavalibrary/djl
Pull request #2245 (2022-12-19 20:12:06)

[pytorch] Upgrade to PyTorch 1.13.1
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/.github/workflows/native_s3_pytorch_android.yml b/.github/workflows/native_s3_pytorch_android.yml
index 70bdeed6a..4d0e23872 100644
--- a/.github/workflows/native_s3_pytorch_android.yml
+++ b/.github/workflows/native_s3_pytorch_android.yml
@@ -10,7 +10,7 @@ jobs:
       matrix:
         format: ["armeabi-v7a", "arm64-v8a", "x86" ,"x86_64"]
     env:
-      PYTORCH_VERSION: "1.13.0"
+      PYTORCH_VERSION: "1.13.1"
       NDK_VERSION: "21.1.6352462"
     steps:
       - uses: actions/checkout@v3
diff --git a/api/src/main/java/ai/djl/modality/cv/ImageFactory.java b/api/src/main/java/ai/djl/modality/cv/ImageFactory.java
index 17167a8b6..4868e15a9 100644
--- a/api/src/main/java/ai/djl/modality/cv/ImageFactory.java
+++ b/api/src/main/java/ai/djl/modality/cv/ImageFactory.java
@@ -50,7 +50,7 @@ public abstract class ImageFactory {
                 Class<? extends ImageFactory> clazz =
                         Class.forName(FACTORIES[i]).asSubclass(ImageFactory.class);
                 return clazz.getConstructor().newInstance();
-            } catch (ReflectiveOperationException e) {
+            } catch (ReflectiveOperationException | ExceptionInInitializerError e) {
                 logger.trace("", e);
             }
         }
diff --git a/engines/pytorch/pytorch-engine/README.md b/engines/pytorch/pytorch-engine/README.md
index a72cba06a..23bca4408 100644
--- a/engines/pytorch/pytorch-engine/README.md
+++ b/engines/pytorch/pytorch-engine/README.md
@@ -45,6 +45,7 @@ The following table illustrates which pytorch version that DJL supports:
 
 | PyTorch engine version | PyTorch native library version            |
 |------------------------|-------------------------------------------|
+| pytorch-engine:0.21.0  | 1.11.0, 1.12.1, 1.13.1                    |
 | pytorch-engine:0.20.0  | 1.11.0, 1.12.1, 1.13.0                    |
 | pytorch-engine:0.19.0  | 1.10.0, 1.11.0, 1.12.1                    |
 | pytorch-engine:0.18.0  | 1.9.1, 1.10.0, 1.11.0                     |
diff --git a/engines/pytorch/pytorch-native/src/main/patch/1.13.1/Parallel.h b/engines/pytorch/pytorch-native/src/main/patch/1.13.1/Parallel.h
new file mode 100644
index 000000000..90ba2f6ca
--- /dev/null
+++ b/engines/pytorch/pytorch-native/src/main/patch/1.13.1/Parallel.h
@@ -0,0 +1,154 @@
+#pragma once
+#include <ATen/Config.h>
+#include <c10/macros/Macros.h>
+#include <functional>
+#include <string>
+
+namespace at {
+
+inline int64_t divup(int64_t x, int64_t y) {
+  return (x + y - 1) / y;
+}
+
+// Called during new thread initialization
+TORCH_API void init_num_threads();
+
+// Sets the number of threads to be used in parallel region
+TORCH_API void set_num_threads(int);
+
+// Returns the maximum number of threads that may be used in a parallel region
+TORCH_API int get_num_threads();
+
+// Returns the current thread number (starting from 0)
+// in the current parallel region, or 0 in the sequential region
+TORCH_API int get_thread_num();
+
+// Checks whether the code runs in parallel region
+TORCH_API bool in_parallel_region();
+
+namespace internal {
+
+// Initialise num_threads lazily at first parallel call
+TORCH_API void lazy_init_num_threads();
+
+TORCH_API void set_thread_num(int);
+
+class TORCH_API ThreadIdGuard {
+ public:
+  ThreadIdGuard(int new_id) : old_id_(at::get_thread_num()) {
+    set_thread_num(new_id);
+  }
+
+  ~ThreadIdGuard() {
+    set_thread_num(old_id_);
+  }
+
+ private:
+  int old_id_;
+};
+
+} // namespace internal
+
+/*
+parallel_for
+
+begin: index at which to start applying user function
+
+end: index at which to stop applying user function
+
+grain_size: number of elements per chunk. impacts the degree of parallelization
+
+f: user function applied in parallel to the chunks, signature:
+  void f(int64_t begin, int64_t end)
+
+Warning: parallel_for does NOT copy thread local
+states from the current thread to the worker threads.
+This means for example that Tensor operations CANNOT be used in the
+body of your function, only data pointers.
+*/
+template <class F>
+inline void parallel_for(
+    const int64_t begin,
+    const int64_t end,
+    const int64_t grain_size,
+    const F& f);
+
+/*
+parallel_reduce
+
+begin: index at which to start applying reduction
+
+end: index at which to stop applying reduction
+
+grain_size: number of elements per chunk. impacts number of elements in
+intermediate results tensor and degree of parallelization.
+
+ident: identity for binary combination function sf. sf(ident, x) needs to return
+x.
+
+f: function for reduction over a chunk. f needs to be of signature scalar_t
+f(int64_t partial_begin, int64_t partial_end, scalar_t identifiy)
+
+sf: function to combine two partial results. sf needs to be of signature
+scalar_t sf(scalar_t x, scalar_t y)
+
+For example, you might have a tensor of 10000 entires and want to sum together
+all the elements. Parallel_reduce with a grain_size of 2500 will then allocate
+an intermediate result tensor with 4 elements. Then it will execute the function
+"f" you provide and pass the beginning and end index of these chunks, so
+0-2499, 2500-4999, etc. and the combination identity. It will then write out
+the result from each of these chunks into the intermediate result tensor. After
+that it'll reduce the partial results from each chunk into a single number using
+the combination function sf and the identity ident. For a total summation this
+would be "+" and 0 respectively. This is similar to tbb's approach [1], where
+you need to provide a function to accumulate a subrange, a function to combine
+two partial results and an identity.
+
+Warning: parallel_reduce does NOT copy thread local
+states from the current thread to the worker threads.
+This means for example that Tensor operations CANNOT be used in the
+body of your function, only data pointers.
+
+[1] https://software.intel.com/en-us/node/506154
+*/
+template <class scalar_t, class F, class SF>
+inline scalar_t parallel_reduce(
+    const int64_t begin,
+    const int64_t end,
+    const int64_t grain_size,
+    const scalar_t ident,
+    const F& f,
+    const SF& sf);
+
+// Returns a detailed string describing parallelization settings
+TORCH_API std::string get_parallel_info();
+
+// Sets number of threads used for inter-op parallelism
+TORCH_API void set_num_interop_threads(int);
+
+// Returns the number of threads used for inter-op parallelism
+TORCH_API int get_num_interop_threads();
+
+// Launches inter-op parallel task
+TORCH_API void launch(std::function<void()> func);
+namespace internal {
+void launch_no_thread_state(std::function<void()> fn);
+} // namespace internal
+
+// Launches intra-op parallel task
+TORCH_API void intraop_launch(std::function<void()> func);
+
+// Returns number of intra-op threads used by default
+TORCH_API int intraop_default_num_threads();
+
+} // namespace at
+
+#if AT_PARALLEL_OPENMP
+#include <ATen/ParallelOpenMP.h> // IWYU pragma: keep
+#elif AT_PARALLEL_NATIVE
+#include <ATen/ParallelNative.h> // IWYU pragma: keep
+#elif AT_PARALLEL_NATIVE_TBB
+#include <ATen/ParallelNativeTBB.h> // IWYU pragma: keep
+#endif
+
+#include <ATen/Parallel-inl.h> // IWYU pragma: keep
diff --git a/examples/build.gradle b/examples/build.gradle
index c551329e0..1489919fe 100644
--- a/examples/build.gradle
+++ b/examples/build.gradle
@@ -8,6 +8,7 @@ dependencies {
     implementation project(":basicdataset")
     implementation project(":model-zoo")
     implementation project(":extensions:timeseries")
+    implementation project(":extensions:tokenizers")
 
     runtimeOnly project(":engines:pytorch:pytorch-model-zoo")
     runtimeOnly project(":engines:tensorflow:tensorflow-model-zoo")
diff --git a/examples/docs/clip_image_text.md b/examples/docs/clip_image_text.md
new file mode 100644
index 000000000..d16afbfb0
--- /dev/null
+++ b/examples/docs/clip_image_text.md
@@ -0,0 +1,34 @@
+# CLIP model example
+
+[CLIP model](https://huggingface.co/openai/clip-vit-base-patch32) is open-sourced by OpenAI for text-image understanding.
+It is widely used to get text and image feature and used for search domain.  User could use image to search image, use text to search image and even use text to search text with this model.
+
+In this short demo, we will do an image to text comparison to find which text is close to the corresponding image.
+
+The image we used is
+
+![](http://images.cocodataset.org/val2017/000000039769.jpg)
+
+And our input text:
+
+```
+"A photo of cats";
+"A photo of dogs";
+```
+
+We expect cats text will win based on the image.
+
+## Run the example
+
+```
+cd examples
+./gradlew run -Dmain=ai.djl.examples.inference.clip.ImageTextComparison
+```
+
+output:
+
+```
+[INFO ] - A photo of cats Probability: 0.9970879546345841
+[INFO ] - A photo of dogs Probability: 0.002912045365415886
+
+```
diff --git a/examples/pom.xml b/examples/pom.xml
index f996872ea..a9142f901 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -59,6 +59,10 @@
             <groupId>ai.djl.timeseries</groupId>
             <artifactId>timeseries</artifactId>
         </dependency>
+        <dependency>
+            <groupId>ai.djl.huggingface</groupId>
+            <artifactId>tokenizers</artifactId>
+        </dependency>
         <!-- MXNet -->
         <dependency>
             <groupId>ai.djl.mxnet</groupId>
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ClipModel.java b/examples/src/main/java/ai/djl/examples/inference/clip/ClipModel.java
new file mode 100644
index 000000000..3b7c87ef0
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ClipModel.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.ModelException;
+import ai.djl.inference.Predictor;
+import ai.djl.modality.cv.Image;
+import ai.djl.ndarray.NDList;
+import ai.djl.repository.zoo.Criteria;
+import ai.djl.repository.zoo.ZooModel;
+import ai.djl.translate.NoopTranslator;
+import ai.djl.translate.TranslateException;
+import ai.djl.util.Pair;
+
+import java.io.IOException;
+
+/**
+ * An example of inference using an CLIP model.
+ *
+ * <p>See this <a
+ * href="https://github.com/deepjavalibrary/djl/blob/master/examples/docs/clip_image_text.md">doc</a>
+ * for information about this example.
+ */
+public class ClipModel implements AutoCloseable {
+
+    private ZooModel<NDList, NDList> clip;
+    private Predictor<Image, float[]> imageFeatureExtractor;
+    private Predictor<String, float[]> textFeatureExtractor;
+    private Predictor<Pair<Image, String>, float[]> imgTextComparator;
+
+    public ClipModel() throws ModelException, IOException {
+        Criteria<NDList, NDList> criteria =
+                Criteria.builder()
+                        .setTypes(NDList.class, NDList.class)
+                        .optModelUrls("https://resources.djl.ai/demo/pytorch/clip.zip")
+                        .optTranslator(new NoopTranslator())
+                        .optEngine("PyTorch")
+                        .build();
+        clip = criteria.loadModel();
+        imageFeatureExtractor = clip.newPredictor(new ImageTranslator());
+        textFeatureExtractor = clip.newPredictor(new TextTranslator());
+        imgTextComparator = clip.newPredictor(new ImageTextTranslator());
+    }
+
+    public float[] extractTextFeatures(String inputs) throws TranslateException {
+        return textFeatureExtractor.predict(inputs);
+    }
+
+    public float[] extractImageFeatures(Image inputs) throws TranslateException {
+        return imageFeatureExtractor.predict(inputs);
+    }
+
+    public float[] compareTextAndImage(Image image, String text) throws TranslateException {
+        return imgTextComparator.predict(new Pair<>(image, text));
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void close() {
+        imageFeatureExtractor.close();
+        textFeatureExtractor.close();
+        imgTextComparator.close();
+        clip.close();
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextComparison.java b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextComparison.java
new file mode 100644
index 000000000..6b65bbe0f
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextComparison.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.ModelException;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.translate.TranslateException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public final class ImageTextComparison {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImageTextComparison.class);
+
+    private ImageTextComparison() {}
+
+    public static void main(String[] args) throws ModelException, IOException, TranslateException {
+        String text = "A photo of cats";
+        String text2 = "A photo of dogs";
+        double[] probs = compareTextAndImage(text, text2);
+        logger.info("{} Probability: {}", text, probs[0]);
+        logger.info("{} Probability: {}", text2, probs[1]);
+    }
+
+    static double[] compareTextAndImage(String text, String text2)
+            throws ModelException, IOException, TranslateException {
+        try (ClipModel model = new ClipModel()) {
+            String url = "http://images.cocodataset.org/val2017/000000039769.jpg";
+            Image img = ImageFactory.getInstance().fromUrl(url);
+            float[] logit0 = model.compareTextAndImage(img, text);
+            float[] logit1 = model.compareTextAndImage(img, text2);
+            double total = Arrays.stream(new double[] {logit0[0], logit1[0]}).map(Math::exp).sum();
+            return new double[] {Math.exp(logit0[0]) / total, Math.exp(logit1[0]) / total};
+        }
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextTranslator.java b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextTranslator.java
new file mode 100644
index 000000000..34a6fcd5a
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextTranslator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.modality.cv.Image;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslatorContext;
+import ai.djl.util.Pair;
+
+public class ImageTextTranslator implements NoBatchifyTranslator<Pair<Image, String>, float[]> {
+
+    private ImageTranslator imgTranslator;
+    private TextTranslator txtTranslator;
+
+    public ImageTextTranslator() {
+        this.imgTranslator = new ImageTranslator();
+        this.txtTranslator = new TextTranslator();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public float[] processOutput(TranslatorContext ctx, NDList list) throws Exception {
+        NDArray logitsPerImage = list.get(0);
+        return logitsPerImage.toFloatArray();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, Pair<Image, String> input) throws Exception {
+        NDList imageInput = imgTranslator.processInput(ctx, input.getKey());
+        NDList textInput = txtTranslator.processInput(ctx, input.getValue());
+        return new NDList(textInput.get(0), imageInput.get(0), textInput.get(1));
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ImageTranslator.java b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTranslator.java
new file mode 100644
index 000000000..46f970458
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTranslator.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.util.NDImageUtils;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslatorContext;
+
+public class ImageTranslator implements NoBatchifyTranslator<Image, float[]> {
+
+    /** {@inheritDoc} */
+    @Override
+    public float[] processOutput(TranslatorContext ctx, NDList list) {
+        NDArray array = list.singletonOrThrow();
+        return array.toFloatArray();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, Image input) {
+        NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
+
+        float percent = 224f / Math.min(input.getWidth(), input.getHeight());
+        int resizedWidth = Math.round(input.getWidth() * percent);
+        int resizedHeight = Math.round(input.getHeight() * percent);
+
+        array =
+                NDImageUtils.resize(
+                        array, resizedWidth, resizedHeight, Image.Interpolation.BICUBIC);
+        array = NDImageUtils.centerCrop(array, 224, 224);
+        array = NDImageUtils.toTensor(array);
+        NDArray placeholder = ctx.getNDManager().create("");
+        placeholder.setName("module_method:get_image_features");
+        return new NDList(array.expandDims(0), placeholder);
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/TextTranslator.java b/examples/src/main/java/ai/djl/examples/inference/clip/TextTranslator.java
new file mode 100644
index 000000000..35c06a0f1
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/TextTranslator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.huggingface.tokenizers.Encoding;
+import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslatorContext;
+
+public class TextTranslator implements NoBatchifyTranslator<String, float[]> {
+
+    HuggingFaceTokenizer tokenizer;
+
+    public TextTranslator() {
+        tokenizer = HuggingFaceTokenizer.newInstance("openai/clip-vit-base-patch32");
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public float[] processOutput(TranslatorContext ctx, NDList list) {
+        return list.singletonOrThrow().toFloatArray();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, String input) {
+        Encoding encoding = tokenizer.encode(input);
+        NDArray attention = ctx.getNDManager().create(encoding.getAttentionMask());
+        NDArray inputIds = ctx.getNDManager().create(encoding.getIds());
+        NDArray placeholder = ctx.getNDManager().create("");
+        placeholder.setName("module_method:get_text_features");
+        return new NDList(inputIds.expandDims(0), attention.expandDims(0), placeholder);
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/package-info.java b/examples/src/main/java/ai/djl/examples/inference/clip/package-info.java
new file mode 100644
index 000000000..36893d995
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+/** Contains examples of Text and Image vectorization. */
+package ai.djl.examples.inference.clip;
diff --git a/gradle.properties b/gradle.properties
index 6930a6271..ed47e5387 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -13,7 +13,7 @@ systemProp.org.gradle.internal.publish.checksums.insecure=true
 
 djl_version=0.21.0
 mxnet_version=1.9.1
-pytorch_version=1.13.0
+pytorch_version=1.13.1
 tensorflow_version=2.7.4
 tflite_version=2.6.2
 dlr_version=1.6.0


Test patch:
diff --git a/examples/src/test/java/ai/djl/examples/inference/clip/ClipModelTest.java b/examples/src/test/java/ai/djl/examples/inference/clip/ClipModelTest.java
new file mode 100644
index 000000000..127949a69
--- /dev/null
+++ b/examples/src/test/java/ai/djl/examples/inference/clip/ClipModelTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.ModelException;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.testing.TestRequirements;
+import ai.djl.translate.TranslateException;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+public class ClipModelTest {
+
+    @Test
+    public void testClipFeature() throws ModelException, IOException, TranslateException {
+        TestRequirements.nightly();
+        TestRequirements.engine("PyTorch");
+
+        try (ClipModel model = new ClipModel()) {
+            String input = "This is a nice day";
+            String url = "http://images.cocodataset.org/val2017/000000039769.jpg";
+            float[] textVector = model.extractTextFeatures(input);
+            Image img = ImageFactory.getInstance().fromUrl(url);
+            float[] imgVector = model.extractImageFeatures(img);
+            Assert.assertEquals(textVector.length, imgVector.length);
+        }
+    }
+
+    @Test
+    public void testClipComparison() throws ModelException, IOException, TranslateException {
+        TestRequirements.nightly();
+        TestRequirements.engine("PyTorch");
+
+        String text = "A photo of cats";
+        String text2 = "A photo of dogs";
+        double[] probs = ImageTextComparison.compareTextAndImage(text, text2);
+        Assert.assertTrue(probs[0] > probs[0]);
+    }
+}
diff --git a/examples/src/test/java/ai/djl/examples/inference/clip/package-info.java b/examples/src/test/java/ai/djl/examples/inference/clip/package-info.java
new file mode 100644
index 000000000..36893d995
--- /dev/null
+++ b/examples/src/test/java/ai/djl/examples/inference/clip/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+/** Contains examples of Text and Image vectorization. */
+package ai.djl.examples.inference.clip;