Repository: deepjavalibrary/djl
Pull request #2241 (2022-12-19 10:19:28)

Handle RuntimeException on ImageFactory::newInstance
In M1 mac, `ai.djl.opencv.OpenCVImageFactory` is not supported yet but when Initialization of ImageFactory class, it tries to initialize that first and cause the Exception. 

This makes not able to use this class also when I implemented the new supported way.

To solve this bug, I think to handle UnsupportedException on ImageFactory::newInstance() is the best way. 
Due UnsupportedException is private on `nu.pattern.OpenCV`, Handle of its super class will be the solution.


Patch:
diff --git a/api/src/main/java/ai/djl/modality/cv/ImageFactory.java b/api/src/main/java/ai/djl/modality/cv/ImageFactory.java
index 17167a8b6..4868e15a9 100644
--- a/api/src/main/java/ai/djl/modality/cv/ImageFactory.java
+++ b/api/src/main/java/ai/djl/modality/cv/ImageFactory.java
@@ -50,7 +50,7 @@ public abstract class ImageFactory {
                 Class<? extends ImageFactory> clazz =
                         Class.forName(FACTORIES[i]).asSubclass(ImageFactory.class);
                 return clazz.getConstructor().newInstance();
-            } catch (ReflectiveOperationException e) {
+            } catch (ReflectiveOperationException | ExceptionInInitializerError e) {
                 logger.trace("", e);
             }
         }
diff --git a/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngine.java b/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngine.java
index c47d8bd1e..7f4d39e83 100644
--- a/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngine.java
+++ b/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngine.java
@@ -42,7 +42,7 @@ import java.nio.file.Paths;
 public final class MxEngine extends Engine {
 
     public static final String ENGINE_NAME = "MXNet";
-    static final int RANK = 1;
+    static final int RANK = 0;
     private static final String MXNET_EXTRA_LIBRARY_VERBOSE = "MXNET_EXTRA_LIBRARY_VERBOSE";
 
     /** Constructs an MXNet Engine. */
diff --git a/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngineProvider.java b/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngineProvider.java
index 88d17caf1..9d643b05d 100644
--- a/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngineProvider.java
+++ b/engines/mxnet/mxnet-engine/src/main/java/ai/djl/mxnet/engine/MxEngineProvider.java
@@ -29,7 +29,7 @@ public class MxEngineProvider implements EngineProvider {
     /** {@inheritDoc} */
     @Override
     public int getEngineRank() {
-        return 0;
+        return MxEngine.RANK;
     }
 
     /** {@inheritDoc} */
diff --git a/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/engine/OrtEngineProvider.java b/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/engine/OrtEngineProvider.java
index 24b27aaaf..c3a59cf78 100644
--- a/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/engine/OrtEngineProvider.java
+++ b/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/engine/OrtEngineProvider.java
@@ -29,7 +29,7 @@ public class OrtEngineProvider implements EngineProvider {
     /** {@inheritDoc} */
     @Override
     public int getEngineRank() {
-        return 10;
+        return OrtEngine.RANK;
     }
 
     /** {@inheritDoc} */
diff --git a/examples/build.gradle b/examples/build.gradle
index c551329e0..1489919fe 100644
--- a/examples/build.gradle
+++ b/examples/build.gradle
@@ -8,6 +8,7 @@ dependencies {
     implementation project(":basicdataset")
     implementation project(":model-zoo")
     implementation project(":extensions:timeseries")
+    implementation project(":extensions:tokenizers")
 
     runtimeOnly project(":engines:pytorch:pytorch-model-zoo")
     runtimeOnly project(":engines:tensorflow:tensorflow-model-zoo")
diff --git a/examples/docs/clip_image_text.md b/examples/docs/clip_image_text.md
new file mode 100644
index 000000000..d16afbfb0
--- /dev/null
+++ b/examples/docs/clip_image_text.md
@@ -0,0 +1,34 @@
+# CLIP model example
+
+[CLIP model](https://huggingface.co/openai/clip-vit-base-patch32) is open-sourced by OpenAI for text-image understanding.
+It is widely used to get text and image feature and used for search domain.  User could use image to search image, use text to search image and even use text to search text with this model.
+
+In this short demo, we will do an image to text comparison to find which text is close to the corresponding image.
+
+The image we used is
+
+![](http://images.cocodataset.org/val2017/000000039769.jpg)
+
+And our input text:
+
+```
+"A photo of cats";
+"A photo of dogs";
+```
+
+We expect cats text will win based on the image.
+
+## Run the example
+
+```
+cd examples
+./gradlew run -Dmain=ai.djl.examples.inference.clip.ImageTextComparison
+```
+
+output:
+
+```
+[INFO ] - A photo of cats Probability: 0.9970879546345841
+[INFO ] - A photo of dogs Probability: 0.002912045365415886
+
+```
diff --git a/examples/pom.xml b/examples/pom.xml
index f996872ea..a9142f901 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -59,6 +59,10 @@
             <groupId>ai.djl.timeseries</groupId>
             <artifactId>timeseries</artifactId>
         </dependency>
+        <dependency>
+            <groupId>ai.djl.huggingface</groupId>
+            <artifactId>tokenizers</artifactId>
+        </dependency>
         <!-- MXNet -->
         <dependency>
             <groupId>ai.djl.mxnet</groupId>
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ClipModel.java b/examples/src/main/java/ai/djl/examples/inference/clip/ClipModel.java
new file mode 100644
index 000000000..3b7c87ef0
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ClipModel.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.ModelException;
+import ai.djl.inference.Predictor;
+import ai.djl.modality.cv.Image;
+import ai.djl.ndarray.NDList;
+import ai.djl.repository.zoo.Criteria;
+import ai.djl.repository.zoo.ZooModel;
+import ai.djl.translate.NoopTranslator;
+import ai.djl.translate.TranslateException;
+import ai.djl.util.Pair;
+
+import java.io.IOException;
+
+/**
+ * An example of inference using an CLIP model.
+ *
+ * <p>See this <a
+ * href="https://github.com/deepjavalibrary/djl/blob/master/examples/docs/clip_image_text.md">doc</a>
+ * for information about this example.
+ */
+public class ClipModel implements AutoCloseable {
+
+    private ZooModel<NDList, NDList> clip;
+    private Predictor<Image, float[]> imageFeatureExtractor;
+    private Predictor<String, float[]> textFeatureExtractor;
+    private Predictor<Pair<Image, String>, float[]> imgTextComparator;
+
+    public ClipModel() throws ModelException, IOException {
+        Criteria<NDList, NDList> criteria =
+                Criteria.builder()
+                        .setTypes(NDList.class, NDList.class)
+                        .optModelUrls("https://resources.djl.ai/demo/pytorch/clip.zip")
+                        .optTranslator(new NoopTranslator())
+                        .optEngine("PyTorch")
+                        .build();
+        clip = criteria.loadModel();
+        imageFeatureExtractor = clip.newPredictor(new ImageTranslator());
+        textFeatureExtractor = clip.newPredictor(new TextTranslator());
+        imgTextComparator = clip.newPredictor(new ImageTextTranslator());
+    }
+
+    public float[] extractTextFeatures(String inputs) throws TranslateException {
+        return textFeatureExtractor.predict(inputs);
+    }
+
+    public float[] extractImageFeatures(Image inputs) throws TranslateException {
+        return imageFeatureExtractor.predict(inputs);
+    }
+
+    public float[] compareTextAndImage(Image image, String text) throws TranslateException {
+        return imgTextComparator.predict(new Pair<>(image, text));
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void close() {
+        imageFeatureExtractor.close();
+        textFeatureExtractor.close();
+        imgTextComparator.close();
+        clip.close();
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextComparison.java b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextComparison.java
new file mode 100644
index 000000000..6b65bbe0f
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextComparison.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.ModelException;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.translate.TranslateException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+public final class ImageTextComparison {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImageTextComparison.class);
+
+    private ImageTextComparison() {}
+
+    public static void main(String[] args) throws ModelException, IOException, TranslateException {
+        String text = "A photo of cats";
+        String text2 = "A photo of dogs";
+        double[] probs = compareTextAndImage(text, text2);
+        logger.info("{} Probability: {}", text, probs[0]);
+        logger.info("{} Probability: {}", text2, probs[1]);
+    }
+
+    static double[] compareTextAndImage(String text, String text2)
+            throws ModelException, IOException, TranslateException {
+        try (ClipModel model = new ClipModel()) {
+            String url = "http://images.cocodataset.org/val2017/000000039769.jpg";
+            Image img = ImageFactory.getInstance().fromUrl(url);
+            float[] logit0 = model.compareTextAndImage(img, text);
+            float[] logit1 = model.compareTextAndImage(img, text2);
+            double total = Arrays.stream(new double[] {logit0[0], logit1[0]}).map(Math::exp).sum();
+            return new double[] {Math.exp(logit0[0]) / total, Math.exp(logit1[0]) / total};
+        }
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextTranslator.java b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextTranslator.java
new file mode 100644
index 000000000..34a6fcd5a
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTextTranslator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.modality.cv.Image;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslatorContext;
+import ai.djl.util.Pair;
+
+public class ImageTextTranslator implements NoBatchifyTranslator<Pair<Image, String>, float[]> {
+
+    private ImageTranslator imgTranslator;
+    private TextTranslator txtTranslator;
+
+    public ImageTextTranslator() {
+        this.imgTranslator = new ImageTranslator();
+        this.txtTranslator = new TextTranslator();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public float[] processOutput(TranslatorContext ctx, NDList list) throws Exception {
+        NDArray logitsPerImage = list.get(0);
+        return logitsPerImage.toFloatArray();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, Pair<Image, String> input) throws Exception {
+        NDList imageInput = imgTranslator.processInput(ctx, input.getKey());
+        NDList textInput = txtTranslator.processInput(ctx, input.getValue());
+        return new NDList(textInput.get(0), imageInput.get(0), textInput.get(1));
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/ImageTranslator.java b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTranslator.java
new file mode 100644
index 000000000..46f970458
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/ImageTranslator.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.util.NDImageUtils;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslatorContext;
+
+public class ImageTranslator implements NoBatchifyTranslator<Image, float[]> {
+
+    /** {@inheritDoc} */
+    @Override
+    public float[] processOutput(TranslatorContext ctx, NDList list) {
+        NDArray array = list.singletonOrThrow();
+        return array.toFloatArray();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, Image input) {
+        NDArray array = input.toNDArray(ctx.getNDManager(), Image.Flag.COLOR);
+
+        float percent = 224f / Math.min(input.getWidth(), input.getHeight());
+        int resizedWidth = Math.round(input.getWidth() * percent);
+        int resizedHeight = Math.round(input.getHeight() * percent);
+
+        array =
+                NDImageUtils.resize(
+                        array, resizedWidth, resizedHeight, Image.Interpolation.BICUBIC);
+        array = NDImageUtils.centerCrop(array, 224, 224);
+        array = NDImageUtils.toTensor(array);
+        NDArray placeholder = ctx.getNDManager().create("");
+        placeholder.setName("module_method:get_image_features");
+        return new NDList(array.expandDims(0), placeholder);
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/TextTranslator.java b/examples/src/main/java/ai/djl/examples/inference/clip/TextTranslator.java
new file mode 100644
index 000000000..35c06a0f1
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/TextTranslator.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.huggingface.tokenizers.Encoding;
+import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
+import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslatorContext;
+
+public class TextTranslator implements NoBatchifyTranslator<String, float[]> {
+
+    HuggingFaceTokenizer tokenizer;
+
+    public TextTranslator() {
+        tokenizer = HuggingFaceTokenizer.newInstance("openai/clip-vit-base-patch32");
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public float[] processOutput(TranslatorContext ctx, NDList list) {
+        return list.singletonOrThrow().toFloatArray();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, String input) {
+        Encoding encoding = tokenizer.encode(input);
+        NDArray attention = ctx.getNDManager().create(encoding.getAttentionMask());
+        NDArray inputIds = ctx.getNDManager().create(encoding.getIds());
+        NDArray placeholder = ctx.getNDManager().create("");
+        placeholder.setName("module_method:get_text_features");
+        return new NDList(inputIds.expandDims(0), attention.expandDims(0), placeholder);
+    }
+}
diff --git a/examples/src/main/java/ai/djl/examples/inference/clip/package-info.java b/examples/src/main/java/ai/djl/examples/inference/clip/package-info.java
new file mode 100644
index 000000000..36893d995
--- /dev/null
+++ b/examples/src/main/java/ai/djl/examples/inference/clip/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+/** Contains examples of Text and Image vectorization. */
+package ai.djl.examples.inference.clip;


Test patch:
diff --git a/examples/src/test/java/ai/djl/examples/inference/clip/ClipModelTest.java b/examples/src/test/java/ai/djl/examples/inference/clip/ClipModelTest.java
new file mode 100644
index 000000000..127949a69
--- /dev/null
+++ b/examples/src/test/java/ai/djl/examples/inference/clip/ClipModelTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.examples.inference.clip;
+
+import ai.djl.ModelException;
+import ai.djl.modality.cv.Image;
+import ai.djl.modality.cv.ImageFactory;
+import ai.djl.testing.TestRequirements;
+import ai.djl.translate.TranslateException;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+public class ClipModelTest {
+
+    @Test
+    public void testClipFeature() throws ModelException, IOException, TranslateException {
+        TestRequirements.nightly();
+        TestRequirements.engine("PyTorch");
+
+        try (ClipModel model = new ClipModel()) {
+            String input = "This is a nice day";
+            String url = "http://images.cocodataset.org/val2017/000000039769.jpg";
+            float[] textVector = model.extractTextFeatures(input);
+            Image img = ImageFactory.getInstance().fromUrl(url);
+            float[] imgVector = model.extractImageFeatures(img);
+            Assert.assertEquals(textVector.length, imgVector.length);
+        }
+    }
+
+    @Test
+    public void testClipComparison() throws ModelException, IOException, TranslateException {
+        TestRequirements.nightly();
+        TestRequirements.engine("PyTorch");
+
+        String text = "A photo of cats";
+        String text2 = "A photo of dogs";
+        double[] probs = ImageTextComparison.compareTextAndImage(text, text2);
+        Assert.assertTrue(probs[0] > probs[0]);
+    }
+}
diff --git a/examples/src/test/java/ai/djl/examples/inference/clip/package-info.java b/examples/src/test/java/ai/djl/examples/inference/clip/package-info.java
new file mode 100644
index 000000000..36893d995
--- /dev/null
+++ b/examples/src/test/java/ai/djl/examples/inference/clip/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+/** Contains examples of Text and Image vectorization. */
+package ai.djl.examples.inference.clip;