Repository: deepjavalibrary/djl
Pull request #3398 (2024-08-08 04:25:08)

[example] Enable PyTorch for some training example
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here


codecov-commenter: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3398?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 58.24%. Comparing base [(`d7c8a74`)](https://app.codecov.io/gh/deepjavalibrary/djl/commit/d7c8a74431769245ca7c2ecad6e2e98205aac72a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary) to head [(`b959e2c`)](https://app.codecov.io/gh/deepjavalibrary/djl/commit/b959e2ce6ed99b58494560781d0e17a967914047?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).
> Report is 5 commits behind head on master.

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.
> :exclamation:  There is a different number of reports uploaded between BASE (d7c8a74) and HEAD (b959e2c). Click for more details.
> 
> <details><summary>HEAD has 2 uploads less than BASE</summary>
>
>| Flag | BASE (d7c8a74) | HEAD (b959e2c) |
>|------|------|------|
>||3|1|
></details>

<details><summary>Additional details and impacted files</summary>


```diff
@@              Coverage Diff              @@
##             master    #3398       +/-   ##
=============================================
- Coverage     72.86%   58.24%   -14.63%     
+ Complexity     7352     5736     -1616     
=============================================
  Files           668      668               
  Lines         32509    32509               
  Branches       3439     3439               
=============================================
- Hits          23688    18934     -4754     
- Misses         7207    12125     +4918     
+ Partials       1614     1450      -164     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/deepjavalibrary/djl/pull/3398?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=deepjavalibrary).


Patch:
diff --git a/api/src/main/java/ai/djl/modality/nlp/TextPrompt.java b/api/src/main/java/ai/djl/modality/nlp/TextPrompt.java
index 2793fea25..202e15e59 100644
--- a/api/src/main/java/ai/djl/modality/nlp/TextPrompt.java
+++ b/api/src/main/java/ai/djl/modality/nlp/TextPrompt.java
@@ -71,6 +71,12 @@ public final class TextPrompt {
      */
     public static TextPrompt parseInput(Input input) throws TranslateException {
         String contentType = input.getProperty("Content-Type", null);
+        if (contentType != null) {
+            int pos = contentType.indexOf(';');
+            if (pos > 0) {
+                contentType = contentType.substring(0, pos);
+            }
+        }
         String text = input.getData().getAsString();
         if (!"application/json".equals(contentType)) {
             return new TextPrompt(text);
diff --git a/api/src/main/java/ai/djl/modality/nlp/translator/CrossEncoderServingTranslator.java b/api/src/main/java/ai/djl/modality/nlp/translator/CrossEncoderServingTranslator.java
index 75338df45..6c92a3c39 100644
--- a/api/src/main/java/ai/djl/modality/nlp/translator/CrossEncoderServingTranslator.java
+++ b/api/src/main/java/ai/djl/modality/nlp/translator/CrossEncoderServingTranslator.java
@@ -25,18 +25,17 @@ import ai.djl.util.JsonUtils;
 import ai.djl.util.PairList;
 import ai.djl.util.StringPair;
 
+import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
 import com.google.gson.JsonParseException;
-import com.google.gson.reflect.TypeToken;
 
-import java.lang.reflect.Type;
+import java.util.ArrayList;
 import java.util.List;
 
 /** A {@link Translator} that can handle generic cross encoder {@link Input} and {@link Output}. */
 public class CrossEncoderServingTranslator implements NoBatchifyTranslator<Input, Output> {
 
-    private static final Type LIST_TYPE = new TypeToken<List<StringPair>>() {}.getType();
-
     private Translator<StringPair, float[]> translator;
 
     /**
@@ -63,31 +62,65 @@ public class CrossEncoderServingTranslator implements NoBatchifyTranslator<Input
         }
 
         String contentType = input.getProperty("Content-Type", null);
-        StringPair pair;
+        if (contentType != null) {
+            int pos = contentType.indexOf(';');
+            if (pos > 0) {
+                contentType = contentType.substring(0, pos);
+            }
+        }
+        StringPair pair = null;
         if ("application/json".equals(contentType)) {
             String json = input.getData().getAsString();
             try {
                 JsonElement element = JsonUtils.GSON.fromJson(json, JsonElement.class);
                 if (element.isJsonArray()) {
                     ctx.setAttachment("batch", Boolean.TRUE);
-                    List<StringPair> inputs = JsonUtils.GSON.fromJson(json, LIST_TYPE);
+                    JsonArray array = element.getAsJsonArray();
+                    int size = array.size();
+                    List<StringPair> inputs = new ArrayList<>(size);
+                    for (int i = 0; i < size; ++i) {
+                        JsonObject obj = array.get(i).getAsJsonObject();
+                        inputs.add(parseStringPair(obj));
+                    }
                     return translator.batchProcessInput(ctx, inputs);
-                }
-
-                pair = JsonUtils.GSON.fromJson(json, StringPair.class);
-                if (pair.getKey() == null || pair.getValue() == null) {
-                    throw new TranslateException("Missing key or value in json.");
+                } else if (element.isJsonObject()) {
+                    JsonObject obj = element.getAsJsonObject();
+                    JsonElement query = obj.get("query");
+                    if (query != null) {
+                        String key = query.getAsString();
+                        JsonArray texts = obj.get("texts").getAsJsonArray();
+                        int size = texts.size();
+                        List<StringPair> inputs = new ArrayList<>(size);
+                        for (int i = 0; i < size; ++i) {
+                            String value = texts.get(i).getAsString();
+                            inputs.add(new StringPair(key, value));
+                        }
+                        ctx.setAttachment("batch", Boolean.TRUE);
+                        return translator.batchProcessInput(ctx, inputs);
+                    } else {
+                        pair = parseStringPair(obj);
+                    }
+                } else {
+                    throw new TranslateException("Unexpected json type");
                 }
             } catch (JsonParseException e) {
                 throw new TranslateException("Input is not a valid json.", e);
             }
         } else {
+            String text = input.getAsString("text");
+            String textPair = input.getAsString("text_pair");
+            if (text != null && textPair != null) {
+                pair = new StringPair(text, textPair);
+            }
             String key = input.getAsString("key");
             String value = input.getAsString("value");
-            if (key == null || value == null) {
-                throw new TranslateException("Missing key or value in input.");
+            if (key != null && value != null) {
+                pair = new StringPair(key, value);
             }
-            pair = new StringPair(key, value);
+        }
+
+        if (pair == null) {
+            throw new TranslateException("Missing key or value in input.");
         }
 
         NDList ret = translator.processInput(ctx, pair);
@@ -115,4 +148,18 @@ public class CrossEncoderServingTranslator implements NoBatchifyTranslator<Input
         }
         return output;
     }
+
+    private StringPair parseStringPair(JsonObject json) throws TranslateException {
+        JsonElement text = json.get("text");
+        JsonElement textPair = json.get("text_pair");
+        if (text != null && textPair != null) {
+            return new StringPair(text.getAsString(), textPair.getAsString());
+        }
+        JsonElement key = json.get("key");
+        JsonElement value = json.get("value");
+        if (key != null && value != null) {
+            return new StringPair(key.getAsString(), value.getAsString());
+        }
+        throw new TranslateException("Missing text or text_pair in json.");
+    }
 }
diff --git a/api/src/main/java/ai/djl/modality/nlp/translator/QaServingTranslator.java b/api/src/main/java/ai/djl/modality/nlp/translator/QaServingTranslator.java
index 4abc0a1b5..45e3f59af 100644
--- a/api/src/main/java/ai/djl/modality/nlp/translator/QaServingTranslator.java
+++ b/api/src/main/java/ai/djl/modality/nlp/translator/QaServingTranslator.java
@@ -65,6 +65,12 @@ public class QaServingTranslator implements NoBatchifyTranslator<Input, Output>
         }
 
         String contentType = input.getProperty("Content-Type", null);
+        if (contentType != null) {
+            int pos = contentType.indexOf(';');
+            if (pos > 0) {
+                contentType = contentType.substring(0, pos);
+            }
+        }
         QAInput qa;
         if ("application/json".equals(contentType)) {
             String json = input.getData().getAsString();
diff --git a/extensions/tokenizers/src/main/python/djl_converter/model_converter.py b/extensions/tokenizers/src/main/python/djl_converter/model_converter.py
index 802439c67..2d0f99827 100644
--- a/extensions/tokenizers/src/main/python/djl_converter/model_converter.py
+++ b/extensions/tokenizers/src/main/python/djl_converter/model_converter.py
@@ -43,7 +43,7 @@ def main():
 
     if os.listdir(output_dir):
         logging.error(f"output directory: {output_dir} is not empty.")
-        return
+        return False
 
     if os.path.exists(args.model_id):
         logging.info(f"converting local model: {args.model_id}")
@@ -66,7 +66,7 @@ def main():
     if not task:
         logging.error(
             f"Unsupported model architecture: {arch} for {args.model_id}.")
-        return
+        return False
 
     converter = SUPPORTED_TASKS[task]
 
@@ -75,12 +75,16 @@ def main():
                                                  output_dir, False)
         if result:
             logging.info(f"Convert model {model_info.modelId} finished.")
+            return True
         else:
             logging.error(f"{model_info.modelId}: {reason}")
     except Exception as e:
         logging.warning(f"Failed to convert model: {model_info.modelId}.")
         logging.warning(e, exc_info=True)
 
+    return False
+
 
 if __name__ == "__main__":
-    main()
+    if not main():
+        sys.exit(1)


Test patch:
diff --git a/examples/src/test/java/ai/djl/examples/training/TrainCaptchaTest.java b/examples/src/test/java/ai/djl/examples/training/TrainCaptchaTest.java
index a2ccbbf43..5c01e1cc8 100644
--- a/examples/src/test/java/ai/djl/examples/training/TrainCaptchaTest.java
+++ b/examples/src/test/java/ai/djl/examples/training/TrainCaptchaTest.java
@@ -27,7 +27,22 @@ public class TrainCaptchaTest {
     public void testTrainCaptcha() throws IOException, TranslateException {
         TestRequirements.linux();
 
-        // TODO: PyTorch -- cuDNN error: CUDNN_STATUS_VERSION_MISMATCH
+        // TODO: PyTorch
+        /*
+        ai.djl.engine.EngineException: index 11 is out of bounds for dimension 1 with size 11
+            at app//ai.djl.pytorch.jni.PyTorchLibrary.torchGather(Native Method)
+            at app//ai.djl.pytorch.jni.JniUtils.pick(JniUtils.java:581)
+            at app//ai.djl.pytorch.jni.JniUtils.indexAdv(JniUtils.java:417)
+            at app//ai.djl.pytorch.engine.PtNDArrayIndexer.get(PtNDArrayIndexer.java:74)
+            at app//ai.djl.ndarray.NDArray.get(NDArray.java:614)
+            at app//ai.djl.ndarray.NDArray.get(NDArray.java:603)
+            at app//ai.djl.training.loss.SoftmaxCrossEntropyLoss.evaluate(SoftmaxCrossEntropyLoss.java:86)
+            at app//ai.djl.training.loss.IndexLoss.evaluate(IndexLoss.java:55)
+            at app//ai.djl.training.loss.AbstractCompositeLoss.evaluate(AbstractCompositeLoss.java:68)
+            at app//ai.djl.training.EasyTrain.trainSplit(EasyTrain.java:124)
+            at app//ai.djl.training.EasyTrain.trainBatch(EasyTrain.java:110)
+            at app//ai.djl.training.EasyTrain.fit(EasyTrain.java:58)
+         */
         String[] args = new String[] {"-g", "1", "-e", "1", "-m", "2", "--engine", "MXNet"};
         TrainingResult result = TrainCaptcha.runExample(args);
         Assert.assertNotNull(result);
diff --git a/examples/src/test/java/ai/djl/examples/training/TrainMnistWithLSTMTest.java b/examples/src/test/java/ai/djl/examples/training/TrainMnistWithLSTMTest.java
index 7693e1954..714c25a9e 100644
--- a/examples/src/test/java/ai/djl/examples/training/TrainMnistWithLSTMTest.java
+++ b/examples/src/test/java/ai/djl/examples/training/TrainMnistWithLSTMTest.java
@@ -12,7 +12,6 @@
  */
 package ai.djl.examples.training;
 
-import ai.djl.engine.Engine;
 import ai.djl.training.TrainingResult;
 import ai.djl.translate.TranslateException;
 
@@ -25,14 +24,7 @@ public class TrainMnistWithLSTMTest {
 
     @Test
     public void testTrainMnistWithLSTM() throws IOException, TranslateException {
-        String[] args;
-        Engine engine = Engine.getEngine("PyTorch");
-        if (engine.getGpuCount() > 0) {
-            // TODO: PyTorch -- cuDNN error: CUDNN_STATUS_VERSION_MISMATCH
-            args = new String[] {"-g", "1", "-e", "1", "-m", "2", "--engine", "MXNet"};
-        } else {
-            args = new String[] {"-g", "1", "-e", "1", "-m", "2"};
-        }
+        String[] args = {"-g", "1", "-e", "1", "-m", "2"};
         TrainingResult result = TrainMnistWithLSTM.runExample(args);
         Assert.assertNotNull(result);
     }
diff --git a/examples/src/test/java/ai/djl/examples/training/TrainResNetTest.java b/examples/src/test/java/ai/djl/examples/training/TrainResNetTest.java
index cde2e1752..9b249df7c 100644
--- a/examples/src/test/java/ai/djl/examples/training/TrainResNetTest.java
+++ b/examples/src/test/java/ai/djl/examples/training/TrainResNetTest.java
@@ -36,7 +36,6 @@ public class TrainResNetTest {
 
         // Limit max 4 gpu for cifar10 training to make it converge faster.
         // and only train 10 batch for unit test.
-        // only MXNet support symbolic model
         String[] args = {"-e", "2", "-g", "4", "-m", "10", "-p"};
         TrainingResult result = TrainResnetWithCifar10.runExample(args);
 
diff --git a/examples/src/test/java/ai/djl/examples/training/TrainSentimentAnalysisTest.java b/examples/src/test/java/ai/djl/examples/training/TrainSentimentAnalysisTest.java
index c5ef28a55..7a0bb9fbd 100644
--- a/examples/src/test/java/ai/djl/examples/training/TrainSentimentAnalysisTest.java
+++ b/examples/src/test/java/ai/djl/examples/training/TrainSentimentAnalysisTest.java
@@ -29,6 +29,7 @@ public class TrainSentimentAnalysisTest {
         TestRequirements.nightly();
         TestRequirements.gpu("MXNet", 1);
 
+        // TODO: Add a PyTorch Glove model to model zoo
         String[] args = {"-e", "1", "-g", "1", "--engine", "MXNet"};
         TrainSentimentAnalysis.runExample(args);
     }
diff --git a/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java b/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java
index 19cfdc3c1..f68efe395 100644
--- a/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java
+++ b/examples/src/test/java/ai/djl/examples/training/TrainTimeSeriesTest.java
@@ -13,7 +13,6 @@
 
 package ai.djl.examples.training;
 
-import ai.djl.engine.Engine;
 import ai.djl.training.TrainingResult;
 import ai.djl.translate.TranslateException;
 
@@ -26,15 +25,7 @@ public class TrainTimeSeriesTest {
 
     @Test
     public void testTrainTimeSeries() throws TranslateException, IOException {
-        String[] args;
-        Engine engine = Engine.getEngine("PyTorch");
-        if (engine.getGpuCount() > 0) {
-            // TODO: PyTorch -- cuDNN error: CUDNN_STATUS_VERSION_MISMATCH
-            args = new String[] {"-g", "1", "-e", "5", "-b", "32", "--engine", "MXNet"};
-        } else {
-            args = new String[] {"-g", "1", "-e", "5", "-b", "32"};
-        }
-
+        String[] args = {"-g", "1", "-e", "5", "-b", "32"};
         TrainingResult result = TrainTimeSeries.runExample(args);
         Assert.assertNotNull(result);
         float loss = result.getTrainLoss();
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java
index 14336182d..b59e840e7 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/CrossEncoderTranslatorTest.java
@@ -38,11 +38,13 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 public class CrossEncoderTranslatorTest {
 
     @Test
+    @SuppressWarnings("unchecked")
     public void testCrossEncoderTranslator()
             throws ModelException, IOException, TranslateException {
         String text1 = "Sentence 1";
@@ -119,6 +121,48 @@ public class CrossEncoderTranslatorTest {
             float[] buf = (float[]) res.getData().getAsObject();
             Assert.assertEquals(buf[0], 0.32455865, 0.0001);
 
+            input = new Input();
+            input.add("text", text1);
+            input.add("text_pair", text2);
+            res = predictor.predict(input);
+            buf = (float[]) res.getData().getAsObject();
+            Assert.assertEquals(buf[0], 0.32455865, 0.0001);
+
+            input = new Input();
+            input.addProperty("Content-Type", "application/json; charset=utf-8");
+            input.add("data", "{\"text\": \"" + text1 + "\", \"text_pair\": \"" + text2 + "\"}");
+            res = predictor.predict(input);
+            buf = (float[]) res.getData().getAsObject();
+            Assert.assertEquals(buf[0], 0.32455865, 0.0001);
+
+            input = new Input();
+            input.addProperty("Content-Type", "application/json; charset=utf-8");
+            input.add("data", "{\"key\": \"" + text1 + "\", \"value\": \"" + text2 + "\"}");
+            res = predictor.predict(input);
+            buf = (float[]) res.getData().getAsObject();
+            Assert.assertEquals(buf[0], 0.32455865, 0.0001);
+
+            input = new Input();
+            input.addProperty("Content-Type", "application/json");
+            input.add("data", "{\"query\": \"" + text1 + "\", \"texts\": [\"" + text2 + "\"]}");
+            res = predictor.predict(input);
+            buf = ((List<float[]>) res.getData().getAsObject()).get(0);
+            Assert.assertEquals(buf[0], 0.32455865, 0.0001);
+
+            input = new Input();
+            input.addProperty("Content-Type", "application/json");
+            input.add("data", "{\"query\": \"" + text1 + "\", \"texts\": [\"" + text2 + "\"]}");
+            res = predictor.predict(input);
+            buf = ((List<float[]>) res.getData().getAsObject()).get(0);
+            Assert.assertEquals(buf[0], 0.32455865, 0.0001);
+
+            input = new Input();
+            input.addProperty("Content-Type", "application/json");
+            input.add("data", "[{\"text\": \"" + text1 + "\", \"text_pair\": \"" + text2 + "\"}]");
+            res = predictor.predict(input);
+            buf = ((List<float[]>) res.getData().getAsObject()).get(0);
+            Assert.assertEquals(buf[0], 0.32455865, 0.0001);
+
             Assert.assertThrows(TranslateException.class, () -> predictor.predict(new Input()));
 
             Assert.assertThrows(
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/QuestionAnsweringTranslatorTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/QuestionAnsweringTranslatorTest.java
index 14e030e76..acb283734 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/QuestionAnsweringTranslatorTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/QuestionAnsweringTranslatorTest.java
@@ -123,7 +123,7 @@ public class QuestionAnsweringTranslatorTest {
                     TranslateException.class,
                     () -> {
                         Input req = new Input();
-                        req.addProperty("Content-Type", "application/json");
+                        req.addProperty("Content-Type", "application/json; charset=utf-8");
                         req.add("Invalid json");
                         predictor.predict(req);
                     });
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/TextEmbeddingTranslatorTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/TextEmbeddingTranslatorTest.java
index 3dddf301f..15a670860 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/TextEmbeddingTranslatorTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/tokenizers/TextEmbeddingTranslatorTest.java
@@ -213,7 +213,7 @@ public class TextEmbeddingTranslatorTest {
             Map<String, String> map = new HashMap<>();
             map.put("inputs", text);
             input.add(JsonUtils.GSON.toJson(map));
-            input.addProperty("Content-Type", "application/json");
+            input.addProperty("Content-Type", "application/json; charset=utf-8");
             out = predictor.predict(input);
             res = (float[]) out.getData().getAsObject();
             Assert.assertEquals(res.length, 384);