Repository: deepjavalibrary/djl
Pull request #3292 (2024-07-01 19:15:10)

[example] Adds PyTorch action recognition model to model zoo
## Description ##

Brief description of what this PR is about

- If this change is a backward incompatible change, why must this change be made?
- Interesting edge cases to note here



Patch:
diff --git a/.github/workflows/build_triton_and_ft.yml b/.github/workflows/build_triton_and_ft.yml
deleted file mode 100644
index beeff41a5..000000000
--- a/.github/workflows/build_triton_and_ft.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-name: Build Triton Server
-
-on:
-  workflow_dispatch:
-    inputs:
-      triton:
-        description: 'triton branch version'
-        required: true
-        default: 'r23.10'
-
-jobs:
-  build-triton:
-    if: github.repository == 'deepjavalibrary/djl'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: triton-inference-server/server
-          ref: ${{ github.event.inputs.triton }}
-      - name: Set up Python3
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.x'
-      - name: Build Triton Binary
-        shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
-        run: |
-          pip3 install requests
-          python3 build.py --enable-logging --enable-metrics --enable-stats --enable-cpu-metrics --enable-gpu --endpoint http
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Copy files to S3 with the AWS CLI
-        run: |
-          aws s3 cp build/install/lib/libtritonserver.so s3://djl-ai/publish/tritonserver/${{ github.event.inputs.triton }}/
-          aws s3 cp build/install/bin/tritonserver s3://djl-ai/publish/tritonserver/${{ github.event.inputs.triton }}/
-          aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/tritonserver/${{ github.event.inputs.triton }}/*"
diff --git a/api/src/main/java/ai/djl/modality/nlp/translator/NamedEntity.java b/api/src/main/java/ai/djl/modality/nlp/translator/NamedEntity.java
index 4af9c141f..72d00c468 100644
--- a/api/src/main/java/ai/djl/modality/nlp/translator/NamedEntity.java
+++ b/api/src/main/java/ai/djl/modality/nlp/translator/NamedEntity.java
@@ -100,6 +100,6 @@ public class NamedEntity {
     /** {@inheritDoc} */
     @Override
     public String toString() {
-        return JsonUtils.GSON_PRETTY.toJson(this);
+        return JsonUtils.toJson(this);
     }
 }
diff --git a/api/src/main/java/ai/djl/ndarray/BytesSupplierImpl.java b/api/src/main/java/ai/djl/ndarray/BytesSupplierImpl.java
index 93ed023a1..d3f7198c1 100644
--- a/api/src/main/java/ai/djl/ndarray/BytesSupplierImpl.java
+++ b/api/src/main/java/ai/djl/ndarray/BytesSupplierImpl.java
@@ -40,7 +40,7 @@ class BytesSupplierImpl implements BytesSupplier {
     public byte[] getAsBytes() {
         if (buf == null) {
             if (value == null) {
-                value = JsonUtils.GSON.toJson(obj) + '\n';
+                value = JsonUtils.toJson(obj) + '\n';
             }
             buf = value.getBytes(StandardCharsets.UTF_8);
         }
@@ -52,7 +52,7 @@ class BytesSupplierImpl implements BytesSupplier {
     public String getAsString() {
         if (value == null) {
             if (obj != null) {
-                value = JsonUtils.GSON.toJson(obj) + '\n';
+                value = JsonUtils.toJson(obj) + '\n';
             } else {
                 value = new String(buf, StandardCharsets.UTF_8);
             }
diff --git a/api/src/main/java/ai/djl/ndarray/internal/NDFormat.java b/api/src/main/java/ai/djl/ndarray/internal/NDFormat.java
index 51e49f6ab..4c36324b0 100644
--- a/api/src/main/java/ai/djl/ndarray/internal/NDFormat.java
+++ b/api/src/main/java/ai/djl/ndarray/internal/NDFormat.java
@@ -13,6 +13,7 @@
 package ai.djl.ndarray.internal;
 
 import ai.djl.ndarray.NDArray;
+import ai.djl.ndarray.NDScope;
 import ai.djl.ndarray.types.DataType;
 import ai.djl.ndarray.types.Shape;
 import ai.djl.util.Utils;
@@ -27,7 +28,7 @@ import java.util.regex.Pattern;
 public abstract class NDFormat {
 
     private static final int PRECISION = 8;
-    private static final String LF = System.getProperty("line.separator");
+    private static final String LF = System.lineSeparator();
     private static final Pattern PATTERN = Pattern.compile("\\s*\\d\\.(\\d*?)0*e[+-](\\d+)");
     private static final boolean DEBUGGER =
             !Boolean.getBoolean("jshell")
@@ -90,9 +91,7 @@ public abstract class NDFormat {
         NDFormat format;
         DataType dataType = array.getDataType();
 
-        if (dataType == DataType.UINT8) {
-            format = new HexFormat();
-        } else if (dataType == DataType.BOOLEAN) {
+        if (dataType == DataType.BOOLEAN) {
             format = new BooleanFormat();
         } else if (dataType == DataType.STRING) {
             format = new StringFormat();
@@ -126,9 +125,13 @@ public abstract class NDFormat {
             init(array);
             sb.append(format(array.toArray()[0])).append(LF);
         } else if (size > maxSize) {
-            sb.append("[ Exceed max print size ]");
+            sb.append("Exceed max print size:").append(LF);
+            int limit = Math.min(maxSize, maxRows * maxColumns);
+            dumpFlat(sb, array, limit);
         } else if (dimension > maxDepth) {
-            sb.append("[ Exceed max print dimension ]");
+            sb.append("Exceed max print dimension:").append(LF);
+            int limit = Math.min(maxSize, maxRows * maxColumns);
+            dumpFlat(sb, array, limit);
         } else {
             init(array);
             dump(sb, array, 0, true, maxRows, maxColumns);
@@ -173,6 +176,17 @@ public abstract class NDFormat {
         }
     }
 
+    @SuppressWarnings("try")
+    private void dumpFlat(StringBuilder sb, NDArray array, int limit) {
+        try (NDScope ignore = new NDScope()) {
+            NDArray tmp = array.flatten().get(":" + limit);
+            init(tmp);
+            sb.append('{');
+            append(sb, array.toArray(), limit);
+            sb.append('}').append(LF);
+        }
+    }
+
     private void append(StringBuilder sb, Number[] values, int maxColumns) {
         if (values.length == 0) {
             return;
@@ -321,15 +335,6 @@ public abstract class NDFormat {
         }
     }
 
-    private static final class HexFormat extends NDFormat {
-
-        /** {@inheritDoc} */
-        @Override
-        public CharSequence format(Number value) {
-            return String.format(Locale.ENGLISH, "0x%02X", value.byteValue());
-        }
-    }
-
     private static final class IntFormat extends NDFormat {
 
         private boolean exponential;
diff --git a/api/src/main/java/ai/djl/util/JsonUtils.java b/api/src/main/java/ai/djl/util/JsonUtils.java
index 88d6af0b1..b691737b3 100644
--- a/api/src/main/java/ai/djl/util/JsonUtils.java
+++ b/api/src/main/java/ai/djl/util/JsonUtils.java
@@ -24,8 +24,9 @@ import java.util.List;
 /** An interface containing Gson constants. */
 public interface JsonUtils {
 
-    Gson GSON = new GsonBuilder().create();
-    Gson GSON_PRETTY = builder().create();
+    boolean PRETTY_PRINT = Boolean.parseBoolean(Utils.getEnvOrSystemProperty("DJL_PRETTY_PRINT"));
+    Gson GSON = builder().create();
+    Gson GSON_PRETTY = builder().setPrettyPrinting().create();
     Type LIST_TYPE = new TypeToken<List<String>>() {}.getType();
 
     /**
@@ -34,19 +35,33 @@ public interface JsonUtils {
      * @return a custom {@code GsonBuilder} instance.
      */
     static GsonBuilder builder() {
-        return new GsonBuilder()
-                .setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
-                .setPrettyPrinting()
-                .serializeSpecialFloatingPointValues()
-                .registerTypeAdapter(
-                        Double.class,
-                        (JsonSerializer<Double>)
-                                (src, t, ctx) -> {
-                                    long v = src.longValue();
-                                    if (src.equals(Double.valueOf(String.valueOf(v)))) {
-                                        return new JsonPrimitive(v);
-                                    }
-                                    return new JsonPrimitive(src);
-                                });
+        GsonBuilder builder =
+                new GsonBuilder()
+                        .setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
+                        .serializeSpecialFloatingPointValues()
+                        .registerTypeAdapter(
+                                Double.class,
+                                (JsonSerializer<Double>)
+                                        (src, t, ctx) -> {
+                                            long v = src.longValue();
+                                            if (src.equals(Double.valueOf(String.valueOf(v)))) {
+                                                return new JsonPrimitive(v);
+                                            }
+                                            return new JsonPrimitive(src);
+                                        });
+        if (PRETTY_PRINT) {
+            builder.setPrettyPrinting();
+        }
+        return builder;
+    }
+
+    /**
+     * Serializes the specified object into its equivalent JSON representation.
+     *
+     * @param src the source object
+     * @return the json string
+     */
+    static String toJson(Object src) {
+        return GSON.toJson(src);
     }
 }
diff --git a/basicdataset/src/main/java/ai/djl/basicdataset/nlp/StanfordQuestionAnsweringDataset.java b/basicdataset/src/main/java/ai/djl/basicdataset/nlp/StanfordQuestionAnsweringDataset.java
index 9912fe877..053395237 100644
--- a/basicdataset/src/main/java/ai/djl/basicdataset/nlp/StanfordQuestionAnsweringDataset.java
+++ b/basicdataset/src/main/java/ai/djl/basicdataset/nlp/StanfordQuestionAnsweringDataset.java
@@ -115,7 +115,7 @@ public class StanfordQuestionAnsweringDataset extends TextDataset implements Raw
         Map<String, Object> data;
         try (BufferedReader reader = Files.newBufferedReader(usagePath)) {
             data =
-                    JsonUtils.GSON_PRETTY.fromJson(
+                    JsonUtils.GSON.fromJson(
                             reader, new TypeToken<Map<String, Object>>() {}.getType());
         }
         List<Map<String, Object>> articles = (List<Map<String, Object>>) data.get("data");
@@ -227,7 +227,7 @@ public class StanfordQuestionAnsweringDataset extends TextDataset implements Raw
         Path usagePath = prepareUsagePath(null);
         Object data;
         try (BufferedReader reader = Files.newBufferedReader(usagePath)) {
-            data = JsonUtils.GSON_PRETTY.fromJson(reader, new TypeToken<Object>() {}.getType());
+            data = JsonUtils.GSON.fromJson(reader, new TypeToken<Object>() {}.getType());
         }
         return data;
     }
diff --git a/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/nlp/embedding/GloveWordEmbeddingTranslatorFactory.java b/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/nlp/embedding/GloveWordEmbeddingTranslatorFactory.java
index de9cb4ffc..a5d6213c1 100644
--- a/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/nlp/embedding/GloveWordEmbeddingTranslatorFactory.java
+++ b/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/nlp/embedding/GloveWordEmbeddingTranslatorFactory.java
@@ -70,6 +70,7 @@ public class GloveWordEmbeddingTranslatorFactory implements TranslatorFactory {
         /** {@inheritDoc} */
         @Override
         public NDList processOutput(TranslatorContext ctx, NDList list) {
+            list.detach();
             return list;
         }
 
diff --git a/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java b/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java
index abb820cce..eacfe74c8 100644
--- a/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java
+++ b/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java
@@ -33,6 +33,12 @@ public class PtModelZoo extends ModelZoo {
     public static final String GROUP_ID = "ai.djl.pytorch";
 
     PtModelZoo() {
+        addModel(
+                REPOSITORY.model(
+                        CV.ACTION_RECOGNITION,
+                        GROUP_ID,
+                        "Human-Action-Recognition-VIT-Base-patch16-224",
+                        "0.0.1"));
         addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1"));
         addModel(
                 REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet18_embedding", "0.0.1"));
diff --git a/examples/src/main/java/ai/djl/examples/inference/cv/ActionRecognition.java b/examples/src/main/java/ai/djl/examples/inference/cv/ActionRecognition.java
index 5530d44f1..a069ed358 100644
--- a/examples/src/main/java/ai/djl/examples/inference/cv/ActionRecognition.java
+++ b/examples/src/main/java/ai/djl/examples/inference/cv/ActionRecognition.java
@@ -26,8 +26,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
 
 /**
  * An example of inference using an action recognition model.
@@ -48,16 +46,16 @@ public final class ActionRecognition {
     }
 
     public static Classifications predict() throws IOException, ModelException, TranslateException {
-        Path imageFile = Paths.get("src/test/resources/action_discus_throw.png");
-        Image img = ImageFactory.getInstance().fromFile(imageFile);
+        String url = "https://resources.djl.ai/images/action_dance.jpg";
+        Image img = ImageFactory.getInstance().fromUrl(url);
 
-        // Use DJL MXNet model zoo model
+        // Use DJL PyTorch model zoo model
         Criteria<Image, Classifications> criteria =
                 Criteria.builder()
                         .setTypes(Image.class, Classifications.class)
                         .optModelUrls(
-                                "djl://ai.djl.mxnet/action_recognition/0.0.1/inceptionv3_ucf101")
-                        .optEngine("MXNet")
+                                "djl://ai.djl.pytorch/Human-Action-Recognition-VIT-Base-patch16-224")
+                        .optEngine("PyTorch")
                         .optProgress(new ProgressBar())
                         .build();
 


Test patch:
diff --git a/api/src/test/java/ai/djl/ndarray/internal/NDFormatTest.java b/api/src/test/java/ai/djl/ndarray/internal/NDFormatTest.java
index cea775eba..6e1fe6453 100644
--- a/api/src/test/java/ai/djl/ndarray/internal/NDFormatTest.java
+++ b/api/src/test/java/ai/djl/ndarray/internal/NDFormatTest.java
@@ -23,7 +23,7 @@ import org.testng.annotations.Test;
 
 public class NDFormatTest {
 
-    private static final String LF = System.getProperty("line.separator");
+    private static final String LF = System.lineSeparator();
 
     @Test
     public void testUint8Format() {
@@ -33,7 +33,7 @@ public class NDFormatTest {
             array.set(data);
 
             String str = NDFormat.format(array, 100, 10, 10, 20);
-            Assert.assertEquals(str, "ND: (3) cpu() uint8" + LF + "[0x7F, 0x80, 0x01]" + LF);
+            Assert.assertEquals(str, "ND: (3) cpu() uint8" + LF + "[127, 128,   1]" + LF);
         }
     }
 
@@ -142,9 +142,11 @@ public class NDFormatTest {
         try (NDManager manager = NDManager.newBaseManager(Device.cpu())) {
             NDArray large = manager.arange(0f, 100000000f).reshape(100, 100, 100, 100);
             String str = large.toString();
-            Assert.assertEquals(
-                    str,
-                    "ND: (100, 100, 100, 100) cpu() float32" + LF + "[ Exceed max print size ]");
+            Assert.assertTrue(
+                    str.startsWith(
+                            "ND: (100, 100, 100, 100) cpu() float32"
+                                    + LF
+                                    + "Exceed max print size:"));
 
             NDArray arr = manager.create(new float[] {1, 0, 0, 0});
             Assert.assertEquals(arr.mean().toString(), "ND: () cpu() float32" + LF + "0.25" + LF);
diff --git a/engines/pytorch/pytorch-model-zoo/src/test/resources/mlrepo/model/cv/action_recognition/ai/djl/pytorch/Human-Action-Recognition-VIT-Base-patch16-224/metadata.json b/engines/pytorch/pytorch-model-zoo/src/test/resources/mlrepo/model/cv/action_recognition/ai/djl/pytorch/Human-Action-Recognition-VIT-Base-patch16-224/metadata.json
new file mode 100644
index 000000000..71fa5bc3c
--- /dev/null
+++ b/engines/pytorch/pytorch-model-zoo/src/test/resources/mlrepo/model/cv/action_recognition/ai/djl/pytorch/Human-Action-Recognition-VIT-Base-patch16-224/metadata.json
@@ -0,0 +1,45 @@
+{
+  "metadataVersion": "0.2",
+  "resourceType": "model",
+  "application": "cv/action_recognition",
+  "groupId": "ai.djl.pytorch",
+  "artifactId": "Human-Action-Recognition-VIT-Base-patch16-224",
+  "name": "Human-Action-Recognition-VIT-Base-patch16-224",
+  "description": "Action Recognition models",
+  "website": "http://www.djl.ai/engines/pytorch/model-zoo",
+  "licenses": {
+    "license": {
+      "name": "The Apache License, Version 2.0",
+      "url": "https://www.apache.org/licenses/LICENSE-2.0"
+    }
+  },
+  "artifacts": [
+    {
+      "version": "0.0.1",
+      "snapshot": false,
+      "name": "Human-Action-Recognition-VIT-Base-patch16-224",
+      "properties": {
+      },
+      "arguments": {
+        "width": 224,
+        "height": 224,
+        "resize": true,
+        "scaleFactor": 0.00392156862745098,
+        "normalize": "0.5,0.5,0.5,0.5,0.5,0.5",
+        "applySoftmax": true,
+        "translatorFactory": "ai.djl.modality.cv.translator.ImageClassificationTranslatorFactory"
+      },
+      "options": {
+        "mapLocation": "true"
+      },
+      "files": {
+        "model": {
+          "uri": "0.0.1/Human-Action-Recognition-VIT-Base-patch16-224.zip",
+          "sha1Hash": "943cf0f9cfab07445489aeb67b9c74c9c85680f4",
+          "name": "",
+          "size": 318499747
+        }
+      }
+    }
+  ]
+}
diff --git a/examples/src/test/java/ai/djl/examples/inference/ActionRecognitionTest.java b/examples/src/test/java/ai/djl/examples/inference/ActionRecognitionTest.java
index 0534c2a4f..63b9bbc2b 100644
--- a/examples/src/test/java/ai/djl/examples/inference/ActionRecognitionTest.java
+++ b/examples/src/test/java/ai/djl/examples/inference/ActionRecognitionTest.java
@@ -15,7 +15,6 @@ package ai.djl.examples.inference;
 import ai.djl.ModelException;
 import ai.djl.examples.inference.cv.ActionRecognition;
 import ai.djl.modality.Classifications;
-import ai.djl.testing.TestRequirements;
 import ai.djl.translate.TranslateException;
 
 import org.testng.Assert;
@@ -27,11 +26,9 @@ public class ActionRecognitionTest {
 
     @Test
     public void testActionRecognition() throws ModelException, TranslateException, IOException {
-        TestRequirements.linux();
-
         Classifications result = ActionRecognition.predict();
         Classifications.Classification best = result.best();
-        Assert.assertEquals(best.getClassName(), "ThrowDiscus");
+        Assert.assertEquals(best.getClassName(), "Dancing");
         Assert.assertTrue(Double.compare(best.getProbability(), 0.9) > 0);
     }
 }
diff --git a/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java b/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java
index ec149df38..4605702a1 100644
--- a/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java
+++ b/extensions/tokenizers/src/test/java/ai/djl/huggingface/zoo/ModelZooTest.java
@@ -84,7 +84,7 @@ public class ModelZooTest {
             Files.createDirectories(dir);
             Path file = dir.resolve("models.json");
             try (Writer writer = Files.newBufferedWriter(file)) {
-                writer.write(JsonUtils.GSON_PRETTY.toJson(map));
+                writer.write(JsonUtils.toJson(map));
             }
 
             // static variables cannot not be initialized properly if directly use new HfModelZoo()