Repository: deepjavalibrary/djl
Pull request #1981 (2022-08-31 16:58:06)

add tokenizer readme for usage
## Description ##

Add readme to instruct how to use tokenizer



Patch:
diff --git a/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java b/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java
index 4a14cb7c5..8abb4ca44 100644
--- a/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java
+++ b/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java
@@ -33,6 +33,8 @@ import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 class RepositoryFactoryImpl implements RepositoryFactory {
 
@@ -40,6 +42,7 @@ class RepositoryFactoryImpl implements RepositoryFactory {
 
     private static final RepositoryFactory FACTORY = new RepositoryFactoryImpl();
     private static final Map<String, RepositoryFactory> REGISTRY = init();
+    private static final Pattern PATTERN = Pattern.compile("(.+)/([\\d.]+)(/(.*))?");
 
     static RepositoryFactory getFactory() {
         return FACTORY;
@@ -217,17 +220,21 @@ class RepositoryFactoryImpl implements RepositoryFactory {
             if (groupId == null) {
                 throw new IllegalArgumentException("Invalid djl URL: " + uri);
             }
-            Path path = parseFilePath(uri);
-            int size = path.getNameCount();
-            if (size == 0) {
-                throw new IllegalArgumentException("Invalid djl URL: " + uri);
+            String artifactId = parseFilePath(uri).toString();
+            if (artifactId.startsWith("/")) {
+                artifactId = artifactId.substring(1);
             }
-            String artifactId = path.getName(0).toString();
             if (artifactId.isEmpty()) {
                 throw new IllegalArgumentException("Invalid djl URL: " + uri);
             }
-            String version = size > 1 ? path.getName(1).toString() : null;
-            String artifactName = size > 2 ? path.getName(2).toString() : null;
+            String version = null;
+            String artifactName = null;
+            Matcher m = PATTERN.matcher(artifactId);
+            if (m.matches()) {
+                artifactId = m.group(1);
+                version = m.group(2);
+                artifactName = m.group(4);
+            }
 
             ModelZoo zoo = ModelZoo.getModelZoo(groupId);
             if (zoo == null) {
diff --git a/api/src/main/java/ai/djl/repository/zoo/DefaultModelZoo.java b/api/src/main/java/ai/djl/repository/zoo/DefaultModelZoo.java
index 17c259dd2..527871067 100644
--- a/api/src/main/java/ai/djl/repository/zoo/DefaultModelZoo.java
+++ b/api/src/main/java/ai/djl/repository/zoo/DefaultModelZoo.java
@@ -19,9 +19,6 @@ import ai.djl.repository.Repository;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** A {@link ModelZoo} that contains models in specified locations. */
@@ -31,10 +28,13 @@ public class DefaultModelZoo extends ModelZoo {
 
     private static final Logger logger = LoggerFactory.getLogger(DefaultModelZoo.class);
 
-    private List<ModelLoader> modelLoaders;
-
     /** Constructs a new {@code LocalModelZoo} instance. */
-    public DefaultModelZoo() {}
+    public DefaultModelZoo() {
+        String locations = System.getProperty("ai.djl.repository.zoo.location");
+        if (locations != null) {
+            parseLocation(locations);
+        }
+    }
 
     /**
      * Constructs a new {@code LocalModelZoo} instance from the given search locations.
@@ -42,20 +42,7 @@ public class DefaultModelZoo extends ModelZoo {
      * @param locations a comma separated urls where the models to be loaded from
      */
     public DefaultModelZoo(String locations) {
-        modelLoaders = parseLocation(locations);
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        if (modelLoaders != null) {
-            return modelLoaders;
-        }
-        String locations = System.getProperty("ai.djl.repository.zoo.location");
-        if (locations != null) {
-            return parseLocation(locations);
-        }
-        return Collections.emptyList();
+        parseLocation(locations);
     }
 
     /** {@inheritDoc} */
@@ -70,21 +57,18 @@ public class DefaultModelZoo extends ModelZoo {
         return Engine.getAllEngines();
     }
 
-    private List<ModelLoader> parseLocation(String locations) {
+    private void parseLocation(String locations) {
         String[] urls = locations.split("\\s*,\\s*");
-        List<ModelLoader> list = new ArrayList<>(urls.length);
         for (String url : urls) {
             if (!url.isEmpty()) {
                 Repository repo = Repository.newInstance(url, url);
                 logger.debug("Scanning models in repo: {}, {}", repo.getClass(), url);
-                List<MRL> mrls = repo.getResources();
-                for (MRL mrl : mrls) {
-                    list.add(new BaseModelLoader(mrl));
+                for (MRL mrl : repo.getResources()) {
+                    addModel(mrl);
                 }
             } else {
                 logger.warn("Model location is empty.");
             }
         }
-        return list;
     }
 }
diff --git a/api/src/main/java/ai/djl/repository/zoo/ModelZoo.java b/api/src/main/java/ai/djl/repository/zoo/ModelZoo.java
index 0674c9d00..50b219be5 100644
--- a/api/src/main/java/ai/djl/repository/zoo/ModelZoo.java
+++ b/api/src/main/java/ai/djl/repository/zoo/ModelZoo.java
@@ -15,6 +15,7 @@ package ai.djl.repository.zoo;
 import ai.djl.Application;
 import ai.djl.MalformedModelException;
 import ai.djl.repository.Artifact;
+import ai.djl.repository.MRL;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -32,6 +33,8 @@ public abstract class ModelZoo {
 
     private static final Map<String, ModelZoo> MODEL_ZOO_MAP = new ConcurrentHashMap<>();
 
+    private Map<String, ModelLoader> modelLoaders = new ConcurrentHashMap<>();
+
     static {
         ServiceLoader<ZooProvider> providers = ServiceLoader.load(ZooProvider.class);
         for (ZooProvider provider : providers) {
@@ -54,7 +57,9 @@ public abstract class ModelZoo {
      *
      * @return the list of all available model families
      */
-    public abstract List<ModelLoader> getModelLoaders();
+    public Collection<ModelLoader> getModelLoaders() {
+        return modelLoaders.values();
+    }
 
     /**
      * Returns the {@link ModelLoader} based on the model name.
@@ -63,12 +68,7 @@ public abstract class ModelZoo {
      * @return the {@link ModelLoader} of the model
      */
     public ModelLoader getModelLoader(String name) {
-        for (ModelLoader loader : getModelLoaders()) {
-            if (name.equals(loader.getArtifactId())) {
-                return loader;
-            }
-        }
-        return null;
+        return modelLoaders.get(name);
     }
 
     /**
@@ -78,6 +78,14 @@ public abstract class ModelZoo {
      */
     public abstract Set<String> getSupportedEngines();
 
+    protected final void addModel(MRL mrl) {
+        modelLoaders.put(mrl.getArtifactId(), new BaseModelLoader(mrl));
+    }
+
+    protected final void addModel(ModelLoader loader) {
+        modelLoaders.put(loader.getArtifactId(), loader);
+    }
+
     /**
      * Refreshes model zoo.
      *
@@ -85,7 +93,7 @@ public abstract class ModelZoo {
      */
     public static void registerModelZoo(ZooProvider provider) {
         ModelZoo zoo = provider.getModelZoo();
-        MODEL_ZOO_MAP.putIfAbsent(zoo.getGroupId(), zoo);
+        MODEL_ZOO_MAP.put(zoo.getGroupId(), zoo);
     }
 
     /**
@@ -174,8 +182,7 @@ public abstract class ModelZoo {
                     continue;
                 }
             }
-            List<ModelLoader> list = zoo.getModelLoaders();
-            for (ModelLoader loader : list) {
+            for (ModelLoader loader : zoo.getModelLoaders()) {
                 Application app = loader.getApplication();
                 String loaderArtifactId = loader.getArtifactId();
                 if (artifactId != null && !artifactId.equals(loaderArtifactId)) {
diff --git a/engines/dlr/dlr-engine/src/main/java/ai/djl/dlr/zoo/DlrModelZoo.java b/engines/dlr/dlr-engine/src/main/java/ai/djl/dlr/zoo/DlrModelZoo.java
index 897a04dbf..a4b8f0944 100644
--- a/engines/dlr/dlr-engine/src/main/java/ai/djl/dlr/zoo/DlrModelZoo.java
+++ b/engines/dlr/dlr-engine/src/main/java/ai/djl/dlr/zoo/DlrModelZoo.java
@@ -14,15 +14,10 @@ package ai.djl.dlr.zoo;
 
 import ai.djl.Application.CV;
 import ai.djl.dlr.engine.DlrEngine;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** DlrModelZoo is a repository that contains all dlr models for DJL. */
@@ -32,17 +27,8 @@ public class DlrModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("Dlr", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.dlr";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL resnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(resnet));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    DlrModelZoo() {
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1"));
     }
 
     /** {@inheritDoc} */
diff --git a/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/MxModelZoo.java b/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/MxModelZoo.java
index 162d4b994..30f7884e0 100644
--- a/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/MxModelZoo.java
+++ b/engines/mxnet/mxnet-model-zoo/src/main/java/ai/djl/mxnet/zoo/MxModelZoo.java
@@ -15,15 +15,10 @@ package ai.djl.mxnet.zoo;
 import ai.djl.Application.CV;
 import ai.djl.Application.NLP;
 import ai.djl.mxnet.engine.MxEngine;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /**
@@ -36,79 +31,28 @@ public class MxModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("MXNet", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.mxnet";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL ssd = REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(ssd));
-
-        MRL yolo = REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "yolo", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(yolo));
-
-        MRL alexnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "alexnet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(alexnet));
-
-        MRL darknet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "darknet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(darknet));
-
-        MRL densenet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "densenet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(densenet));
-
-        MRL googlenet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "googlenet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(googlenet));
-
-        MRL inceptionv3 =
-                REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "inceptionv3", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(inceptionv3));
-
-        MRL mlp = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mlp", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(mlp));
-
-        MRL mobilenet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mobilenet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(mobilenet));
-
-        MRL resnest = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnest", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(resnest));
-
-        MRL resnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(resnet));
-
-        MRL senet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "senet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(senet));
-
-        MRL seresnext = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "se_resnext", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(seresnext));
-
-        MRL squeezenet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "squeezenet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(squeezenet));
-
-        MRL vgg = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "vgg", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(vgg));
-
-        MRL xception = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "xception", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(xception));
-
-        MRL simplePose = REPOSITORY.model(CV.POSE_ESTIMATION, GROUP_ID, "simple_pose", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(simplePose));
-
-        MRL maskrcnn = REPOSITORY.model(CV.INSTANCE_SEGMENTATION, GROUP_ID, "mask_rcnn", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(maskrcnn));
-
-        MRL actionRecognition =
-                REPOSITORY.model(CV.ACTION_RECOGNITION, GROUP_ID, "action_recognition", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(actionRecognition));
-
-        MRL bertQa = REPOSITORY.model(NLP.QUESTION_ANSWER, GROUP_ID, "bertqa", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(bertQa));
-
-        MRL glove = REPOSITORY.model(NLP.WORD_EMBEDDING, GROUP_ID, "glove", "0.0.2");
-        MODEL_LOADERS.add(new BaseModelLoader(glove));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    MxModelZoo() {
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.1"));
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "yolo", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "alexnet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "darknet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "densenet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "googlenet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "inceptionv3", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mlp", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mobilenet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnest", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "senet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "se_resnext", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "squeezenet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "vgg", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "xception", "0.0.1"));
+        addModel(REPOSITORY.model(CV.POSE_ESTIMATION, GROUP_ID, "simple_pose", "0.0.1"));
+        addModel(REPOSITORY.model(CV.INSTANCE_SEGMENTATION, GROUP_ID, "mask_rcnn", "0.0.1"));
+        addModel(REPOSITORY.model(CV.ACTION_RECOGNITION, GROUP_ID, "action_recognition", "0.0.1"));
+        addModel(REPOSITORY.model(NLP.QUESTION_ANSWER, GROUP_ID, "bertqa", "0.0.1"));
+        addModel(REPOSITORY.model(NLP.WORD_EMBEDDING, GROUP_ID, "glove", "0.0.2"));
     }
 
     /** {@inheritDoc} */
diff --git a/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/zoo/OrtModelZoo.java b/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/zoo/OrtModelZoo.java
index 0421a5f96..d4d27b9ed 100644
--- a/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/zoo/OrtModelZoo.java
+++ b/engines/onnxruntime/onnxruntime-engine/src/main/java/ai/djl/onnxruntime/zoo/OrtModelZoo.java
@@ -15,15 +15,10 @@ package ai.djl.onnxruntime.zoo;
 import ai.djl.Application.CV;
 import ai.djl.Application.Tabular;
 import ai.djl.onnxruntime.engine.OrtEngine;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** OrtModelZoo is a repository that contains all Onnx models for DJL. */
@@ -33,21 +28,9 @@ public class OrtModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("Ort", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.onnxruntime";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL resnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(resnet));
-
-        MRL irisFlower =
-                REPOSITORY.model(Tabular.SOFTMAX_REGRESSION, GROUP_ID, "iris_flowers", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(irisFlower));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    OrtModelZoo() {
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1"));
+        addModel(REPOSITORY.model(Tabular.SOFTMAX_REGRESSION, GROUP_ID, "iris_flowers", "0.0.1"));
     }
 
     /** {@inheritDoc} */
diff --git a/engines/paddlepaddle/paddlepaddle-model-zoo/src/main/java/ai/djl/paddlepaddle/zoo/PpModelZoo.java b/engines/paddlepaddle/paddlepaddle-model-zoo/src/main/java/ai/djl/paddlepaddle/zoo/PpModelZoo.java
index 22af1f070..55d392b57 100644
--- a/engines/paddlepaddle/paddlepaddle-model-zoo/src/main/java/ai/djl/paddlepaddle/zoo/PpModelZoo.java
+++ b/engines/paddlepaddle/paddlepaddle-model-zoo/src/main/java/ai/djl/paddlepaddle/zoo/PpModelZoo.java
@@ -14,15 +14,10 @@ package ai.djl.paddlepaddle.zoo;
 
 import ai.djl.Application.CV;
 import ai.djl.paddlepaddle.engine.PpEngine;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** PpModelZoo is a repository that contains all PaddlePaddle models for DJL. */
@@ -32,34 +27,14 @@ public class PpModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("Paddle", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.paddlepaddle";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL maskDetection =
-                REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mask_classification", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(maskDetection));
-
-        MRL wordRotation =
-                REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "word_rotation", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(wordRotation));
-
-        MRL faceDetection =
-                REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "face_detection", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(faceDetection));
-
-        MRL wordDetection =
-                REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "word_detection", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(wordDetection));
-
-        MRL wordRecognition =
-                REPOSITORY.model(CV.WORD_RECOGNITION, GROUP_ID, "word_recognition", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(wordRecognition));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    PpModelZoo() {
+        addModel(
+                REPOSITORY.model(
+                        CV.IMAGE_CLASSIFICATION, GROUP_ID, "mask_classification", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "word_rotation", "0.0.1"));
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "face_detection", "0.0.1"));
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "word_detection", "0.0.1"));
+        addModel(REPOSITORY.model(CV.WORD_RECOGNITION, GROUP_ID, "word_recognition", "0.0.1"));
     }
 
     /** {@inheritDoc} */
diff --git a/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java b/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java
index e2cb7f790..c53aeedd5 100644
--- a/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java
+++ b/engines/pytorch/pytorch-model-zoo/src/main/java/ai/djl/pytorch/zoo/PtModelZoo.java
@@ -15,15 +15,10 @@ package ai.djl.pytorch.zoo;
 import ai.djl.Application.CV;
 import ai.djl.Application.NLP;
 import ai.djl.pytorch.engine.PtEngine;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /**
@@ -36,36 +31,14 @@ public class PtModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("PyTorch", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.pytorch";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL resnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(resnet));
-
-        MRL ssd = REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(ssd));
-
-        MRL bertQa = REPOSITORY.model(NLP.QUESTION_ANSWER, GROUP_ID, "bertqa", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(bertQa));
-
-        MRL sentimentAnalysis =
-                REPOSITORY.model(NLP.SENTIMENT_ANALYSIS, GROUP_ID, "distilbert", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(sentimentAnalysis));
-
-        MRL bigGan = REPOSITORY.model(CV.IMAGE_GENERATION, GROUP_ID, "biggan-deep", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(bigGan));
-
-        MRL cyclegan = REPOSITORY.model(CV.IMAGE_GENERATION, GROUP_ID, "cyclegan", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(cyclegan));
-
-        MRL deeplabv3 = REPOSITORY.model(CV.SEMANTIC_SEGMENTATION, GROUP_ID, "deeplabv3", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(deeplabv3));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    PtModelZoo() {
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.1"));
+        addModel(REPOSITORY.model(NLP.QUESTION_ANSWER, GROUP_ID, "bertqa", "0.0.1"));
+        addModel(REPOSITORY.model(NLP.SENTIMENT_ANALYSIS, GROUP_ID, "distilbert", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_GENERATION, GROUP_ID, "biggan-deep", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_GENERATION, GROUP_ID, "cyclegan", "0.0.1"));
+        addModel(REPOSITORY.model(CV.SEMANTIC_SEGMENTATION, GROUP_ID, "deeplabv3", "0.0.1"));
     }
 
     /** {@inheritDoc} */
diff --git a/engines/tensorflow/tensorflow-model-zoo/src/main/java/ai/djl/tensorflow/zoo/TfModelZoo.java b/engines/tensorflow/tensorflow-model-zoo/src/main/java/ai/djl/tensorflow/zoo/TfModelZoo.java
index ac27a9e3b..4a54be6c0 100644
--- a/engines/tensorflow/tensorflow-model-zoo/src/main/java/ai/djl/tensorflow/zoo/TfModelZoo.java
+++ b/engines/tensorflow/tensorflow-model-zoo/src/main/java/ai/djl/tensorflow/zoo/TfModelZoo.java
@@ -14,16 +14,11 @@
 package ai.djl.tensorflow.zoo;
 
 import ai.djl.Application.CV;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 import ai.djl.tensorflow.engine.TfEngine;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** TfModelZoo is a repository that contains the TensorFlow models for DJL. */
@@ -33,23 +28,10 @@ public class TfModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("TensorFlow", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.tensorflow";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL resnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(resnet));
-
-        MRL mobilenet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mobilenet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(mobilenet));
-
-        MRL ssd = REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(ssd));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    TfModelZoo() {
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mobilenet", "0.0.1"));
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.1"));
     }
 
     /** {@inheritDoc} */
diff --git a/engines/tflite/tflite-engine/src/main/java/ai/djl/tflite/zoo/TfLiteModelZoo.java b/engines/tflite/tflite-engine/src/main/java/ai/djl/tflite/zoo/TfLiteModelZoo.java
index 6c1ec7875..48614b915 100644
--- a/engines/tflite/tflite-engine/src/main/java/ai/djl/tflite/zoo/TfLiteModelZoo.java
+++ b/engines/tflite/tflite-engine/src/main/java/ai/djl/tflite/zoo/TfLiteModelZoo.java
@@ -13,16 +13,11 @@
 package ai.djl.tflite.zoo;
 
 import ai.djl.Application.CV;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 import ai.djl.tflite.engine.TfLiteEngine;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** TfLiteModelZoo is a repository that contains all TFLite models for DJL. */
@@ -32,17 +27,8 @@ public class TfLiteModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("TFLite", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.tflite";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL mobilenet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mobilenet", "0.0.1");
-        MODEL_LOADERS.add(new BaseModelLoader(mobilenet));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    TfLiteModelZoo() {
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mobilenet", "0.0.1"));
     }
 
     /** {@inheritDoc} */
diff --git a/extensions/fasttext/src/main/java/ai/djl/fasttext/zoo/FtModelZoo.java b/extensions/fasttext/src/main/java/ai/djl/fasttext/zoo/FtModelZoo.java
index a44ba6700..29e6077d3 100644
--- a/extensions/fasttext/src/main/java/ai/djl/fasttext/zoo/FtModelZoo.java
+++ b/extensions/fasttext/src/main/java/ai/djl/fasttext/zoo/FtModelZoo.java
@@ -15,12 +15,9 @@ package ai.djl.fasttext.zoo;
 import ai.djl.engine.Engine;
 import ai.djl.fasttext.zoo.nlp.textclassification.TextClassificationModelLoader;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.Collections;
-import java.util.List;
 import java.util.Set;
 
 /** FtModelZoo is a repository that contains all fastText models for DJL. */
@@ -30,16 +27,8 @@ public class FtModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("Fasttext", DJL_REPO_URL);
     public static final String GROUP_ID = "ai.djl.fasttext";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MODEL_LOADERS.add(new TextClassificationModelLoader(REPOSITORY));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    FtModelZoo() {
+        addModel(new TextClassificationModelLoader(REPOSITORY));
     }
 
     /** {@inheritDoc} */
diff --git a/extensions/tokenizers/README.md b/extensions/tokenizers/README.md
index 38cc8529a..d10ea87a0 100644
--- a/extensions/tokenizers/README.md
+++ b/extensions/tokenizers/README.md
@@ -24,3 +24,83 @@ You can pull the module from the central Maven repository by including the follo
     <version>0.18.0</version>
 </dependency>
 ```
+
+## Usage
+
+### Use DJL HuggingFace model converter (experimental)
+
+If you are trying to convert a complete HuggingFace (transformers) model,
+you can try to use our all-in-one conversion solution to convert to Java:
+
+```bash
+python3 -m pip install -r src/main/python/requirements.txt
+python3 src/main/python/model_zoo_importer.py -m deepset/bert-base-cased-squad2
+```
+
+This will generate a zip file into your local folder:
+
+```
+model/nlp/question_answer/ai/djl/huggingface/pytorch/deepset/bert-base-cased-squad2/0.0.1/bert-base-cased-squad2.zip
+```
+
+Then, all you need to do, is to load this model in DJL:
+
+```java
+Criteria<QAInput, String> criteria = Criteria.builder()
+    .setTypes(QAInput.class, String.class)
+    .optModelPath(Paths.get("model/nlp/question_answer/ai/djl/huggingface/pytorch/deepset/bert-base-cased-squad2/0.0.1/bert-base-cased-squad2.zip"))
+    .optProgress(new ProgressBar()).build();
+```
+
+Currently, this converter support:
+
+- fill-mask
+- question-answering
+- sentence-similarity
+- text-classification
+- token-classification
+
+### From HuggingFace AutoTokenizer
+In most of the cases, you can easily use a pre-existing tokenizer in DJL:
+
+Python
+
+```python
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/msmarco-distilbert-dot-v5")
+```
+
+Java
+
+```java
+HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance("sentence-transformers/msmarco-distilbert-dot-v5")
+```
+
+This way requires network connection to huggingface repo.
+The way to determine if you can use this way is through looking into the "Files and versions" in [HuggingFace model tab](https://huggingface.co/sentence-transformers/msmarco-distilbert-dot-v5)
+and see if there is a `tokenizer.json`. 
+
+If there is a `tokenizer.json`, you can get it directly through DJL. Otherwise, use the other way below to obtain a `tokenizer.json`.
+
+### From HuggingFace Pipeline
+
+If you are trying to get tokenizer from a HuggingFace pipeline,
+you can use the followings to extract `tokenizer.json` file.
+
+Python
+
+```python
+ pipeline.tokenizer.save_pretrained("./")
+```
+
+From your local directory, you will find a `tokenizer.json` file.
+
+Java
+
+```java
+HuggingFaceTokenizer tokenizer = HuggingFaceTokenizer.newInstance(Paths.get("./tokenizer.json"))
+```
+
+### From pretrained json file
+
+Same as above step, just save your tokenizer into `tokenizer.json` (done by huggingface).
diff --git a/model-zoo/src/main/java/ai/djl/basicmodelzoo/BasicModelZoo.java b/model-zoo/src/main/java/ai/djl/basicmodelzoo/BasicModelZoo.java
index 34b2ab8a6..7cdfc040c 100644
--- a/model-zoo/src/main/java/ai/djl/basicmodelzoo/BasicModelZoo.java
+++ b/model-zoo/src/main/java/ai/djl/basicmodelzoo/BasicModelZoo.java
@@ -13,15 +13,10 @@
 package ai.djl.basicmodelzoo;
 
 import ai.djl.Application.CV;
-import ai.djl.repository.MRL;
 import ai.djl.repository.Repository;
-import ai.djl.repository.zoo.BaseModelLoader;
-import ai.djl.repository.zoo.ModelLoader;
 import ai.djl.repository.zoo.ModelZoo;
 
-import java.util.ArrayList;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Set;
 
 /** Model Zoo is a repository that contains all models for DJL. */
@@ -31,23 +26,10 @@ public class BasicModelZoo extends ModelZoo {
     private static final Repository REPOSITORY = Repository.newInstance("zoo", REPO_URL);
     public static final String GROUP_ID = "ai.djl.zoo";
 
-    private static final List<ModelLoader> MODEL_LOADERS = new ArrayList<>();
-
-    static {
-        MRL mlp = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mlp", "0.0.3");
-        MODEL_LOADERS.add(new BaseModelLoader(mlp));
-
-        MRL resnet = REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.2");
-        MODEL_LOADERS.add(new BaseModelLoader(resnet));
-
-        MRL ssd = REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.2");
-        MODEL_LOADERS.add(new BaseModelLoader(ssd));
-    }
-
-    /** {@inheritDoc} */
-    @Override
-    public List<ModelLoader> getModelLoaders() {
-        return MODEL_LOADERS;
+    BasicModelZoo() {
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "mlp", "0.0.3"));
+        addModel(REPOSITORY.model(CV.IMAGE_CLASSIFICATION, GROUP_ID, "resnet", "0.0.2"));
+        addModel(REPOSITORY.model(CV.OBJECT_DETECTION, GROUP_ID, "ssd", "0.0.2"));
     }
 
     /** {@inheritDoc} */


Test patch:
diff --git a/examples/src/test/java/ai/djl/examples/inference/InstanceSegmentationTest.java b/examples/src/test/java/ai/djl/examples/inference/InstanceSegmentationTest.java
index 68a4b0c96..33812b409 100644
--- a/examples/src/test/java/ai/djl/examples/inference/InstanceSegmentationTest.java
+++ b/examples/src/test/java/ai/djl/examples/inference/InstanceSegmentationTest.java
@@ -28,6 +28,7 @@ public class InstanceSegmentationTest {
     @Test
     public void testInstanceSegmentation() throws ModelException, TranslateException, IOException {
         TestRequirements.engine("MXNet");
+        TestRequirements.nightly();
 
         DetectedObjects result = InstanceSegmentation.predict();
         Classifications.Classification best = result.best();
diff --git a/examples/src/test/java/ai/djl/examples/inference/ListModelsTest.java b/examples/src/test/java/ai/djl/examples/inference/ListModelsTest.java
index a8266c2b1..2a0c58723 100644
--- a/examples/src/test/java/ai/djl/examples/inference/ListModelsTest.java
+++ b/examples/src/test/java/ai/djl/examples/inference/ListModelsTest.java
@@ -18,6 +18,7 @@ import ai.djl.Application.NLP;
 import ai.djl.ModelException;
 import ai.djl.repository.Artifact;
 import ai.djl.repository.zoo.Criteria;
+import ai.djl.repository.zoo.DefaultZooProvider;
 import ai.djl.repository.zoo.ModelZoo;
 
 import org.testng.Assert;
@@ -36,6 +37,7 @@ public class ListModelsTest {
     @AfterClass
     public void tearDown() {
         System.clearProperty("ai.djl.repository.zoo.location");
+        ModelZoo.registerModelZoo(new DefaultZooProvider());
     }
 
     @Test
@@ -43,6 +45,7 @@ public class ListModelsTest {
         Path path = Paths.get("../model-zoo/src/test/resources/mlrepo");
         String repoUrl = path.toRealPath().toAbsolutePath().toUri().toURL().toExternalForm();
         System.setProperty("ai.djl.repository.zoo.location", "src/test/resources," + repoUrl);
+        ModelZoo.registerModelZoo(new DefaultZooProvider());
         Map<Application, List<Artifact>> models = ModelZoo.listModels();
         System.out.println(Arrays.toString(models.keySet().toArray()));
         List<Artifact> artifacts = models.get(Application.UNDEFINED);
@@ -54,6 +57,7 @@ public class ListModelsTest {
         Path path = Paths.get("../model-zoo/src/test/resources/mlrepo");
         String repoUrl = path.toRealPath().toAbsolutePath().toUri().toURL().toExternalForm();
         System.setProperty("ai.djl.repository.zoo.location", "src/test/resources," + repoUrl);
+        ModelZoo.registerModelZoo(new DefaultZooProvider());
         Criteria<?, ?> criteria = Criteria.builder().optApplication(NLP.ANY).build();
         Map<Application, List<Artifact>> models = ModelZoo.listModels(criteria);