Repository: deepjavalibrary/djl
Pull request #3725 (2025-07-06 21:01:06)

[api] Implements remote REST API call as a model
1. Created a RpcEngine to load a REST API as model
2. Use Criteria to load a model deployed on remote model server
3. Allows inject custom translators to make a REST API call


Patch:
diff --git a/api/build.gradle.kts b/api/build.gradle.kts
index 36c077741..e3effc89f 100644
--- a/api/build.gradle.kts
+++ b/api/build.gradle.kts
@@ -11,6 +11,7 @@ dependencies {
     }
     api(libs.slf4j.api)
 
+    testImplementation(project(":testing"))
     testImplementation(libs.testng)
     testImplementation(libs.slf4j.simple)
     testRuntimeOnly(project(":engines:pytorch:pytorch-model-zoo"))
diff --git a/api/src/main/java/ai/djl/engine/rpc/RpcClient.java b/api/src/main/java/ai/djl/engine/rpc/RpcClient.java
new file mode 100644
index 000000000..a86478dae
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/RpcClient.java
@@ -0,0 +1,252 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.inference.streaming.ChunkedBytesSupplier;
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.ndarray.BytesSupplier;
+import ai.djl.util.Utils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** A client used to connect to remote model server. */
+public final class RpcClient {
+
+    private static final Logger logger = LoggerFactory.getLogger(RpcClient.class);
+    private static final Set<String> RESERVED_KEYS =
+            new HashSet<>(
+                    Arrays.asList(
+                            "engine",
+                            "translatorfactory",
+                            "translator",
+                            "model_name",
+                            "artifact_id",
+                            "application",
+                            "task",
+                            "djl_rpc_uri",
+                            "method",
+                            "api_key",
+                            "content-type"));
+
+    private URL url;
+    private String method;
+    private Map<CaseInsensitiveKey, String> headers;
+
+    private RpcClient(URL url, String method, Map<CaseInsensitiveKey, String> headers) {
+        this.url = url;
+        this.method = method;
+        this.headers = headers;
+    }
+
+    /**
+     * Returns a new {@code Client} instance from criteria arguments.
+     *
+     * @param arguments the criteria arguments.
+     * @return a new {@code Client} instance
+     * @throws MalformedURLException if url is invalid
+     */
+    public static RpcClient getClient(Map<String, ?> arguments) throws MalformedURLException {
+        String url = arguments.get("djl_rpc_uri").toString();
+        String method = getOrDefault(arguments, "method", "POST");
+        String apiKey = getOrDefault(arguments, "api_key", null);
+        String contentType = getOrDefault(arguments, "content-type", "application/json");
+        Map<CaseInsensitiveKey, String> httpHeaders = new ConcurrentHashMap<>();
+        for (Map.Entry<String, ?> entry : arguments.entrySet()) {
+            String key = entry.getKey();
+            String value = entry.getValue().toString().trim();
+            if (RESERVED_KEYS.contains(key.toLowerCase(Locale.ROOT))) {
+                continue;
+            }
+            httpHeaders.put(new CaseInsensitiveKey(key), value);
+        }
+        httpHeaders.put(new CaseInsensitiveKey("Content-Type"), contentType);
+        if (url.startsWith("https://generativelanguage.googleapis.com")) {
+            if (apiKey == null) {
+                apiKey = Utils.getenv("GEMINI_API_KEY");
+                if (apiKey == null) {
+                    apiKey = Utils.getenv("GOOGLE_API_KEY");
+                }
+            }
+            if (apiKey != null) {
+                if (url.endsWith("/openai/chat/completions")) {
+                    httpHeaders.put(new CaseInsensitiveKey("Authorization"), "Bearer " + apiKey);
+                } else {
+                    httpHeaders.put(new CaseInsensitiveKey("x-goog-api-key"), apiKey);
+                }
+            }
+        } else if (apiKey != null) {
+            httpHeaders.put(new CaseInsensitiveKey("Authorization"), "Bearer " + apiKey);
+        }
+        return new RpcClient(new URL(url), method, httpHeaders);
+    }
+
+    /**
+     * Sends request to remote server.
+     *
+     * @param input the input
+     * @return the output
+     * @throws IOException if connection failed
+     */
+    public Output send(Input input) throws IOException {
+        if (Utils.isOfflineMode()) {
+            throw new IOException("Offline mode is enabled.");
+        }
+        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        boolean isSse = false;
+        try {
+            conn.setRequestMethod(method);
+            if ("POST".equals(method) || "PUT".equals(method)) {
+                conn.setDoOutput(true);
+            }
+            Map<String, String> prop = input.getProperties();
+            Map<CaseInsensitiveKey, String> reqHeaders = new ConcurrentHashMap<>(headers);
+            for (Map.Entry<String, String> entry : prop.entrySet()) {
+                reqHeaders.put(new CaseInsensitiveKey(entry.getKey()), entry.getValue());
+            }
+            for (Map.Entry<CaseInsensitiveKey, String> header : reqHeaders.entrySet()) {
+                conn.addRequestProperty(header.getKey().key, header.getValue());
+            }
+
+            conn.connect();
+            BytesSupplier content = input.getData();
+            if (content != null) {
+                try (OutputStream os = conn.getOutputStream()) {
+                    os.write(content.getAsBytes());
+                }
+            }
+            int code = conn.getResponseCode();
+            Output out = new Output(code, conn.getResponseMessage());
+            Map<String, List<String>> respHeaders = conn.getHeaderFields();
+            for (Map.Entry<String, List<String>> entry : respHeaders.entrySet()) {
+                String key = entry.getKey();
+                String value = entry.getValue().get(0);
+                if (key != null) {
+                    if ("content-type".equalsIgnoreCase(key)
+                            && "text/event-stream".equalsIgnoreCase(value)) {
+                        isSse = true;
+                    }
+                    out.addProperty(key, value);
+                }
+            }
+            if (code == 200) {
+                if (isSse) {
+                    ChunkedBytesSupplier cbs = new ChunkedBytesSupplier();
+                    out.add(cbs);
+                    CompletableFuture.<Void>supplyAsync(
+                            () -> {
+                                try (BufferedReader reader =
+                                        new BufferedReader(
+                                                new InputStreamReader(
+                                                        conn.getInputStream(),
+                                                        StandardCharsets.UTF_8))) {
+                                    String line;
+                                    while ((line = reader.readLine()) != null) {
+                                        if (line.startsWith("data: ")) {
+                                            line = line.substring(6);
+                                        }
+                                        if (!line.isEmpty()) {
+                                            cbs.appendContent(BytesSupplier.wrap(line), false);
+                                        }
+                                    }
+                                } catch (IOException e) {
+                                    logger.warn("Failed run inference.", e);
+                                    cbs.appendContent(
+                                            BytesSupplier.wrap("connection abort exceptionally"),
+                                            false);
+                                } finally {
+                                    cbs.appendContent(new byte[0], true);
+                                    conn.disconnect();
+                                }
+                                return null;
+                            });
+                } else {
+                    try (InputStream is = conn.getInputStream()) {
+                        out.add(Utils.toByteArray(is));
+                    }
+                }
+            } else {
+                try (InputStream is = conn.getErrorStream()) {
+                    if (is != null) {
+                        String error = Utils.toString(is);
+                        out.add(error);
+                        logger.warn("Failed to invoke model server: {}", error);
+                    } else {
+                        logger.warn("Failed to invoke model server, code: {}", code);
+                    }
+                }
+            }
+            return out;
+        } finally {
+            if (!isSse) {
+                conn.disconnect();
+            }
+        }
+    }
+
+    private static String getOrDefault(Map<String, ?> arguments, String key, String def) {
+        for (Map.Entry<String, ?> entry : arguments.entrySet()) {
+            if (entry.getKey().equalsIgnoreCase(key)) {
+                return entry.getValue().toString();
+            }
+        }
+        return def;
+    }
+
+    static final class CaseInsensitiveKey {
+        String key;
+
+        public CaseInsensitiveKey(String key) {
+            this.key = key;
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (!(o instanceof CaseInsensitiveKey)) {
+                return false;
+            }
+            CaseInsensitiveKey header = (CaseInsensitiveKey) o;
+            return key.equalsIgnoreCase(header.key);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public int hashCode() {
+            return Objects.hashCode(key.toLowerCase(Locale.ROOT));
+        }
+    }
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/RpcEngine.java b/api/src/main/java/ai/djl/engine/rpc/RpcEngine.java
new file mode 100644
index 000000000..62b72a6e2
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/RpcEngine.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.Device;
+import ai.djl.Model;
+import ai.djl.engine.Engine;
+import ai.djl.ndarray.NDManager;
+import ai.djl.util.passthrough.PassthroughNDManager;
+
+/** The {@code RpcEngine} is a client side implementation for remote model server. */
+public class RpcEngine extends Engine {
+
+    public static final String ENGINE_NAME = "RPC";
+    static final int RANK = 15;
+
+    private Engine alternativeEngine;
+    private boolean initialized;
+
+    static Engine newInstance() {
+        return new RpcEngine();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Engine getAlternativeEngine() {
+        if (!initialized && !Boolean.getBoolean("ai.djl.java.disable_alternative")) {
+            Engine engine = Engine.getInstance();
+            if (engine.getRank() < getRank()) {
+                // alternativeEngine should not have the same rank as OnnxRuntime
+                alternativeEngine = engine;
+            }
+            initialized = true;
+        }
+        return alternativeEngine;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public String getEngineName() {
+        return ENGINE_NAME;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public int getRank() {
+        return RANK;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public String getVersion() {
+        return Engine.class.getPackage().getSpecificationVersion();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public boolean hasCapability(String capability) {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Model newModel(String name, Device device) {
+        return new RpcModel(name);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDManager newBaseManager() {
+        return newBaseManager(null);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDManager newBaseManager(Device device) {
+        return PassthroughNDManager.INSTANCE;
+    }
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/RpcEngineProvider.java b/api/src/main/java/ai/djl/engine/rpc/RpcEngineProvider.java
new file mode 100644
index 000000000..deec07b2b
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/RpcEngineProvider.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.engine.Engine;
+import ai.djl.engine.EngineProvider;
+
+/** {@code RpcEngineProvider} is the client side implementation for remote model server. */
+public class RpcEngineProvider implements EngineProvider {
+
+    /** {@inheritDoc} */
+    @Override
+    public String getEngineName() {
+        return RpcEngine.ENGINE_NAME;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public int getEngineRank() {
+        return RpcEngine.RANK;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Engine getEngine() {
+        return InstanceHolder.INSTANCE;
+    }
+
+    private static class InstanceHolder {
+        static final Engine INSTANCE = RpcEngine.newInstance();
+    }
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/RpcModel.java b/api/src/main/java/ai/djl/engine/rpc/RpcModel.java
new file mode 100644
index 000000000..8c17aa6f4
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/RpcModel.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.BaseModel;
+import ai.djl.MalformedModelException;
+import ai.djl.Model;
+import ai.djl.ndarray.types.DataType;
+import ai.djl.util.passthrough.PassthroughNDManager;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+
+/** {@code RpcModel} is an implementation for the {@link Model} deployed on remote model server. */
+public class RpcModel extends BaseModel {
+
+    /**
+     * Constructs a new Model on a given device.
+     *
+     * @param name the model name
+     */
+    RpcModel(String name) {
+        super(name);
+        manager = PassthroughNDManager.INSTANCE;
+        dataType = DataType.FLOAT32;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void load(Path modelPath, String prefix, Map<String, ?> options)
+            throws IOException, MalformedModelException {
+        setModelDir(modelPath);
+        wasLoaded = true;
+    }
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/RpcTranslator.java b/api/src/main/java/ai/djl/engine/rpc/RpcTranslator.java
new file mode 100644
index 000000000..9c2fa5cfe
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/RpcTranslator.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.ndarray.NDList;
+import ai.djl.translate.NoBatchifyTranslator;
+import ai.djl.translate.TranslateException;
+import ai.djl.translate.TranslatorContext;
+
+import java.io.IOException;
+
+/** The {@link ai.djl.translate.Translator} for model deploy on remote model server. */
+public class RpcTranslator<I, O> implements NoBatchifyTranslator<I, O> {
+
+    private RpcClient client;
+    private TypeConverter<I, O> converter;
+
+    /**
+     * Constructs a {@code RpcTranslator} instance.
+     *
+     * @param client a {@link RpcClient} connects to remote model server
+     * @param converter a {@link TypeConverter} to convert data type
+     */
+    protected RpcTranslator(RpcClient client, TypeConverter<I, O> converter) {
+        this.client = client;
+        this.converter = converter;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public NDList processInput(TranslatorContext ctx, I input) throws IOException {
+        Input in = converter.toInput(input);
+        Output output = client.send(in);
+        ctx.setAttachment("output", output);
+        return new NDList();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public O processOutput(TranslatorContext ctx, NDList list) throws TranslateException {
+        Output output = (Output) ctx.getAttachment("output");
+        return converter.fromOutput(output);
+    }
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/RpcTranslatorFactory.java b/api/src/main/java/ai/djl/engine/rpc/RpcTranslatorFactory.java
new file mode 100644
index 000000000..a4530820a
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/RpcTranslatorFactory.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.Model;
+import ai.djl.inference.streaming.ChunkedBytesSupplier;
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.ndarray.BytesSupplier;
+import ai.djl.translate.TranslateException;
+import ai.djl.translate.Translator;
+import ai.djl.translate.TranslatorFactory;
+import ai.djl.util.JsonUtils;
+import ai.djl.util.Pair;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/** A {@link TranslatorFactory} that creates an {@link RpcTranslator}. */
+public class RpcTranslatorFactory implements TranslatorFactory {
+
+    private TypeConverter<?, ?> converter;
+    private Set<Pair<Type, Type>> supportedTypes;
+
+    /** Constructs a {@code RpcTranslatorFactory} instance. */
+    public RpcTranslatorFactory() {
+        supportedTypes = Collections.emptySet();
+    }
+
+    /**
+     * Constructs a {@code RpcTranslatorFactory} instance.
+     *
+     * @param converter the {@code TypeConverter}
+     */
+    public RpcTranslatorFactory(TypeConverter<?, ?> converter) {
+        this.converter = converter;
+        supportedTypes = Collections.singleton(converter.getSupportedType());
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    @SuppressWarnings("unchecked")
+    public <I, O> Translator<I, O> newInstance(
+            Class<I> input, Class<O> output, Model model, Map<String, ?> arguments)
+            throws TranslateException {
+        try {
+            if (!isSupported(input, output)) {
+                throw new IllegalArgumentException("Unsupported input/output types.");
+            }
+            RpcClient client = RpcClient.getClient(arguments);
+            if (converter != null) {
+                return new RpcTranslator<>(client, (TypeConverter<I, O>) converter);
+            }
+            return new RpcTranslator<>(client, new DefaultTypeConverter<>(input, output));
+        } catch (IOException e) {
+            throw new TranslateException(e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Set<Pair<Type, Type>> getSupportedTypes() {
+        return supportedTypes;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public boolean isSupported(Class<?> input, Class<?> output) {
+        if (converter == null) {
+            return true;
+        }
+        return TranslatorFactory.super.isSupported(input, output);
+    }
+
+    private static final class DefaultTypeConverter<I, O> implements TypeConverter<I, O> {
+
+        private Class<I> input;
+        private Class<O> output;
+        private Method fromJson;
+        private Method fromJsonStream;
+
+        DefaultTypeConverter(Class<I> input, Class<O> output) {
+            this.input = input;
+            this.output = output;
+            try {
+                fromJsonStream = output.getDeclaredMethod("fromJson", Iterator.class);
+            } catch (ReflectiveOperationException e) {
+                // ignore
+            }
+            try {
+                fromJson = output.getDeclaredMethod("fromJson", String.class);
+            } catch (ReflectiveOperationException e) {
+                // ignore
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public Pair<Type, Type> getSupportedType() {
+            return new Pair<>(input, output);
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public Input toInput(I in) {
+            if (in instanceof Input) {
+                return (Input) in;
+            }
+            Input converted = new Input();
+            converted.add(BytesSupplier.wrapAsJson(in));
+            return converted;
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        @SuppressWarnings("unchecked")
+        public O fromOutput(Output out) throws TranslateException {
+            if (output == Output.class) {
+                return (O) out;
+            }
+            int code = out.getCode();
+            BytesSupplier data = out.getData();
+            if (code != 200) {
+                String error;
+                if (data == null) {
+                    error = out.getMessage();
+                } else {
+                    error = out.getMessage() + " " + data.getAsString();
+                }
+                throw new TranslateException(error);
+            }
+            if (output == String.class) {
+                return (O) data.getAsString();
+            }
+            try {
+                if (data instanceof ChunkedBytesSupplier && fromJsonStream != null) {
+                    Iterator<String> it = new ChunkIterator((ChunkedBytesSupplier) data);
+                    return (O) fromJsonStream.invoke(null, it);
+                } else if (fromJson != null) {
+                    return (O) fromJson.invoke(null, data.getAsString());
+                }
+            } catch (ReflectiveOperationException e) {
+                throw new TranslateException("Failed convert from json", e);
+            }
+            return JsonUtils.GSON.fromJson(data.getAsString(), output);
+        }
+    }
+
+    private static final class ChunkIterator implements Iterator<String> {
+
+        private ChunkedBytesSupplier cbs;
+        private boolean error;
+
+        ChunkIterator(ChunkedBytesSupplier cbs) {
+            this.cbs = cbs;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (error) {
+                return false;
+            }
+            return cbs.hasNext();
+        }
+
+        @Override
+        public String next() {
+            try {
+                return new String(cbs.nextChunk(20, TimeUnit.SECONDS), StandardCharsets.UTF_8);
+            } catch (InterruptedException e) {
+                error = true;
+                return null;
+            }
+        }
+    }
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/TypeConverter.java b/api/src/main/java/ai/djl/engine/rpc/TypeConverter.java
new file mode 100644
index 000000000..2179cc77f
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/TypeConverter.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.translate.TranslateException;
+import ai.djl.util.Pair;
+
+import java.lang.reflect.Type;
+
+/**
+ * A {code TypeConverter} interface defines how data type is converted to Input/Output.
+ *
+ * @param <I> the target input type
+ * @param <O> the target output type
+ */
+public interface TypeConverter<I, O> {
+
+    /**
+     * Returns the supported type.
+     *
+     * @return the supported type
+     */
+    Pair<Type, Type> getSupportedType();
+
+    /**
+     * Convert the data type to {@link Input}.
+     *
+     * @param in the input data
+     * @return the converted data
+     */
+    Input toInput(I in);
+
+    /**
+     * Convert the {@link Output} to target data type.
+     *
+     * @param out the output data
+     * @return the converted data
+     */
+    O fromOutput(Output out) throws TranslateException;
+}
diff --git a/api/src/main/java/ai/djl/engine/rpc/package-info.java b/api/src/main/java/ai/djl/engine/rpc/package-info.java
new file mode 100644
index 000000000..1c76700a0
--- /dev/null
+++ b/api/src/main/java/ai/djl/engine/rpc/package-info.java
@@ -0,0 +1,15 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+
+/** Contains classes to interface with the underlying RPC engine. */
+package ai.djl.engine.rpc;
diff --git a/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java b/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java
index 7079d9406..751a9a00d 100644
--- a/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java
+++ b/api/src/main/java/ai/djl/repository/RepositoryFactoryImpl.java
@@ -96,7 +96,7 @@ class RepositoryFactoryImpl implements RepositoryFactory {
             fileName = FilenameUtils.getNamePart(fileName);
             return new SimpleUrlRepository(name, uri, fileName);
         }
-        throw new IllegalArgumentException("============: " + uri);
+        return new RpcRepository(name, uri);
     }
 
     /** {@inheritDoc} */
diff --git a/api/src/main/java/ai/djl/repository/RpcRepository.java b/api/src/main/java/ai/djl/repository/RpcRepository.java
new file mode 100644
index 000000000..57efe0187
--- /dev/null
+++ b/api/src/main/java/ai/djl/repository/RpcRepository.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.repository;
+
+import ai.djl.Application;
+import ai.djl.engine.rpc.RpcEngine;
+import ai.djl.engine.rpc.RpcTranslatorFactory;
+import ai.djl.repository.zoo.DefaultModelZoo;
+import ai.djl.util.Progress;
+import ai.djl.util.Utils;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A {@code RpcRepository} is a {@link Repository} as a remote model server.
+ *
+ * @see Repository
+ */
+public class RpcRepository extends AbstractRepository {
+
+    private String artifactId;
+    private String modelName;
+
+    RpcRepository(String name, URI uri) {
+        super(name, uri);
+        modelName = arguments.get("model_name");
+        artifactId = arguments.get("artifact_id");
+        if (artifactId == null) {
+            artifactId = "rpc";
+        }
+        if (modelName == null) {
+            modelName = artifactId;
+        }
+        arguments.put("translatorFactory", RpcTranslatorFactory.class.getName());
+        arguments.put("engine", RpcEngine.ENGINE_NAME);
+        arguments.put("djl_rpc_uri", uri.toString());
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public boolean isRemote() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Metadata locate(MRL mrl) throws IOException {
+        return getMetadata();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public Artifact resolve(MRL mrl, Map<String, String> filter) throws IOException {
+        List<Artifact> artifacts = locate(mrl).getArtifacts();
+        if (artifacts.isEmpty()) {
+            return null;
+        }
+        return artifacts.get(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public List<MRL> getResources() {
+        MRL mrl = MRL.undefined(this, DefaultModelZoo.GROUP_ID, artifactId);
+        return Collections.singletonList(mrl);
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    protected void download(Path tmp, URI baseUri, Artifact.Item item, Progress progress) {}
+
+    private synchronized Metadata getMetadata() {
+        Artifact artifact = new Artifact();
+        artifact.setName(modelName);
+        artifact.getArguments().putAll(arguments);
+        Map<String, Artifact.Item> files = new ConcurrentHashMap<>();
+        Artifact.Item item = new Artifact.Item();
+        item.setUri(uri.getPath());
+        item.setName(""); // avoid creating extra folder
+        item.setArtifact(artifact);
+        item.setSize(0);
+        files.put(artifactId, item);
+        artifact.setFiles(files);
+
+        Metadata metadata = new Metadata.MatchAllMetadata();
+        metadata.setArtifactId(artifactId);
+        metadata.setArtifacts(Collections.singletonList(artifact));
+        String hash = Utils.hash(uri.toString());
+        MRL mrl = model(Application.UNDEFINED, DefaultModelZoo.GROUP_ID, hash);
+        metadata.setRepositoryUri(mrl.toURI());
+        return metadata;
+    }
+}
diff --git a/api/src/main/resources/META-INF/services/ai.djl.engine.EngineProvider b/api/src/main/resources/META-INF/services/ai.djl.engine.EngineProvider
new file mode 100644
index 000000000..c33aeb888
--- /dev/null
+++ b/api/src/main/resources/META-INF/services/ai.djl.engine.EngineProvider
@@ -0,0 +1,1 @@
+ai.djl.engine.rpc.RpcEngineProvider


Test patch:
diff --git a/api/src/test/java/ai/djl/engine/rpc/RpcEngineTest.java b/api/src/test/java/ai/djl/engine/rpc/RpcEngineTest.java
new file mode 100644
index 000000000..61c13cae7
--- /dev/null
+++ b/api/src/test/java/ai/djl/engine/rpc/RpcEngineTest.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.engine.Engine;
+import ai.djl.ndarray.NDManager;
+import ai.djl.util.passthrough.PassthroughNDManager;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class RpcEngineTest {
+
+    @Test
+    public void testRpcEngine() {
+        Engine engine = RpcEngine.getEngine(RpcEngine.ENGINE_NAME);
+        Assert.assertEquals(engine.getEngineName(), RpcEngine.ENGINE_NAME);
+        Assert.assertEquals(engine.getRank(), 15);
+        Assert.assertEquals(
+                engine.getVersion(), Engine.class.getPackage().getSpecificationVersion());
+        Assert.assertFalse(engine.hasCapability("CUDA"));
+        try (NDManager manager = engine.newBaseManager()) {
+            Assert.assertTrue(manager instanceof PassthroughNDManager);
+        }
+        Engine alt = engine.getAlternativeEngine();
+        Assert.assertNotNull(alt);
+    }
+}
diff --git a/api/src/test/java/ai/djl/engine/rpc/RpcTranslatorFactoryTest.java b/api/src/test/java/ai/djl/engine/rpc/RpcTranslatorFactoryTest.java
new file mode 100644
index 000000000..0087760cd
--- /dev/null
+++ b/api/src/test/java/ai/djl/engine/rpc/RpcTranslatorFactoryTest.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.engine.rpc;
+
+import ai.djl.ModelException;
+import ai.djl.inference.Predictor;
+import ai.djl.modality.Input;
+import ai.djl.modality.Output;
+import ai.djl.repository.zoo.Criteria;
+import ai.djl.repository.zoo.ZooModel;
+import ai.djl.testing.TestServer;
+import ai.djl.translate.TranslateException;
+import ai.djl.translate.Translator;
+import ai.djl.util.JsonSerializable;
+import ai.djl.util.JsonUtils;
+import ai.djl.util.Pair;
+
+import com.google.gson.JsonElement;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.Iterator;
+import java.util.Map;
+
+public class RpcTranslatorFactoryTest {
+
+    @Test
+    public void testRpcTranslatorFactory() throws TranslateException {
+        RpcTranslatorFactory factory = new RpcTranslatorFactory();
+        Map<String, String> arguments = Map.of("djl_rpc_uri", "http://localhost");
+        Translator<String, String> translator =
+                factory.newInstance(String.class, String.class, null, arguments);
+        Assert.assertNotNull(translator);
+
+        RpcTranslatorFactory customFactory =
+                new RpcTranslatorFactory(
+                        new TypeConverter<String, String>() {
+
+                            /** {@inheritDoc} */
+                            @Override
+                            public Pair<Type, Type> getSupportedType() {
+                                return new Pair<>(String.class, String.class);
+                            }
+
+                            /** {@inheritDoc} */
+                            @Override
+                            public Input toInput(String in) {
+                                return new Input();
+                            }
+
+                            /** {@inheritDoc} */
+                            @Override
+                            public String fromOutput(Output out) {
+                                return "";
+                            }
+                        });
+
+        translator = customFactory.newInstance(String.class, String.class, null, arguments);
+        Assert.assertNotNull(translator);
+
+        Assert.assertThrows(
+                () -> customFactory.newInstance(Integer.class, String.class, null, arguments));
+    }
+
+    @Test
+    public void testRpcRepository() throws ModelException, IOException, TranslateException {
+        String content = "{\"data\": \"echo\"}";
+        try (TestServer server = TestServer.newInstance(content)) {
+            int port = server.getPort();
+
+            Criteria<Input, Output> criteria =
+                    Criteria.builder()
+                            .setTypes(Input.class, Output.class)
+                            .optModelUrls("http://localhost:" + port + "/invocations")
+                            .optArgument("method", "POST")
+                            .optArgument("API_KEY", "1234")
+                            .optArgument("Content-Type", "text/plain")
+                            .optArgument("Accept", "*/*")
+                            .build();
+            try (ZooModel<Input, Output> model = criteria.loadModel();
+                    Predictor<Input, Output> predictor = model.newPredictor()) {
+                Input in = new Input();
+                in.add("Hello, I am a [MASK] model.");
+                Output ret = predictor.predict(in);
+                Assert.assertEquals(ret.getData().getAsString(), content);
+            }
+
+            Criteria<String, String> criteria1 =
+                    Criteria.builder()
+                            .setTypes(String.class, String.class)
+                            .optModelUrls("http://localhost:" + port + "/invocations")
+                            .optArgument("method", "POST")
+                            .optArgument("API_KEY", "1234")
+                            .optArgument("Content-Type", "application/json")
+                            .optArgument("Accept", "*/*")
+                            .build();
+            try (ZooModel<String, String> model = criteria1.loadModel();
+                    Predictor<String, String> predictor = model.newPredictor()) {
+                String ret = predictor.predict("Hello, I am a [MASK] model.");
+                Assert.assertEquals(ret, content);
+            }
+
+            TestData testInput = new TestData("Hello");
+            Criteria<TestData, TestData> criteria2 =
+                    Criteria.builder()
+                            .setTypes(TestData.class, TestData.class)
+                            .optModelUrls("http://localhost:" + port + "/invocations")
+                            .optArgument("method", "POST")
+                            .optArgument("API_KEY", "1234")
+                            .optArgument("Content-Type", "application/json")
+                            .optArgument("Accept", "*/*")
+                            .build();
+            try (ZooModel<TestData, TestData> model = criteria2.loadModel();
+                    Predictor<TestData, TestData> predictor = model.newPredictor()) {
+                TestData ret = predictor.predict(testInput);
+                Assert.assertEquals(ret.data, "echo");
+
+                // test SSE response
+                server.setContent("data: line1\n\ndata:  line2\n\n");
+                server.setContentType("text/event-stream");
+                ret = predictor.predict(testInput);
+                Assert.assertEquals(ret.data, "line1 line2");
+
+                server.setCode(400);
+                Assert.assertThrows(() -> predictor.predict(testInput));
+            }
+        }
+    }
+
+    private static final class TestData implements JsonSerializable {
+
+        private static final long serialVersionUID = 1L;
+
+        String data;
+
+        public TestData(String data) {
+            this.data = data;
+        }
+
+        public static TestData fromJson(String json) {
+            return JsonUtils.GSON.fromJson(json, TestData.class);
+        }
+
+        public static TestData fromJson(Iterator<String> it) {
+            StringBuilder sb = new StringBuilder();
+            it.forEachRemaining(sb::append);
+            return new TestData(sb.toString());
+        }
+
+        /** {@inheritDoc} */
+        @Override
+        public JsonElement serialize() {
+            return JsonUtils.GSON.toJsonTree(data);
+        }
+    }
+}
diff --git a/api/src/test/java/ai/djl/engine/rpc/package-info.java b/api/src/test/java/ai/djl/engine/rpc/package-info.java
new file mode 100644
index 000000000..3b2a9de6a
--- /dev/null
+++ b/api/src/test/java/ai/djl/engine/rpc/package-info.java
@@ -0,0 +1,14 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+/** Contains tests for {@link ai.djl.engine.rpc}. */
+package ai.djl.engine.rpc;
diff --git a/testing/src/main/java/ai/djl/testing/TestServer.java b/testing/src/main/java/ai/djl/testing/TestServer.java
new file mode 100644
index 000000000..7edaac5c0
--- /dev/null
+++ b/testing/src/main/java/ai/djl/testing/TestServer.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance
+ * with the License. A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ * OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
+ */
+package ai.djl.testing;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+
+/** A test model server that can accept inference requests. */
+@SuppressWarnings("PMD")
+public final class TestServer implements Runnable, AutoCloseable {
+
+    private static final Logger logger = LoggerFactory.getLogger(TestServer.class);
+
+    private ServerSocket serverSocket;
+    private Thread thread;
+    private boolean stopped;
+    private int code = 200;
+    private String content;
+    private String contentType;
+
+    private TestServer(ServerSocket serverSocket, String content) {
+        this.serverSocket = serverSocket;
+        this.content = content;
+        this.contentType = "application/json";
+    }
+
+    /**
+     * Returns a new {@code TestServer} instance.
+     *
+     * @return a new {@code TestServer} instance
+     * @throws IOException if failed to start the listener
+     */
+    public static TestServer newInstance() throws IOException {
+        return newInstance("");
+    }
+
+    /**
+     * Returns a new {@code TestServer} instance.
+     *
+     * @param content the server response
+     * @return a new {@code TestServer} instance
+     * @throws IOException if failed to start the listener
+     */
+    public static TestServer newInstance(String content) throws IOException {
+        ServerSocket serverSocket = new ServerSocket(0);
+        TestServer server = new TestServer(serverSocket, content);
+        server.thread = new Thread(server);
+        server.thread.start();
+        return server;
+    }
+
+    /**
+     * Sets the server response.
+     *
+     * @param content the server response
+     */
+    public void setContent(String content) {
+        this.content = content;
+    }
+
+    /**
+     * Sets the server response contentType.
+     *
+     * @param contentType the server response contentType
+     */
+    public void setContentType(String contentType) {
+        this.contentType = contentType;
+    }
+
+    /**
+     * Sets the server response code.
+     *
+     * @param code the server response code
+     */
+    public void setCode(int code) {
+        this.code = code;
+    }
+
+    /**
+     * Returns the listening port.
+     *
+     * @return the listening port
+     */
+    public int getPort() {
+        return serverSocket.getLocalPort();
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void run() {
+        try {
+            while (!stopped) {
+                try (Socket socket = serverSocket.accept();
+                        OutputStream out = socket.getOutputStream();
+                        InputStream in = socket.getInputStream()) {
+                    readLine(in);
+                    readLine(in);
+                    int length = 0;
+                    while (true) {
+                        String line = readLine(in);
+                        if (line.isEmpty()) {
+                            in.readNBytes(length);
+                            break;
+                        } else {
+                            String[] header = line.split(":");
+                            if ("Content-Length".equalsIgnoreCase(header[0].trim())) {
+                                length = Integer.parseInt(header[1].trim());
+                            }
+                        }
+                    }
+                    out.write(
+                            ("HTTP/1.1 "
+                                            + code
+                                            + " OK\r\nContent-Type: "
+                                            + contentType
+                                            + "\r\n\r\n")
+                                    .getBytes(StandardCharsets.UTF_8));
+                    out.write(content.getBytes(StandardCharsets.UTF_8));
+                    out.flush();
+                } catch (IOException e) {
+                    logger.warn("", e);
+                }
+            }
+            serverSocket.close();
+        } catch (IOException e) {
+            logger.warn("", e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override
+    public void close() {
+        stopped = true;
+        thread.interrupt();
+    }
+
+    private String readLine(InputStream is) throws IOException {
+        StringBuilder sb = new StringBuilder();
+        int c;
+        do {
+            c = is.read();
+            if (c == '\n') {
+                break;
+            }
+            if (c != '\r') {
+                sb.append((char) c);
+            }
+        } while (c != -1);
+        return sb.toString();
+    }
+}