Repository: langchain4j/langchain4j (2026-04-08T16:56:59Z)

Issue:
OnnxScoringModel can't release resources until JVM is stopped
I've noticed that for `OnnxScoringModel`, there is no way to call `close()` that should be propagated to `OrtSession.close()`, which is responsible for releasing any non-jvm allocated memory allocated by OrtSession. When OnnxScoringModel is called multiple times, especially with larger models, it can allocate significant amount of memory and might cause machine swapping.

OnnxScoringModel doesn't implement AutoCloseable - will that be hard to add? Also, I think the above behavior should be documented, at least until `close()` is added.


Discussion:
@sanel thanks a lot for reporting! This makes sense, would you like to open a PR with a desired improvement?
Yeah, sounds good! I'll prepare PR and link it here when is done.



Patch:
diff --git a/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/AbstractInProcessScoringModel.java b/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/AbstractInProcessScoringModel.java
index 2dec182ffb3..eab573eabf3 100644
--- a/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/AbstractInProcessScoringModel.java
+++ b/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/AbstractInProcessScoringModel.java
@@ -5,16 +5,19 @@
 import dev.langchain4j.model.output.Response;
 import dev.langchain4j.model.output.TokenUsage;
 import dev.langchain4j.model.scoring.ScoringModel;
-
 import java.util.List;
 import java.util.stream.Collectors;
 
-abstract class AbstractInProcessScoringModel implements ScoringModel {
+abstract class AbstractInProcessScoringModel implements ScoringModel, AutoCloseable {
 
-    public AbstractInProcessScoringModel() {
-    }
+    public AbstractInProcessScoringModel() {}
 
-    static OnnxScoringBertCrossEncoder loadFromFileSystem(String pathToModel, OrtSession.SessionOptions options, String pathToTokenizer, int modelMaxLength, boolean normalize) {
+    static OnnxScoringBertCrossEncoder loadFromFileSystem(
+            String pathToModel,
+            OrtSession.SessionOptions options,
+            String pathToTokenizer,
+            int modelMaxLength,
+            boolean normalize) {
         try {
             return new OnnxScoringBertCrossEncoder(pathToModel, options, pathToTokenizer, modelMaxLength, normalize);
         } catch (Exception e) {
@@ -25,8 +28,14 @@ static OnnxScoringBertCrossEncoder loadFromFileSystem(String pathToModel, OrtSes
     protected abstract OnnxScoringBertCrossEncoder model();
 
     public Response<List<Double>> scoreAll(List<TextSegment> segments, String query) {
-        OnnxScoringBertCrossEncoder.ScoringAndTokenCount scoresAndTokenCount = this.model().scoreAll(query,
-                segments.stream().map(TextSegment::text).collect(Collectors.toList()));
+        OnnxScoringBertCrossEncoder.ScoringAndTokenCount scoresAndTokenCount = this.model()
+                .scoreAll(query, segments.stream().map(TextSegment::text).collect(Collectors.toList()));
         return Response.from(scoresAndTokenCount.scores, new TokenUsage(scoresAndTokenCount.tokenCount));
     }
+
+    @Override
+    public void close() {
+        OnnxScoringBertCrossEncoder m = model();
+        if (m != null) m.close();
+    }
 }
diff --git a/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/OnnxScoringBertCrossEncoder.java b/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/OnnxScoringBertCrossEncoder.java
index b1a74dac5ae..7ef876ec55e 100644
--- a/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/OnnxScoringBertCrossEncoder.java
+++ b/langchain4j-onnx-scoring/src/main/java/dev/langchain4j/model/scoring/onnx/OnnxScoringBertCrossEncoder.java
@@ -1,5 +1,7 @@
 package dev.langchain4j.model.scoring.onnx;
 
+import static ai.onnxruntime.OnnxTensor.createTensor;
+
 import ai.djl.huggingface.tokenizers.Encoding;
 import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer;
 import ai.djl.util.PairList;
@@ -8,30 +10,38 @@
 import ai.onnxruntime.OrtException;
 import ai.onnxruntime.OrtSession;
 import ai.onnxruntime.OrtSession.Result;
-
 import java.nio.file.Paths;
 import java.util.*;
 
-import static ai.onnxruntime.OnnxTensor.createTensor;
-
-class OnnxScoringBertCrossEncoder {
+class OnnxScoringBertCrossEncoder implements AutoCloseable {
 
     private final OrtEnvironment environment;
     private final OrtSession session;
     private final Set<String> expectedInputs;
     private final HuggingFaceTokenizer tokenizer;
     private final boolean normalize;
-
-    public OnnxScoringBertCrossEncoder(String modelPath, OrtSession.SessionOptions options, String pathToTokenizer, int modelMaxLength, boolean normalize) {
-        try {
+    private boolean closed;
+
+    public OnnxScoringBertCrossEncoder(
+            String modelPath,
+            OrtSession.SessionOptions options,
+            String pathToTokenizer,
+            int modelMaxLength,
+            boolean normalize) {
+        try (options) { // properly release parent session at the end of this block to prevent leaks
             this.environment = OrtEnvironment.getEnvironment();
             this.session = this.environment.createSession(modelPath, options);
             this.expectedInputs = session.getInputNames();
-            Map<String, String> tokenizerOptions = new HashMap<String, String>() {{
-                put("padding", "true");
-                put("truncation", "LONGEST_FIRST"); // Default maximum length limit, LONGEST-FIRST prioritizes truncating the longest part
-                put("modelMaxLength", String.valueOf(modelMaxLength - 2));
-            }};
+            Map<String, String> tokenizerOptions = new HashMap<String, String>() {
+                {
+                    put("padding", "true");
+                    put(
+                            "truncation",
+                            "LONGEST_FIRST"); // Default maximum length limit, LONGEST-FIRST prioritizes truncating the
+                    // longest part
+                    put("modelMaxLength", String.valueOf(modelMaxLength - 2));
+                }
+            };
             this.normalize = normalize;
             this.tokenizer = HuggingFaceTokenizer.newInstance(Paths.get(pathToTokenizer), tokenizerOptions);
         } catch (Exception e) {
@@ -50,6 +60,37 @@ static class ScoringAndTokenCount {
         }
     }
 
+    @Override
+    public void close() {
+        if (closed) {
+            return;
+        }
+
+        Exception firstException = null;
+
+        try {
+            if (session != null) session.close();
+        } catch (Exception e) {
+            firstException = e;
+        }
+
+        try {
+            if (tokenizer != null) tokenizer.close();
+        } catch (Exception e) {
+            if (firstException != null) {
+                firstException.addSuppressed(e);
+            } else {
+                firstException = e;
+            }
+        }
+
+        closed = true;
+
+        if (firstException != null) {
+            throw new RuntimeException(firstException);
+        }
+    }
+
     ScoringAndTokenCount scoreAll(String query, List<String> documents) {
         List<Double> scores;
         int tokenCount = 0;
@@ -57,7 +98,9 @@ ScoringAndTokenCount scoreAll(String query, List<String> documents) {
         PairList<String, String> pairs = new PairList<>();
         for (String document : documents) {
             pairs.add(query, document);
-            tokenCount += queryTokenCount + tokenizer.tokenize(document).size() - 2; // do not count special tokens [CLS] and [SEP]
+            tokenCount += queryTokenCount
+                    + tokenizer.tokenize(document).size()
+                    - 2; // do not count special tokens [CLS] and [SEP]
         }
         try (Result result = this.encode(pairs)) {
             scores = this.toScore(result);
@@ -79,11 +122,9 @@ private Result encode(PairList<String, String> pairs) throws OrtException {
             tokenTypeIds[i] = encodings[i].getTypeIds();
         }
 
-        try (
-                OnnxTensor inputIdsTensor = createTensor(environment, inputIds);
+        try (OnnxTensor inputIdsTensor = createTensor(environment, inputIds);
                 OnnxTensor attentionMaskTensor = createTensor(environment, attentionMask);
-                OnnxTensor tokenTypeIdsTensor = createTensor(this.environment, tokenTypeIds);
-        ) {
+                OnnxTensor tokenTypeIdsTensor = createTensor(this.environment, tokenTypeIds)) {
             Map<String, OnnxTensor> inputs = new HashMap<>();
             inputs.put("input_ids", inputIdsTensor);
             inputs.put("attention_mask", attentionMaskTensor);
@@ -103,7 +144,7 @@ private List<Double> toScore(OrtSession.Result result) throws OrtException {
             if (normalize) {
                 scores.add(sigmoid(floats[0]));
             } else {
-                scores.add((double)floats[0]);
+                scores.add((double) floats[0]);
             }
         }
         return scores;
@@ -112,5 +153,4 @@ private List<Double> toScore(OrtSession.Result result) throws OrtException {
     private double sigmoid(float x) {
         return 1 / (1 + Math.exp(-x));
     }
-
 }


Test patch:
diff --git a/langchain4j-onnx-scoring/src/test/java/dev/langchain4j/model/scoring/onnx/OnnxScoringModelIT.java b/langchain4j-onnx-scoring/src/test/java/dev/langchain4j/model/scoring/onnx/OnnxScoringModelIT.java
index 44398d57489..90b046d9285 100644
--- a/langchain4j-onnx-scoring/src/test/java/dev/langchain4j/model/scoring/onnx/OnnxScoringModelIT.java
+++ b/langchain4j-onnx-scoring/src/test/java/dev/langchain4j/model/scoring/onnx/OnnxScoringModelIT.java
@@ -17,6 +17,7 @@
 
 import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatNoException;
 import static org.assertj.core.data.Percentage.withPercentage;
 
 class OnnxScoringModelIT {
@@ -26,6 +27,9 @@ class OnnxScoringModelIT {
 
     private static ScoringModel model;
 
+    private static Path modelPath;
+    private static Path tokenizerPath;
+
     @BeforeAll
     static void initModel() throws IOException {
 
@@ -34,17 +38,21 @@ static void initModel() throws IOException {
         // System.setProperty("https.proxyPort","7890" );
 
         URL modelUrl = new URL("https://huggingface.co/Xenova/ms-marco-MiniLM-L-6-v2/resolve/main/onnx/model_quantized.onnx?download=true");
-        Path modelPath = tempDir.resolve("model_quantized.onnx");
+        modelPath = tempDir.resolve("model_quantized.onnx");
         Files.copy(modelUrl.openStream(), modelPath, REPLACE_EXISTING);
 
         URL tokenizerUrl = new URL("https://huggingface.co/Xenova/ms-marco-MiniLM-L-6-v2/resolve/main/tokenizer.json?download=true");
-        Path tokenizerPath = tempDir.resolve("tokenizer.json");
+        tokenizerPath = tempDir.resolve("tokenizer.json");
         Files.copy(tokenizerUrl.openStream(), tokenizerPath, REPLACE_EXISTING);
 
         // To check the modelMaxLength parameter, refer to the model configuration file at  https://huggingface.co/Xenova/ms-marco-MiniLM-L-6-v2/resolve/main/tokenizer_config.json
         model = new OnnxScoringModel(modelPath.toString(), new OrtSession.SessionOptions(), tokenizerPath.toString(), 512, false);
     }
 
+    private OnnxScoringModel createModel() {
+        return new OnnxScoringModel(modelPath.toString(), new OrtSession.SessionOptions(), tokenizerPath.toString(), 512, false);
+    }
+
     @Test
     void should_score_multiple_segments_with_all_parameters() {
         List<TextSegment> segments = new ArrayList<>();
@@ -68,4 +76,37 @@ void should_score_multiple_segments_with_all_parameters() {
 
         assertThat(response.finishReason()).isNull();
     }
+
+    @Test
+    void should_implement_autocloseable() {
+        assertThat(model).isInstanceOf(AutoCloseable.class);
+    }
+
+    @Test
+    void should_close_without_exception() {
+        OnnxScoringModel closeable = createModel();
+        assertThatNoException().isThrownBy(closeable::close);
+    }
+
+    @Test
+    void should_be_usable_with_try_with_resources() {
+        List<TextSegment> segments = new ArrayList<>();
+        segments.add(TextSegment.from("Berlin has a population of 3,520,031 registered inhabitants in an area of 891.82 square kilometers."));
+
+        try (OnnxScoringModel m = createModel()) {
+            Response<List<Double>> response = m.scoreAll(segments, "How many people live in Berlin?");
+
+            assertThat(response.content()).hasSize(1);
+            assertThat(response.content().get(0)).isCloseTo(8.663132667541504, withPercentage(1));
+        }
+    }
+
+    @Test
+    void should_allow_calling_close_multiple_times() {
+        OnnxScoringModel closeable = createModel();
+        assertThatNoException().isThrownBy(() -> {
+            closeable.close();
+            closeable.close();
+        });
+    }
 }
\ No newline at end of file