Repository: epam/ai-dial-core
Pull request #731: feat: Implement LLM Caching #729
Author: login: astsiapanay
Created: 2025-03-13T09:48:50Z | Merged: 2025-03-19T11:26:48Z



Diff:
diff --git a/config/src/main/java/com/epam/aidial/core/config/Features.java b/config/src/main/java/com/epam/aidial/core/config/Features.java
index 20a0592ef..1b6016106 100644
--- a/config/src/main/java/com/epam/aidial/core/config/Features.java
+++ b/config/src/main/java/com/epam/aidial/core/config/Features.java
@@ -40,4 +40,15 @@ public class Features {
     private Boolean temperatureSupported;
     @JsonAlias({"addonsSupported", "addons_supported"})
     private Boolean addonsSupported;
+    @JsonAlias({"cacheSupported", "cache_supported"})
+    private Boolean cacheSupported;
+    /**
+     * Try to create automatic cache, where it's possible. We will route request that is cached to same upstream.
+     * But in case, when cache upstream is not available, we will fallback to another available upstream, and will create cache there.
+     * <p>
+     *  <b>Note</b>. Core should calculate hash for all prefixes for each request, and check their existence in Redis.
+     * </p>
+     */
+    @JsonAlias({"autoCachingSupported", "auto_caching_supported"})
+    private Boolean autoCachingSupported;
 }
\ No newline at end of file
diff --git a/config/src/main/java/com/epam/aidial/core/config/Model.java b/config/src/main/java/com/epam/aidial/core/config/Model.java
index 7bedb6b2c..90ec73037 100644
--- a/config/src/main/java/com/epam/aidial/core/config/Model.java
+++ b/config/src/main/java/com/epam/aidial/core/config/Model.java
@@ -1,5 +1,6 @@
 package com.epam.aidial.core.config;
 
+import com.fasterxml.jackson.annotation.JsonAlias;
 import lombok.Data;
 import lombok.EqualsAndHashCode;
 import lombok.experimental.Accessors;
@@ -17,6 +18,8 @@ public class Model extends Deployment {
     private List<Upstream> upstreams = List.of();
     // if it's set then the model name is overridden with that name in the request body to the model adapter
     private String overrideName;
+    @JsonAlias({"fieldsHashingOrder", "fields_hashing_order"})
+    private List<String> fieldsHashingOrder = List.of("prefix.body.tools", "prefix.body.messages");
 
     public Model() {
         setMaxRetryAttempts(5);
diff --git a/server/src/main/java/com/epam/aidial/core/server/AiDial.java b/server/src/main/java/com/epam/aidial/core/server/AiDial.java
index c6cf43a1c..a71feda49 100644
--- a/server/src/main/java/com/epam/aidial/core/server/AiDial.java
+++ b/server/src/main/java/com/epam/aidial/core/server/AiDial.java
@@ -18,6 +18,7 @@
 import com.epam.aidial.core.server.service.ResourceOperationService;
 import com.epam.aidial.core.server.service.RuleService;
 import com.epam.aidial.core.server.service.ShareService;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import com.epam.aidial.core.server.service.VertxTimerService;
 import com.epam.aidial.core.server.service.codeinterpreter.CodeInterpreterService;
 import com.epam.aidial.core.server.token.TokenStatsTracker;
@@ -103,7 +104,6 @@ void start() throws Exception {
             client = vertx.createHttpClient(new HttpClientOptions(settings("client")));
 
             LogStore logStore = new GfLogStore(vertx);
-            UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, Random::new);
 
             if (accessTokenValidator == null) {
                 accessTokenValidator = new AccessTokenValidator(settings("identityProviders"), vertx, client);
@@ -143,11 +143,15 @@ void start() throws Exception {
 
             HeartbeatService heartbeatService = new HeartbeatService(
                     vertx, settings("resources").getLong("heartbeatPeriod"));
+
+            UpstreamCacheService upstreamCacheService = new UpstreamCacheService(redis, lockService, clock, storage.getPrefix());
+            UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, Random::new, upstreamCacheService);
+
             proxy = new Proxy(vertx, client, configStore, logStore,
                     rateLimiter, upstreamRouteProvider, accessTokenValidator,
                     storage, encryptionService, apiKeyStore, tokenStatsTracker, resourceService, invitationService,
                     shareService, publicationService, accessService, lockService, resourceOperationService, ruleService,
-                    notificationService, applicationService, codeInterpreterService, heartbeatService, version());
+                    notificationService, applicationService, codeInterpreterService, heartbeatService, upstreamCacheService, version());
 
             server = vertx.createHttpServer(new HttpServerOptions(settings("server"))).requestHandler(proxy);
             open(server, HttpServer::listen);
diff --git a/server/src/main/java/com/epam/aidial/core/server/Proxy.java b/server/src/main/java/com/epam/aidial/core/server/Proxy.java
index 50b1eea78..ca85fedca 100644
--- a/server/src/main/java/com/epam/aidial/core/server/Proxy.java
+++ b/server/src/main/java/com/epam/aidial/core/server/Proxy.java
@@ -21,6 +21,7 @@
 import com.epam.aidial.core.server.service.ResourceOperationService;
 import com.epam.aidial.core.server.service.RuleService;
 import com.epam.aidial.core.server.service.ShareService;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import com.epam.aidial.core.server.service.codeinterpreter.CodeInterpreterService;
 import com.epam.aidial.core.server.token.TokenStatsTracker;
 import com.epam.aidial.core.server.upstream.UpstreamRouteProvider;
@@ -67,6 +68,10 @@ public class Proxy implements Handler<HttpServerRequest> {
     public static final String HEADER_UPSTREAM_KEY = "X-UPSTREAM-KEY";
     public static final String HEADER_UPSTREAM_EXTRA_DATA = "X-UPSTREAM-EXTRA-DATA";
     public static final String HEADER_UPSTREAM_ATTEMPTS = "X-UPSTREAM-ATTEMPTS";
+    public static final String HEADER_CACHE_POLICY = "X-DIAL-CACHE-POLICY";
+    public static final String HEADER_CACHE_BREAKPOINT_PATH = "X-DIAL-CACHE-BREAKPOINT-PATH";
+    public static final String HEADER_CACHE_EXPIRE_AT = "X-DIAL-CACHE-EXPIRE-AT";
+    public static final String HEADER_CACHE_EXTRA_METADATA = "X-DIAL-CACHE-EXTRA-METADATA";
     public static final String HEADER_CONTENT_TYPE_APPLICATION_JSON = "application/json";
     public static final String HEADER_APPLICATION_PROPERTIES = "X-DIAL-APPLICATION-PROPERTIES";
     public static final String HEADER_APPLICATION_ID = "X-DIAL-APPLICATION-ID";
@@ -95,6 +100,7 @@ public class Proxy implements Handler<HttpServerRequest> {
     private final ApplicationService applicationService;
     private final CodeInterpreterService codeInterpreterService;
     private final HeartbeatService heartbeatService;
+    private final UpstreamCacheService upstreamCacheService;
     private final String version;
 
     @Override
diff --git a/server/src/main/java/com/epam/aidial/core/server/ProxyContext.java b/server/src/main/java/com/epam/aidial/core/server/ProxyContext.java
index f208935fd..87a4a3456 100644
--- a/server/src/main/java/com/epam/aidial/core/server/ProxyContext.java
+++ b/server/src/main/java/com/epam/aidial/core/server/ProxyContext.java
@@ -5,6 +5,7 @@
 import com.epam.aidial.core.config.Key;
 import com.epam.aidial.core.config.Route;
 import com.epam.aidial.core.server.data.ApiKeyData;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
 import com.epam.aidial.core.server.security.ExtractedClaims;
 import com.epam.aidial.core.server.token.TokenUsage;
 import com.epam.aidial.core.server.upstream.UpstreamRoute;
@@ -97,6 +98,7 @@ public class ProxyContext {
     private String traceOperation;
     // userName to be extracted from JWT or project name belongs to API key
     private String userDisplayName;
+    private CacheBreakpointContext cacheBreakpointContext;
 
     public ProxyContext(Proxy proxy, Config config, HttpServerRequest request, ApiKeyData apiKeyData, ExtractedClaims extractedClaims, String traceId, String spanId) {
         this.proxy = proxy;
diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentPostController.java b/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentPostController.java
index e62ac24a0..cdef16506 100644
--- a/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentPostController.java
+++ b/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentPostController.java
@@ -12,7 +12,9 @@
 import com.epam.aidial.core.server.ProxyContext;
 import com.epam.aidial.core.server.data.ApiKeyData;
 import com.epam.aidial.core.server.data.ErrorData;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
 import com.epam.aidial.core.server.function.BaseRequestFunction;
+import com.epam.aidial.core.server.function.BuildUpstreamCacheFn;
 import com.epam.aidial.core.server.function.CollectRequestApplicationFilesFn;
 import com.epam.aidial.core.server.function.CollectRequestAttachmentsFn;
 import com.epam.aidial.core.server.function.CollectRequestDataFn;
@@ -78,7 +80,8 @@ public DeploymentPostController(Proxy proxy, ProxyContext context) {
                 new ApplyDefaultDeploymentSettingsFn(proxy, context),
                 new EnhanceAssistantRequestFn(proxy, context),
                 new EnhanceModelRequestFn(proxy, context),
-                new CollectRequestApplicationFilesFn(proxy, context));
+                new CollectRequestApplicationFilesFn(proxy, context),
+                new BuildUpstreamCacheFn(proxy, context));
     }
 
     public Future<?> handle(String deploymentId, String deploymentApi) {
@@ -190,13 +193,6 @@ private Future<?> handleRateLimitSuccess() {
                 context.getProject(), context.getDeployment().getName(),
                 context.getRequest().headers().size());
 
-        Deployment deployment = context.getDeployment();
-        UpstreamRoute upstreamRoute = proxy.getUpstreamRouteProvider().get(deployment);
-        if (!canRetry(upstreamRoute)) {
-            return Future.succeededFuture();
-        }
-        context.setUpstreamRoute(upstreamRoute);
-
         setupProxyApiKeyData(new ApiKeyData());
         return proxy.getTokenStatsTracker().startSpan(context).map(ignore -> {
             context.getRequest().body()
@@ -287,6 +283,12 @@ void handleRequestBody(Buffer requestBody) {
             return;
         }
 
+        UpstreamRoute upstreamRoute = proxy.getUpstreamRouteProvider().get(deployment, context.getCacheBreakpointContext());
+        if (!canRetry(upstreamRoute)) {
+            return;
+        }
+        context.setUpstreamRoute(upstreamRoute);
+
         sendRequest();
     }
 
@@ -318,10 +320,12 @@ void handleProxyRequest(HttpClientRequest proxyRequest) {
         proxyRequest.headers().add(Proxy.HEADER_API_KEY, proxyApiKeyData.getPerRequestKey());
 
         if (context.getDeployment() instanceof Model model && !model.getUpstreams().isEmpty()) {
-            Upstream upstream = context.getUpstreamRoute().get();
+            Upstream upstream = Objects.requireNonNull(context.getUpstreamRoute().get());
             proxyRequest.putHeader(Proxy.HEADER_UPSTREAM_ENDPOINT, upstream.getEndpoint());
             proxyRequest.putHeader(Proxy.HEADER_UPSTREAM_KEY, upstream.getKey());
             proxyRequest.putHeader(Proxy.HEADER_UPSTREAM_EXTRA_DATA, upstream.getExtraData());
+            proxyRequest.putHeader(Proxy.HEADER_CACHE_BREAKPOINT_PATH, context.getUpstreamRoute().getBreakpointPath());
+            proxyRequest.putHeader(Proxy.HEADER_CACHE_EXTRA_METADATA, context.getUpstreamRoute().getExtraMetadata());
         }
 
         if ((deployment instanceof Application application && application.hasApplicationTypeSchemaId())) {
@@ -367,7 +371,7 @@ private void handleProxyResponse(HttpClientResponse proxyResponse) {
         }
 
         if (responseStatusCode == 200) {
-            upstreamRoute.succeed();
+            upstreamRoute.succeed(proxyResponse, context.getDeployment());
         } else if (!HttpStatus.fromStatusCode(responseStatusCode).is4xx()) {
             // mark the upstream as failed
             // and the next time we will select another one
diff --git a/server/src/main/java/com/epam/aidial/core/server/data/cache/CacheBreakpointContext.java b/server/src/main/java/com/epam/aidial/core/server/data/cache/CacheBreakpointContext.java
new file mode 100644
index 000000000..284507f6b
--- /dev/null
+++ b/server/src/main/java/com/epam/aidial/core/server/data/cache/CacheBreakpointContext.java
@@ -0,0 +1,6 @@
+package com.epam.aidial.core.server.data.cache;
+
+import java.util.List;
+import java.util.Map;
+
+public record CacheBreakpointContext(List<String> breakpoints, Map<String, String> prefixToHash, CachePolicy policy) {}
diff --git a/server/src/main/java/com/epam/aidial/core/server/data/cache/CachePolicy.java b/server/src/main/java/com/epam/aidial/core/server/data/cache/CachePolicy.java
new file mode 100644
index 000000000..b089a788f
--- /dev/null
+++ b/server/src/main/java/com/epam/aidial/core/server/data/cache/CachePolicy.java
@@ -0,0 +1,34 @@
+package com.epam.aidial.core.server.data.cache;
+
+import com.epam.aidial.core.storage.http.HttpException;
+import com.epam.aidial.core.storage.http.HttpStatus;
+
+public enum CachePolicy {
+    /**
+     * Prioritize service availability over cache hits. We will try to route his request to same upstream, that will provide caching feature.
+     * But in case, when cache upstream in not available, we will fallback to another available upstream, and will create cache there.
+     */
+    AVAILABILITY_PRIORITY("availability-priority"),
+    /**
+     * Prioritize cache hits over service availability. We will route request with cache flag to the same upstream.
+     * Even if cache upstream is not available, we will retry request to the same upstream.
+     */
+    CACHE_PRIORITY("cache-priority");
+
+    private final String val;
+    CachePolicy(String val) {
+        this.val = val;
+    }
+
+    public static CachePolicy fromString(String val) {
+        if (val == null) {
+            return AVAILABILITY_PRIORITY;
+        }
+        for (CachePolicy policy : CachePolicy.values()) {
+            if (policy.val.equals(val)) {
+                return policy;
+            }
+        }
+        throw new HttpException(HttpStatus.BAD_REQUEST, "Invalid cache policy: " + val);
+    }
+}
diff --git a/server/src/main/java/com/epam/aidial/core/server/data/cache/CachedUpstreamEntry.java b/server/src/main/java/com/epam/aidial/core/server/data/cache/CachedUpstreamEntry.java
new file mode 100644
index 000000000..ef9da0529
--- /dev/null
+++ b/server/src/main/java/com/epam/aidial/core/server/data/cache/CachedUpstreamEntry.java
@@ -0,0 +1,4 @@
+package com.epam.aidial.core.server.data.cache;
+
+public record CachedUpstreamEntry(String endpoint, String prefixPath, String extraMetadata) {
+}
diff --git a/server/src/main/java/com/epam/aidial/core/server/function/BuildUpstreamCacheFn.java b/server/src/main/java/com/epam/aidial/core/server/function/BuildUpstreamCacheFn.java
new file mode 100644
index 000000000..c05e61024
--- /dev/null
+++ b/server/src/main/java/com/epam/aidial/core/server/function/BuildUpstreamCacheFn.java
@@ -0,0 +1,39 @@
+package com.epam.aidial.core.server.function;
+
+import com.epam.aidial.core.config.Features;
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.Proxy;
+import com.epam.aidial.core.server.ProxyContext;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class BuildUpstreamCacheFn extends BaseRequestFunction<ObjectNode> {
+
+    private final UpstreamCacheService upstreamCacheService;
+
+    public BuildUpstreamCacheFn(Proxy proxy, ProxyContext context) {
+        super(proxy, context);
+        this.upstreamCacheService = proxy.getUpstreamCacheService();
+    }
+
+    @Override
+    public Boolean apply(ObjectNode body) {
+        if (context.getDeployment() instanceof Model model && isCacheSupported(model)) {
+            CachePolicy policy = CachePolicy.fromString(context.getRequestHeader(Proxy.HEADER_CACHE_POLICY));
+            CacheBreakpointContext cacheBreakpointContext = upstreamCacheService.buildCacheBreakpointContext(body, policy, model);
+            context.setCacheBreakpointContext(cacheBreakpointContext);
+        }
+        return false;
+    }
+
+    private static boolean isCacheSupported(Model model) {
+        Features features = model.getFeatures();
+        if (features == null) {
+            return false;
+        }
+        Boolean cacheSupported = features.getCacheSupported();
+        return cacheSupported != null && cacheSupported;
+    }
+}
diff --git a/server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java b/server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
new file mode 100644
index 000000000..4820a8583
--- /dev/null
+++ b/server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
@@ -0,0 +1,244 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Features;
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.server.util.ProxyUtil;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+    private static final String EXTRA_METADATA_FIELD = "extra_metadata";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD, EXTRA_METADATA_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = isAutoCaching(model);
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            for (int index = 0; index < node.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) sortObjectProperties(node.get(index));
+                for (Map.Entry<String, JsonNode> entry : objectNode.properties()) {
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        JsonNode attachments = entry.getValue().get("attachments");
+                        if (attachments != null && !attachments.isEmpty()) {
+                            messageDigest.update(attachments.toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // model doesn't support caching
+            return null;
+        }
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                RMapAsync<String, String> rmap = batch.getMap(key, REDIS_MAP_CODEC);
+                rmap.isExistsAsync();
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < responses.size(); j++) {
+                Boolean exists = (Boolean) responses.get(j);
+                if (exists) {
+                    int index = i + count - j;
+                    String prefixPath = breakpoints.get(index);
+                    String hash = prefixToHash.get(prefixPath);
+                    String key = getEntryKey(model.getName(), hash);
+                    RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+                    Map<String, String> fields = map.getAll(ALL_FIELDS);
+                    if (!fields.isEmpty()) {
+                        return new CachedUpstreamEntry(fields.get(UPSTREAM_ENDPOINT_FIELD), fields.get(PREFIX_PATH_FIELD), fields.get(EXTRA_METADATA_FIELD));
+                    }
+                }
+            }
+        }
+        // take the last breakpoint
+        String prefixPath = breakpoints.get(breakpoints.size() - 1);
+        return new CachedUpstreamEntry(null, prefixPath, null);
+    }
+
+    public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, String expireAtStr) {
+        String key = getEntryKey(model.getName(), hash);
+        Map<String, String> fields = new HashMap<>();
+        fields.put(UPSTREAM_ENDPOINT_FIELD, entry.endpoint());
+        fields.put(PREFIX_PATH_FIELD, entry.prefixPath());
+        if (entry.extraMetadata() != null) {
+            fields.put(EXTRA_METADATA_FIELD, entry.extraMetadata());
+        }
+        Instant expireAt = expireAt(expireAtStr);
+
+        try (var ignore = lockService.lock(key)) {
+
+            RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+
+            map.putAll(fields);
+            long ttl = map.remainTimeToLive();
+
+            if (expireAt == null && ttl == -1) {
+                expireAt = Instant.ofEpochMilli(clock.getAsLong()).plus(DEFAULT_TTL);
+            }
+
+            if (expireAt != null) {
+                map.expire(expireAt);
+            }
+        }
+    }
+
+    @VisibleForTesting
+    static JsonNode sortObjectProperties(JsonNode node) {
+        if (node.isArray()) {
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int i = 0; i < node.size(); i++) {
+                arrayNode.set(i, sortObjectProperties(arrayNode.get(i)));
+            }
+        } else if (node.isObject()) {
+            ObjectNode result = new ObjectNode(ProxyUtil.MAPPER.getNodeFactory(), new TreeMap<>());
+            for (Map.Entry<String, JsonNode> entry : node.properties()) {
+                result.set(entry.getKey(), sortObjectProperties(entry.getValue()));
+            }
+            node = result;
+        }
+        return node;
+    }
+
+    private boolean isAutoCaching(Model model) {
+        Features features = model.getFeatures();
+        if (features == null) {
+            return false;
+        }
+        Boolean autoCaching = features.getAutoCachingSupported();
+        return autoCaching != null && autoCaching;
+    }
+
+    private static Instant expireAt(String val) {
+        if (val == null) {
+            return null;
+        }
+        try {
+            return Instant.ofEpochSecond(Long.parseLong(val));
+        } catch (NumberFormatException e) {
+            log.error("Invalid expireAt datetime format: " + val);
+            return null;
+        }
+    }
+
+    private static String toString(byte[] digest) {
+        StringBuilder hexString = new StringBuilder();
+        for (byte b : digest) {
+            hexString.append(String.format("%02x", b));
+        }
+        return hexString.toString();
+    }
+
+    private String getEntryKey(String modelName, String hash) {
+        return "upstream_cache:" + BlobStorageUtil.toStoragePath(prefix, BlobStorageUtil.toStoragePath(modelName, hash));
+    }
+
+    @SneakyThrows
+    private static MessageDigest createMessageDigest() {
+        return MessageDigest.getInstance("SHA-1");
+    }
+
+}
diff --git a/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamCacheContext.java b/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamCacheContext.java
new file mode 100644
index 000000000..f6ed72f0f
--- /dev/null
+++ b/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamCacheContext.java
@@ -0,0 +1,16 @@
+package com.epam.aidial.core.server.upstream;
+
+import com.epam.aidial.core.config.Upstream;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import lombok.Data;
+
+import java.util.Map;
+
+@Data
+class UpstreamCacheContext {
+    private CachedUpstreamEntry entry;
+    private CachePolicy policy;
+    private Map<String, String> prefixToHash;
+    private Upstream originalUpstream;
+}
diff --git a/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java b/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java
index 4c9d2f042..cd2046ef2 100644
--- a/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java
+++ b/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java
@@ -1,7 +1,15 @@
 package com.epam.aidial.core.server.upstream;
 
+import com.epam.aidial.core.config.Deployment;
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.config.ModelType;
 import com.epam.aidial.core.config.Upstream;
+import com.epam.aidial.core.server.Proxy;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import com.epam.aidial.core.storage.http.HttpStatus;
+import io.vertx.core.Vertx;
 import io.vertx.core.http.HttpClientResponse;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -40,7 +48,6 @@ public class UpstreamRoute {
     /**
      * Current upstream
      */
-    @Nullable
     private Upstream upstream;
     /**
      * Attempt counter
@@ -50,9 +57,18 @@ public class UpstreamRoute {
 
     private final Set<Upstream> usedUpstreams = new HashSet<>();
 
-    public UpstreamRoute(TieredBalancer balancer, int maxRetryAttempts) {
+    private final UpstreamCacheContext upstreamCacheContext;
+
+    private final UpstreamCacheService upstreamCacheService;
+
+    private final Vertx vertx;
+
+    public UpstreamRoute(Vertx vertx, UpstreamCacheService upstreamCacheService, TieredBalancer balancer, int maxRetryAttempts, UpstreamCacheContext upstreamCacheContext) {
         this.balancer = balancer;
         this.maxRetryAttempts = maxRetryAttempts;
+        this.upstreamCacheContext = upstreamCacheContext;
+        this.vertx = vertx;
+        this.upstreamCacheService = upstreamCacheService;
     }
 
     /**
@@ -77,9 +93,15 @@ public Upstream next() {
             throw balancer.createUpstreamUnavailableException();
         }
         attemptCount++;
-        upstream = balancer.next(usedUpstreams);
-        if (upstream == null) {
-            throw balancer.createUpstreamUnavailableException();
+        if (upstreamCacheContext == null
+                || (upstreamCacheContext.getPolicy() != CachePolicy.CACHE_PRIORITY && attemptCount > 1)
+                || upstreamCacheContext.getOriginalUpstream() == null) {
+            upstream = balancer.next(usedUpstreams);
+            if (upstream == null) {
+                throw balancer.createUpstreamUnavailableException();
+            }
+        } else {
+            upstream = upstreamCacheContext.getOriginalUpstream();
         }
         return upstream;
     }
@@ -114,10 +136,53 @@ void fail(HttpStatus status, long retryAfterSeconds) {
     }
 
     public void succeed() {
+        succeed(null, null);
+    }
+
+    public void succeed(HttpClientResponse proxyResponse, Deployment deployment) {
         verifyCurrentUpstream();
+        if (deployment instanceof Model model) {
+            updateUpstreamCache(proxyResponse, model);
+        }
         balancer.succeed(upstream);
     }
 
+    private void updateUpstreamCache(HttpClientResponse proxyResponse, Model model) {
+        String breakpointPath = proxyResponse.getHeader(Proxy.HEADER_CACHE_BREAKPOINT_PATH);
+        if (breakpointPath == null) {
+            // no cache
+            return;
+        }
+        String hash = upstreamCacheContext.getPrefixToHash().get(breakpointPath);
+        if (hash == null) {
+            if (model.getType() == ModelType.CHAT) {
+                log.warn("prefix is not found: {}", breakpointPath);
+            }
+            return;
+        }
+        String expireAt = proxyResponse.getHeader(Proxy.HEADER_CACHE_EXPIRE_AT);
+        String extraMetadata = proxyResponse.getHeader(Proxy.HEADER_CACHE_EXTRA_METADATA);
+        CachedUpstreamEntry entry = new CachedUpstreamEntry(upstream.getEndpoint(), breakpointPath, extraMetadata);
+        vertx.executeBlocking(() -> {
+            upstreamCacheService.updateEntry(hash, entry, model, expireAt);
+            return null;
+        }, false).onFailure(error -> log.error("Error occurred while updating cached upstream entry", error));
+    }
+
+    public String getBreakpointPath() {
+        CachedUpstreamEntry entry  = getCachedUpstreamEntry();
+        return entry == null ? null : entry.prefixPath();
+    }
+
+    public String getExtraMetadata() {
+        CachedUpstreamEntry entry  = getCachedUpstreamEntry();
+        return entry == null ? null : entry.extraMetadata();
+    }
+
+    private CachedUpstreamEntry getCachedUpstreamEntry() {
+        return upstreamCacheContext == null ? null : upstreamCacheContext.getEntry();
+    }
+
     private void verifyCurrentUpstream() {
         Objects.requireNonNull(upstream, "current upstream is undefined");
     }
diff --git a/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRouteProvider.java b/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRouteProvider.java
index 2d5991248..33b3dbe47 100644
--- a/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRouteProvider.java
+++ b/server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRouteProvider.java
@@ -6,6 +6,9 @@
 import com.epam.aidial.core.config.Model;
 import com.epam.aidial.core.config.Route;
 import com.epam.aidial.core.config.Upstream;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import io.vertx.core.Vertx;
 import lombok.extern.slf4j.Slf4j;
 
@@ -37,23 +40,37 @@ public class UpstreamRouteProvider {
 
     private final Supplier<Random> generatorFactory;
 
-    public UpstreamRouteProvider(Vertx vertx, Supplier<Random> generatorFactory) {
+    private final UpstreamCacheService upstreamCacheService;
+
+    private final Vertx vertx;
+
+    public UpstreamRouteProvider(Vertx vertx, Supplier<Random> generatorFactory, UpstreamCacheService upstreamCacheService) {
         this.generatorFactory = generatorFactory;
+        this.upstreamCacheService = upstreamCacheService;
         vertx.setPeriodic(0, TimeUnit.MINUTES.toMillis(1), event -> evictExpiredBalancers());
+        this.vertx = vertx;
     }
 
-    public UpstreamRoute get(Deployment deployment) {
+    public UpstreamRoute get(Deployment deployment, CacheBreakpointContext breakpointContext) {
         String key = getKey(deployment);
         List<Upstream> upstreams = getUpstreams(deployment);
-        return get(key, upstreams, deployment.getMaxRetryAttempts());
+        UpstreamCacheContext context = null;
+        if (deployment instanceof Model model && breakpointContext != null) {
+            CachedUpstreamEntry entry = upstreamCacheService.getCacheEntry(breakpointContext, model);
+            context = new UpstreamCacheContext();
+            context.setPolicy(breakpointContext.policy());
+            context.setEntry(entry);
+            context.setPrefixToHash(breakpointContext.prefixToHash());
+        }
+        return get(key, upstreams, deployment.getMaxRetryAttempts(), context);
     }
 
     public UpstreamRoute get(Route route) {
         String key = getKey(route);
-        return get(key, route.getUpstreams(), route.getMaxRetryAttempts());
+        return get(key, route.getUpstreams(), route.getMaxRetryAttempts(), null);
     }
 
-    private UpstreamRoute get(String key, List<Upstream> upstreams, int maxRetryAttempts) {
+    private UpstreamRoute get(String key, List<Upstream> upstreams, int maxRetryAttempts, UpstreamCacheContext context) {
         BalancerWrapper wrapper = balancers.compute(key, (k, cur) -> {
             BalancerWrapper result;
             if (cur != null && isUpstreamsTheSame(cur.upstreams, upstreams)
@@ -70,7 +87,22 @@ private UpstreamRoute get(String key, List<Upstream> upstreams, int maxRetryAtte
         if (result <= 0) {
             throw new IllegalArgumentException("max retry attempts must be positive integer");
         }
-        return new UpstreamRoute(wrapper.balancer, result);
+        if (context != null && context.getEntry() != null) {
+            String endpoint = context.getEntry().endpoint();
+            Upstream originalUpstream = null;
+            for (Upstream upstream : upstreams) {
+                if (upstream.getEndpoint().equals(endpoint)) {
+                    originalUpstream = upstream;
+                    break;
+                }
+            }
+            if (originalUpstream != null) {
+                context.setOriginalUpstream(originalUpstream);
+            } else {
+                log.warn("cached upstream doesn't exist any longer in config: {}", endpoint);
+            }
+        }
+        return new UpstreamRoute(vertx, upstreamCacheService, wrapper.balancer, result, context);
     }
 
     private List<Upstream> getUpstreams(Deployment deployment) {
diff --git a/server/src/test/java/com/epam/aidial/core/server/controller/DeploymentPostControllerTest.java b/server/src/test/java/com/epam/aidial/core/server/controller/DeploymentPostControllerTest.java
index 3d9a3eca9..d94c6b1fa 100644
--- a/server/src/test/java/com/epam/aidial/core/server/controller/DeploymentPostControllerTest.java
+++ b/server/src/test/java/com/epam/aidial/core/server/controller/DeploymentPostControllerTest.java
@@ -9,6 +9,8 @@
 import com.epam.aidial.core.server.Proxy;
 import com.epam.aidial.core.server.ProxyContext;
 import com.epam.aidial.core.server.data.ApiKeyData;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
 import com.epam.aidial.core.server.limiter.RateLimiter;
 import com.epam.aidial.core.server.log.LogStore;
 import com.epam.aidial.core.server.security.ApiKeyStore;
@@ -75,7 +77,7 @@ public class DeploymentPostControllerTest {
 
     @Mock
     private ProxyContext context;
-    @Mock
+    @Mock(answer = Answers.RETURNS_DEEP_STUBS)
     private Proxy proxy;
 
     @Mock
@@ -192,7 +194,7 @@ public void testNoRoute() {
         UpstreamRouteProvider balancerProvider = mock(UpstreamRouteProvider.class);
         when(proxy.getUpstreamRouteProvider()).thenReturn(balancerProvider);
         UpstreamRoute endpointRoute = mock(UpstreamRoute.class);
-        when(balancerProvider.get(any(Deployment.class))).thenReturn(endpointRoute);
+        when(balancerProvider.get(any(Deployment.class), any(CacheBreakpointContext.class))).thenReturn(endpointRoute);
         when(endpointRoute.next()).thenThrow(new HttpException(BAD_GATEWAY, "no route"));
         MultiMap headers = mock(MultiMap.class);
         when(request.headers()).thenReturn(headers);
@@ -223,7 +225,7 @@ public void testHandler_Ok() {
         UpstreamRouteProvider balancerProvider = mock(UpstreamRouteProvider.class);
         when(proxy.getUpstreamRouteProvider()).thenReturn(balancerProvider);
         UpstreamRoute endpointRoute = mock(UpstreamRoute.class);
-        when(balancerProvider.get(any(Deployment.class))).thenReturn(endpointRoute);
+        when(balancerProvider.get(any(Deployment.class), any(CacheBreakpointContext.class))).thenReturn(endpointRoute);
         when(endpointRoute.next()).thenReturn(new Upstream());
         MultiMap headers = mock(MultiMap.class);
         when(request.headers()).thenReturn(headers);
@@ -442,11 +444,6 @@ public void testCustomApplication() {
         application.setEndpoint("http://fake.com");
         when(proxy.getVertx()).thenReturn(vertx);
         when(vertx.executeBlocking(any(Callable.class), eq(false))).thenReturn(Future.succeededFuture(application));
-        UpstreamRouteProvider balancerProvider = mock(UpstreamRouteProvider.class);
-        when(proxy.getUpstreamRouteProvider()).thenReturn(balancerProvider);
-        UpstreamRoute endpointRoute = mock(UpstreamRoute.class);
-        when(balancerProvider.get(any(Deployment.class))).thenReturn(endpointRoute);
-        when(endpointRoute.next()).thenReturn(new Upstream());
         MultiMap headers = mock(MultiMap.class);
         when(request.headers()).thenReturn(headers);
         when(context.getDeployment()).thenReturn(application);
diff --git a/server/src/test/java/com/epam/aidial/core/server/data/cache/CachePolicyTest.java b/server/src/test/java/com/epam/aidial/core/server/data/cache/CachePolicyTest.java
new file mode 100644
index 000000000..26a0a00c2
--- /dev/null
+++ b/server/src/test/java/com/epam/aidial/core/server/data/cache/CachePolicyTest.java
@@ -0,0 +1,18 @@
+package com.epam.aidial.core.server.data.cache;
+
+import com.epam.aidial.core.storage.http.HttpException;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class CachePolicyTest {
+
+    @Test
+    public void testFromString() {
+        assertEquals(CachePolicy.AVAILABILITY_PRIORITY, CachePolicy.fromString(null));
+        assertEquals(CachePolicy.AVAILABILITY_PRIORITY, CachePolicy.fromString("availability-priority"));
+        assertEquals(CachePolicy.CACHE_PRIORITY, CachePolicy.fromString("cache-priority"));
+        assertThrows(HttpException.class, () -> CachePolicy.fromString("unknown"));
+    }
+}
diff --git a/server/src/test/java/com/epam/aidial/core/server/function/BuildUpstreamCacheFnTest.java b/server/src/test/java/com/epam/aidial/core/server/function/BuildUpstreamCacheFnTest.java
new file mode 100644
index 000000000..e9fdedec3
--- /dev/null
+++ b/server/src/test/java/com/epam/aidial/core/server/function/BuildUpstreamCacheFnTest.java
@@ -0,0 +1,52 @@
+package com.epam.aidial.core.server.function;
+
+import com.epam.aidial.core.config.Features;
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.Proxy;
+import com.epam.aidial.core.server.ProxyContext;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.util.ProxyUtil;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Answers;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class BuildUpstreamCacheFnTest {
+
+    @Mock(answer = Answers.RETURNS_DEEP_STUBS)
+    private Proxy proxy;
+
+    @Mock
+    private ProxyContext context;
+
+    @InjectMocks
+    private BuildUpstreamCacheFn fn;
+
+    @Test
+    public void testApply_WhenCacheSupported() {
+        Model model = new Model();
+        Features features = new Features();
+        features.setCacheSupported(true);
+        model.setFeatures(features);
+        when(context.getDeployment()).thenReturn(model);
+        when(context.getRequestHeader(eq(Proxy.HEADER_CACHE_POLICY))).thenReturn("cache-priority");
+        ObjectNode objectNode = ProxyUtil.MAPPER.createObjectNode();
+
+        Boolean res = fn.apply(objectNode);
+
+        assertFalse(res);
+        verify(proxy.getUpstreamCacheService()).buildCacheBreakpointContext(eq(objectNode), eq(CachePolicy.CACHE_PRIORITY), eq(model));
+        verify(context).setCacheBreakpointContext(any(CacheBreakpointContext.class));
+    }
+}
diff --git a/server/src/test/java/com/epam/aidial/core/server/service/UpstreamCacheServiceTest.java b/server/src/test/java/com/epam/aidial/core/server/service/UpstreamCacheServiceTest.java
new file mode 100644
index 000000000..acaf51ae1
--- /dev/null
+++ b/server/src/test/java/com/epam/aidial/core/server/service/UpstreamCacheServiceTest.java
@@ -0,0 +1,491 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Features;
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.server.util.ProxyUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.redisson.Redisson;
+import org.redisson.api.RedissonClient;
+import org.redisson.config.ConfigSupport;
+import redis.embedded.RedisServer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class UpstreamCacheServiceTest {
+
+    private static RedisServer redisServer;
+
+    private static RedissonClient redissonClient;
+
+    private LockService lockService;
+
+    private UpstreamCacheService service;
+
+    @BeforeAll
+    public static void beforeAll() throws IOException {
+        redisServer = RedisServer.newRedisServer()
+                .port(16370)
+                .bind("127.0.0.1")
+                .setting("maxmemory 16M")
+                .setting("maxmemory-policy volatile-lfu")
+                .build();
+        redisServer.start();
+        ConfigSupport configSupport = new ConfigSupport();
+        org.redisson.config.Config redisClientConfig = configSupport.fromJSON("""
+                {
+                  "singleServerConfig": {
+                     "address": "redis://localhost:16370"
+                  }
+                }
+                """, org.redisson.config.Config.class);
+
+        redissonClient = Redisson.create(redisClientConfig);
+    }
+
+    @AfterAll
+    public static void afterAll() throws IOException {
+        if (redissonClient != null) {
+            redissonClient.shutdown();
+        }
+        if (redisServer != null) {
+            redisServer.stop();
+        }
+    }
+
+    @BeforeEach
+    public void beforeEach() {
+        lockService = new LockService(redissonClient, null);
+    }
+
+    @Test
+    public void testUpdateEntry() {
+        service = new UpstreamCacheService(redissonClient, lockService, System::currentTimeMillis, null);
+
+        service.updateEntry("hash", new CachedUpstreamEntry("http://localhost:8080/chat", "prefix.body.messages[1]", null), new Model(), null);
+
+        assertTrue(redissonClient.getKeys().getKeys().iterator().hasNext());
+    }
+
+    @Test
+    public void testBuildCacheBreakpointContext_withBreakpoints() throws JsonProcessingException {
+        service = new UpstreamCacheService(redissonClient, lockService, System::currentTimeMillis, null);
+        String body = """
+                {
+                    "tools": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function",
+                                "description": "desc"
+                            }
+                        },
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function_1",
+                                "description": "...",
+                                "args": {
+                                    "type": "object",
+                                    "properties": {
+                                        "a": {
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "custom_fields": {
+                                "cache_breakpoint": {},
+                                "some_random_key": "some_random_value"
+                            }
+                        }
+                    ],
+                    "messages": [
+                        {
+                            "role": "system",
+                            "content": "System prompt",
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-02T15:01:23Z"
+                                }
+                            }
+                        },
+                        {
+                            "role": "user",
+                            "content": "Here is a file, say hello",
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-03T15:01:23Z"
+                                }
+                            },
+                            "custom_content": {
+                                "state": {
+                                    "some_field": "some_value",
+                                    "some_random_key": "some_random_value"
+                                },
+                                "attachments": [
+                                    {
+                                        "url": "URL"
+                                    },
+                                    {
+                                        "data": "long long data..."
+                                    }
+                                ]
+                            }
+                        },
+                        {
+                            "role": "assistant",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": "..."
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": "..."
+                                    }
+                                }
+                            ],
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-04T15:01:23Z"
+                                }
+                            }
+                        }
+                    ]
+                }
+                """;
+        ObjectNode objectNode = (ObjectNode) ProxyUtil.MAPPER.readTree(body);
+        Model model = new Model();
+        model.setName("gpt-4");
+
+        CacheBreakpointContext context = service.buildCacheBreakpointContext(objectNode, CachePolicy.AVAILABILITY_PRIORITY, model);
+
+        assertNotNull(context);
+        assertEquals(4, context.breakpoints().size());
+        List<String> expectedBreakpoints = List.of("prefix.body.tools[1]", "prefix.body.messages[0]", "prefix.body.messages[1]", "prefix.body.messages[2]");
+        assertEquals(expectedBreakpoints, context.breakpoints());
+        assertEquals(5, context.prefixToHash().size());
+        assertEquals(CachePolicy.AVAILABILITY_PRIORITY, context.policy());
+    }
+
+    @Test
+    public void testBuildCacheBreakpointContext_autoCaching() throws JsonProcessingException {
+        service = new UpstreamCacheService(redissonClient, lockService, System::currentTimeMillis, null);
+        String body = """
+                {
+                    "tools": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function",
+                                "description": "desc"
+                            }
+                        }
+                    ],
+                    "messages": [
+                        {
+                            "role": "system",
+                            "content": "System prompt"
+                        },
+                        {
+                            "role": "user",
+                            "content": "Here is a file, say hello",
+                            "custom_content": {
+                                "state": {
+                                    "some_field": "some_value",
+                                    "some_random_key": "some_random_value"
+                                },
+                                "attachments": [
+                                    {
+                                        "url": "URL"
+                                    },
+                                    {
+                                        "data": "long long data..."
+                                    }
+                                ]
+                            }
+                        },
+                        {
+                            "role": "assistant",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": "..."
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": "..."
+                                    }
+                                }
+                            ]
+                        }
+                    ]
+                }
+                """;
+        ObjectNode objectNode = (ObjectNode) ProxyUtil.MAPPER.readTree(body);
+        Model model = new Model();
+        model.setName("gpt-4");
+        Features features = new Features();
+        features.setAutoCachingSupported(true);
+        model.setFeatures(features);
+
+        CacheBreakpointContext context = service.buildCacheBreakpointContext(objectNode, CachePolicy.AVAILABILITY_PRIORITY, model);
+
+        assertNotNull(context);
+        assertEquals(4, context.breakpoints().size());
+        List<String> expectedBreakpoints = List.of("prefix.body.tools[0]", "prefix.body.messages[0]", "prefix.body.messages[1]", "prefix.body.messages[2]");
+        assertEquals(expectedBreakpoints, context.breakpoints());
+        assertEquals(4, context.prefixToHash().size());
+        assertEquals(CachePolicy.AVAILABILITY_PRIORITY, context.policy());
+    }
+
+    @Test
+    public void testGetCacheEntry_entryFound() throws JsonProcessingException {
+        service = new UpstreamCacheService(redissonClient, lockService, System::currentTimeMillis, null);
+        String body = """
+                {
+                    "tools": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function",
+                                "description": "desc"
+                            }
+                        },
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function_1",
+                                "description": "...",
+                                "args": {
+                                    "type": "object",
+                                    "properties": {
+                                        "a": {
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "custom_fields": {
+                                "cache_breakpoint": {},
+                                "some_random_key": "some_random_value"
+                            }
+                        }
+                    ],
+                    "messages": [
+                        {
+                            "role": "system",
+                            "content": "System prompt",
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-02T15:01:23Z"
+                                }
+                            }
+                        },
+                        {
+                            "role": "user",
+                            "content": "Here is a file, say hello",
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-03T15:01:23Z"
+                                }
+                            },
+                            "custom_content": {
+                                "state": {
+                                    "some_field": "some_value",
+                                    "some_random_key": "some_random_value"
+                                },
+                                "attachments": [
+                                    {
+                                        "url": "URL"
+                                    },
+                                    {
+                                        "data": "long long data..."
+                                    }
+                                ]
+                            }
+                        },
+                        {
+                            "role": "assistant",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": "..."
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": "..."
+                                    }
+                                }
+                            ],
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-04T15:01:23Z"
+                                }
+                            }
+                        }
+                    ]
+                }
+                """;
+        ObjectNode objectNode = (ObjectNode) ProxyUtil.MAPPER.readTree(body);
+        Model model = new Model();
+        model.setName("gpt-4");
+
+        CacheBreakpointContext context = service.buildCacheBreakpointContext(objectNode, CachePolicy.AVAILABILITY_PRIORITY, model);
+        Map<String, String> prefixToHash = context.prefixToHash();
+        for (var breakpoint : context.breakpoints()) {
+            CachedUpstreamEntry cachedUpstreamEntry = new CachedUpstreamEntry("http://host/chat", breakpoint, null);
+            service.updateEntry(prefixToHash.get(breakpoint), cachedUpstreamEntry, model, null);
+        }
+
+        CachedUpstreamEntry entry = service.getCacheEntry(context, model);
+        assertNotNull(entry);
+        assertEquals("prefix.body.messages[2]", entry.prefixPath());
+        assertEquals("http://host/chat", entry.endpoint());
+    }
+
+    @Test
+    public void testGetCacheEntry_entryNotFound() throws JsonProcessingException {
+        service = new UpstreamCacheService(redissonClient, lockService, System::currentTimeMillis, null);
+        String body = """
+                {
+                    "tools": [
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function",
+                                "description": "desc"
+                            }
+                        },
+                        {
+                            "type": "function",
+                            "function": {
+                                "name": "some_long_function_1",
+                                "description": "...",
+                                "args": {
+                                    "type": "object",
+                                    "properties": {
+                                        "a": {
+                                            "type": "string"
+                                        }
+                                    }
+                                }
+                            },
+                            "custom_fields": {
+                                "cache_breakpoint": {},
+                                "some_random_key": "some_random_value"
+                            }
+                        }
+                    ],
+                    "messages": [
+                        {
+                            "role": "system",
+                            "content": "System prompt",
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-02T15:01:23Z"
+                                }
+                            }
+                        },
+                        {
+                            "role": "user",
+                            "content": "Here is a file, say hello",
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-03T15:01:23Z"
+                                }
+                            },
+                            "custom_content": {
+                                "state": {
+                                    "some_field": "some_value",
+                                    "some_random_key": "some_random_value"
+                                },
+                                "attachments": [
+                                    {
+                                        "url": "URL"
+                                    },
+                                    {
+                                        "data": "long long data..."
+                                    }
+                                ]
+                            }
+                        },
+                        {
+                            "role": "assistant",
+                            "content": [
+                                {
+                                    "type": "text",
+                                    "text": "..."
+                                },
+                                {
+                                    "type": "image_url",
+                                    "image_url": {
+                                        "url": "..."
+                                    }
+                                }
+                            ],
+                            "custom_fields": {
+                                "cache_breakpoint": {
+                                    "expire_at": "2025-10-04T15:01:23Z"
+                                }
+                            }
+                        }
+                    ]
+                }
+                """;
+        ObjectNode objectNode = (ObjectNode) ProxyUtil.MAPPER.readTree(body);
+        Model model = new Model();
+        model.setName("gpt-4");
+
+        CacheBreakpointContext context = service.buildCacheBreakpointContext(objectNode, CachePolicy.AVAILABILITY_PRIORITY, model);
+
+        CachedUpstreamEntry entry = service.getCacheEntry(context, model);
+        assertNotNull(entry);
+        assertEquals("prefix.body.messages[2]", entry.prefixPath());
+        assertNull(entry.endpoint());
+    }
+
+    @Test
+    public void testSortObjectProperties_simple() throws JsonProcessingException {
+        String simpleJson = """
+                 {
+                   "d" : {
+                     "b" : 1,
+                     "a": 2
+                   },
+                   "c": [
+                     {"d": true,"a": "text"},
+                     {"z": 3, "a": false}
+                   ],
+                   "a" : {
+                     "z" : 1,
+                     "f": 2
+                   }
+                 }
+                 """;
+        JsonNode node = ProxyUtil.MAPPER.readTree(simpleJson);
+        JsonNode result = UpstreamCacheService.sortObjectProperties(node);
+        assertEquals("""
+                 {"a":{"f":2,"z":1},"c":[{"a":"text","d":true},{"a":false,"z":3}],"d":{"a":2,"b":1}}""", result.toString());
+    }
+}
diff --git a/server/src/test/java/com/epam/aidial/core/server/upstream/TieredBalancerTest.java b/server/src/test/java/com/epam/aidial/core/server/upstream/TieredBalancerTest.java
index 84bb26fed..d61800bc1 100644
--- a/server/src/test/java/com/epam/aidial/core/server/upstream/TieredBalancerTest.java
+++ b/server/src/test/java/com/epam/aidial/core/server/upstream/TieredBalancerTest.java
@@ -2,6 +2,9 @@
 
 import com.epam.aidial.core.config.Model;
 import com.epam.aidial.core.config.Upstream;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import com.epam.aidial.core.storage.http.HttpException;
 import com.epam.aidial.core.storage.http.HttpStatus;
 import io.vertx.core.Vertx;
@@ -12,6 +15,7 @@
 
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -30,6 +34,9 @@ public class TieredBalancerTest {
     @Mock
     private Vertx vertx;
 
+    @Mock
+    private UpstreamCacheService upstreamCacheService;
+
     @Mock
     private Random generator;
 
@@ -114,9 +121,11 @@ void testUpstreamFallback() {
         when(generator.nextInt(4)).thenAnswer(cb -> counter.incrementAndGet());
         Supplier<Random> factory = () -> generator;
 
-        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, factory);
+        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, factory, upstreamCacheService);
+
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
 
-        UpstreamRoute route1 = upstreamRouteProvider.get(model);
+        UpstreamRoute route1 = upstreamRouteProvider.get(model, cacheBreakpointContext);
         assertNotNull(route1.next());
         assertEquals(upstreams.get(0), route1.get());
         route1.fail(HttpStatus.SERVICE_UNAVAILABLE, -1);
@@ -127,7 +136,7 @@ void testUpstreamFallback() {
         assertEquals(upstreams.get(3), route1.next());
         route1.fail(HttpStatus.TOO_MANY_REQUESTS, 5);
 
-        UpstreamRoute route2 = upstreamRouteProvider.get(model);
+        UpstreamRoute route2 = upstreamRouteProvider.get(model, cacheBreakpointContext);
         assertNotNull(route2.next());
         assertEquals(upstreams.get(0), route2.get());
         route2.fail(HttpStatus.SERVICE_UNAVAILABLE, -1);
diff --git a/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteProviderTest.java b/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteProviderTest.java
index a015d1190..8ff28f04a 100644
--- a/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteProviderTest.java
+++ b/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteProviderTest.java
@@ -3,6 +3,10 @@
 import com.epam.aidial.core.config.Application;
 import com.epam.aidial.core.config.Model;
 import com.epam.aidial.core.config.Upstream;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import com.epam.aidial.core.storage.http.HttpException;
 import com.epam.aidial.core.storage.http.HttpStatus;
 import io.vertx.core.Vertx;
@@ -12,11 +16,15 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
 public class UpstreamRouteProviderTest {
@@ -24,20 +32,23 @@ public class UpstreamRouteProviderTest {
     @Mock
     private Vertx vertx;
 
+    @Mock
+    private UpstreamCacheService upstreamCacheService;
+
     @Mock
     private Random generator;
 
     @Test
     public void testGet_UpstreamsNotChanged() {
-        UpstreamRouteProvider provider = new UpstreamRouteProvider(vertx, () -> generator);
+        UpstreamRouteProvider provider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
         Application application = new Application();
         application.setName("app");
-        UpstreamRoute route1 = provider.get(application);
+        UpstreamRoute route1 = provider.get(application, null);
         route1.next();
         route1.fail(HttpStatus.TOO_MANY_REQUESTS);
         assertThrows(HttpException.class, route1::next);
         // make sure new router doesn't have any upstreams for the same application
-        UpstreamRoute route2 = provider.get(application);
+        UpstreamRoute route2 = provider.get(application, null);
         assertNotNull(route2.next());
         assertThrows(HttpException.class, route2::next);
     }
@@ -52,8 +63,9 @@ public void testGet_UpstreamsChanged() {
         upstream1.setWeight(2);
         model.setUpstreams(List.of(upstream1));
 
-        UpstreamRouteProvider provider = new UpstreamRouteProvider(vertx, () -> generator);
-        UpstreamRoute route1 = provider.get(model);
+        UpstreamRouteProvider provider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
+        UpstreamRoute route1 = provider.get(model, cacheBreakpointContext);
         route1.next();
         route1.fail(HttpStatus.TOO_MANY_REQUESTS);
         assertThrows(HttpException.class, route1::next);
@@ -64,9 +76,59 @@ public void testGet_UpstreamsChanged() {
         upstream2.setWeight(1);
         model.setUpstreams(List.of(upstream2));
         // change upstreams in the model
-        UpstreamRoute route2 = provider.get(model);
+        UpstreamRoute route2 = provider.get(model, cacheBreakpointContext);
         route2.next();
         // the upstream is found
         assertTrue(route2.available());
     }
+
+    @Test
+    public void testGet_CachedUpstreamPresent() {
+        Model model = new Model();
+        model.setName("model");
+        Upstream upstream1 = new Upstream();
+        upstream1.setEndpoint("upstream1");
+        upstream1.setTier(0);
+        upstream1.setWeight(2);
+        Upstream upstream2 = new Upstream();
+        upstream2.setEndpoint("upstream2");
+        upstream2.setTier(1);
+        upstream2.setWeight(2);
+        model.setUpstreams(List.of(upstream1, upstream2));
+
+        UpstreamRouteProvider provider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
+        CachedUpstreamEntry entry = new CachedUpstreamEntry("upstream2", "prefix", null);
+        when(upstreamCacheService.getCacheEntry(eq(cacheBreakpointContext), eq(model))).thenReturn(entry);
+
+        UpstreamRoute route1 = provider.get(model, cacheBreakpointContext);
+        Upstream result = route1.next();
+
+        assertEquals(upstream2, result);
+    }
+
+    @Test
+    public void testGet_CachedUpstreamMissed() {
+        Model model = new Model();
+        model.setName("model");
+        Upstream upstream1 = new Upstream();
+        upstream1.setEndpoint("upstream1");
+        upstream1.setTier(0);
+        upstream1.setWeight(2);
+        Upstream upstream2 = new Upstream();
+        upstream2.setEndpoint("upstream2");
+        upstream2.setTier(1);
+        upstream2.setWeight(2);
+        model.setUpstreams(List.of(upstream1, upstream2));
+
+        UpstreamRouteProvider provider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
+        CachedUpstreamEntry entry = new CachedUpstreamEntry("test", "prefix", null);
+        when(upstreamCacheService.getCacheEntry(eq(cacheBreakpointContext), eq(model))).thenReturn(entry);
+
+        UpstreamRoute route1 = provider.get(model, cacheBreakpointContext);
+        Upstream result = route1.next();
+
+        assertEquals(upstream1, result);
+    }
 }
diff --git a/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteTest.java b/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteTest.java
index f80fbe06b..0f2e4f46d 100644
--- a/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteTest.java
+++ b/server/src/test/java/com/epam/aidial/core/server/upstream/UpstreamRouteTest.java
@@ -2,16 +2,25 @@
 
 import com.epam.aidial.core.config.Model;
 import com.epam.aidial.core.config.Upstream;
+import com.epam.aidial.core.server.Proxy;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.server.service.UpstreamCacheService;
 import com.epam.aidial.core.storage.http.HttpException;
 import com.epam.aidial.core.storage.http.HttpStatus;
+import io.vertx.core.Future;
 import io.vertx.core.Vertx;
+import io.vertx.core.http.HttpClientResponse;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
 
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.Callable;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -19,6 +28,12 @@
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
 
 @ExtendWith(MockitoExtension.class)
 public class UpstreamRouteTest {
@@ -26,6 +41,9 @@ public class UpstreamRouteTest {
     @Mock
     private Vertx vertx;
 
+    @Mock
+    private UpstreamCacheService upstreamCacheService;
+
     @Mock
     private Random generator;
 
@@ -40,8 +58,9 @@ void testUpstreamRouteWithRetry() {
                 new Upstream("endpoint4", null, null, 1, 1)
         ));
 
-        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator);
-        UpstreamRoute route = upstreamRouteProvider.get(model);
+        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
+        UpstreamRoute route = upstreamRouteProvider.get(model, cacheBreakpointContext);
         assertNotNull(route.next());
 
         assertTrue(route.available());
@@ -87,8 +106,9 @@ void testUpstreamRouteWithRetry2() {
                 new Upstream("endpoint2", null, null, 1, 1)
         ));
 
-        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator);
-        UpstreamRoute route = upstreamRouteProvider.get(model);
+        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
+        UpstreamRoute route = upstreamRouteProvider.get(model, cacheBreakpointContext);
         assertNotNull(route.next());
 
         assertTrue(route.available());
@@ -109,4 +129,85 @@ void testUpstreamRouteWithRetry2() {
         assertNull(route.get());
         assertEquals(2, route.getAttemptCount());
     }
+
+    @Test
+    void testUpstreamRouteWithCachedUpstream() {
+        Model model = new Model();
+        model.setName("model1");
+        model.setUpstreams(List.of(
+                new Upstream("endpoint1", null, null, 1, 1),
+                new Upstream("endpoint2", null, null, 1, 1)
+        ));
+
+        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.AVAILABILITY_PRIORITY);
+        CachedUpstreamEntry entry = new CachedUpstreamEntry("endpoint2", "prefix", null);
+        when(upstreamCacheService.getCacheEntry(eq(cacheBreakpointContext), eq(model))).thenReturn(entry);
+        UpstreamRoute route = upstreamRouteProvider.get(model, cacheBreakpointContext);
+        assertNotNull(route.next());
+
+        assertTrue(route.available());
+        assertEquals(model.getUpstreams().get(1), route.get());
+        assertEquals(1, route.getAttemptCount());
+
+        route.fail(HttpStatus.TOO_MANY_REQUESTS, 30);
+        assertEquals(model.getUpstreams().get(0), route.next());
+    }
+
+    @Test
+    void testUpstreamRouteWithCachedUpstream_CachePriority() {
+        Model model = new Model();
+        model.setName("model1");
+        model.setUpstreams(List.of(
+                new Upstream("endpoint1", null, null, 1, 1),
+                new Upstream("endpoint2", null, null, 1, 1)
+        ));
+
+        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of(), Map.of(), CachePolicy.CACHE_PRIORITY);
+        CachedUpstreamEntry entry = new CachedUpstreamEntry("endpoint2", "prefix", null);
+        when(upstreamCacheService.getCacheEntry(eq(cacheBreakpointContext), eq(model))).thenReturn(entry);
+        UpstreamRoute route = upstreamRouteProvider.get(model, cacheBreakpointContext);
+        assertNotNull(route.next());
+
+        assertTrue(route.available());
+        assertEquals(model.getUpstreams().get(1), route.get());
+        assertEquals(1, route.getAttemptCount());
+
+        route.fail(HttpStatus.TOO_MANY_REQUESTS, 30);
+        assertEquals(model.getUpstreams().get(1), route.next());
+    }
+
+    @Test
+    void testSuccess_UpstreamCache() {
+        Model model = new Model();
+        model.setName("model1");
+        model.setUpstreams(List.of(
+                new Upstream("endpoint1", null, null, 1, 1),
+                new Upstream("endpoint2", null, null, 1, 1)
+        ));
+
+        UpstreamRouteProvider upstreamRouteProvider = new UpstreamRouteProvider(vertx, () -> generator, upstreamCacheService);
+        CacheBreakpointContext cacheBreakpointContext = new CacheBreakpointContext(List.of("prefix"), Map.of("prefix", "hash"), CachePolicy.CACHE_PRIORITY);
+        CachedUpstreamEntry entry = new CachedUpstreamEntry("endpoint2", "prefix", null);
+        when(upstreamCacheService.getCacheEntry(eq(cacheBreakpointContext), eq(model))).thenReturn(entry);
+        UpstreamRoute route = upstreamRouteProvider.get(model, cacheBreakpointContext);
+        assertNotNull(route.next());
+
+        assertTrue(route.available());
+        assertEquals(model.getUpstreams().get(1), route.get());
+        assertEquals(1, route.getAttemptCount());
+
+        HttpClientResponse response = mock(HttpClientResponse.class);
+        when(response.getHeader(Proxy.HEADER_CACHE_BREAKPOINT_PATH)).thenReturn("prefix");
+        when(vertx.executeBlocking(any(Callable.class), eq(false))).thenAnswer(invocation -> {
+            Callable callable = invocation.getArgument(0);
+            callable.call();
+            return Future.succeededFuture();
+        });
+
+        route.succeed(response, model);
+
+        verify(upstreamCacheService).updateEntry(anyString(), any(CachedUpstreamEntry.class), eq(model), any());
+    }
 }


Reviews:
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T13:33:41Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T13:39:13Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T13:43:07Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T14:36:03Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T14:50:25Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-14T14:51:43Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T12:30:34Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T12:31:05Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T12:32:57Z
user: login: artsiomkorzun
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T12:45:59Z
user: login: artsiomkorzun
state: DISMISSED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T13:52:46Z
user: login: adubovik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:03:26Z
user: login: adubovik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:24:40Z
user: login: adubovik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:25:47Z
user: login: adubovik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T11:25:02Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T12:48:06Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T12:48:54Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T15:02:00Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T15:02:09Z
user: login: adubovik
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T15:44:48Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T16:27:33Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-18T16:49:05Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T08:44:08Z
user: login: astsiapanay
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-19T08:45:43Z
user: login: adubovik
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T09:49:22Z

Review comments:
diff_hunk: @@ -0,0 +1,194 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final Set<Object> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // embedding request
+            return null;
+        }
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                batch.getMap(key, REDIS_MAP_CODEC).getAllAsync(ALL_FIELDS);
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < count; j++) {
+                Map<String, String> map = (Map<String, String>) responses.get(j);
+                if (!map.isEmpty()) {
+                    return new CachedUpstreamEntry(map.get(UPSTREAM_ENDPOINT_FIELD), map.get(PREFIX_PATH_FIELD));
+                }
+            }
+        }
+        // take the last breakpoint
+        String prefixPath = breakpoints.get(breakpoints.size() - 1);
+        return new CachedUpstreamEntry(null, prefixPath);
+    }
+
+    public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, String expireAtStr) {
+        String key = getEntryKey(model.getName(), hash);
+        Map<String, String> fields = new HashMap<>();
+        fields.put(UPSTREAM_ENDPOINT_FIELD, entry.endpoint());
+        fields.put(PREFIX_PATH_FIELD, entry.prefixPath());
+        try (var ignore = lockService.lock(key)) {
+
+            RMap<String, String> map = redisClient.getMap(key);
+            boolean exists = map.isExists();
+
+            map.putAll(fields);
+
+            Instant expireAt = expireAt(expireAtStr);
+            if (expireAt == null && exists) {
+                // adapter didn't return expireAt for a new cache entry
+                expireAt = Instant.now().plus(DEFAULT_TTL);
+            }
+            if (expireAt != null) {
+                map.expire(expireAt);
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: Why don't you always reset expiration?
created_at: 2025-03-14T13:33:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,194 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final Set<Object> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // embedding request
+            return null;
+        }
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                batch.getMap(key, REDIS_MAP_CODEC).getAllAsync(ALL_FIELDS);
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < count; j++) {
+                Map<String, String> map = (Map<String, String>) responses.get(j);
+                if (!map.isEmpty()) {
+                    return new CachedUpstreamEntry(map.get(UPSTREAM_ENDPOINT_FIELD), map.get(PREFIX_PATH_FIELD));
+                }
+            }
+        }
+        // take the last breakpoint
+        String prefixPath = breakpoints.get(breakpoints.size() - 1);
+        return new CachedUpstreamEntry(null, prefixPath);
+    }
+
+    public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, String expireAtStr) {
+        String key = getEntryKey(model.getName(), hash);
+        Map<String, String> fields = new HashMap<>();
+        fields.put(UPSTREAM_ENDPOINT_FIELD, entry.endpoint());
+        fields.put(PREFIX_PATH_FIELD, entry.prefixPath());
+        try (var ignore = lockService.lock(key)) {
+
+            RMap<String, String> map = redisClient.getMap(key);
+            boolean exists = map.isExists();
+
+            map.putAll(fields);
+
+            Instant expireAt = expireAt(expireAtStr);
+            if (expireAt == null && exists) {
+                // adapter didn't return expireAt for a new cache entry
+                expireAt = Instant.now().plus(DEFAULT_TTL);
+            }
+            if (expireAt != null) {
+                map.expire(expireAt);
+            }
+        }
+    }
+
+    private static Instant expireAt(String val) {
+        if (val == null) {
+            return null;
+        }
+        try {
+            return Instant.ofEpochSecond(Long.parseLong(val));
+        } catch (NumberFormatException e) {
+            log.error("Invalid expireAt datetime format: " + val);
+            return null;
+        }
+    }
+
+    private static String toString(byte[] digest) {
+        StringBuilder hexString = new StringBuilder();
+        for (byte b : digest) {
+            hexString.append(String.format("%02x", b));
+        }
+        return hexString.toString();
+    }
+
+    private String getEntryKey(String modelName, String hash) {
+        return prefix + ":" + modelName + ":" + hash;
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: Please use the same way of how we build prefix. Something like:
```
"upstream:" + BlobStorageUtil.toStoragePath(prefix, BlobStorageUtil.toStoragePath(modelName, hash))
```
created_at: 2025-03-14T13:39:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -77,9 +93,15 @@ public Upstream next() {
             throw balancer.createUpstreamUnavailableException();
         }
         attemptCount++;
-        upstream = balancer.next(usedUpstreams);
-        if (upstream == null) {
-            throw balancer.createUpstreamUnavailableException();
+        if (upstreamCacheContext == null
+                || (upstreamCacheContext.getPolicy() != CachePolicy.CACHE_PRIORITY && attemptCount > 1)
+                || upstreamCacheContext.getOriginalUpstream() == null) {
+            upstream = balancer.next(usedUpstreams);
+            if (upstream == null) {
+                throw balancer.createUpstreamUnavailableException();
+            }
+        } else {
+            upstream = upstreamCacheContext.getOriginalUpstream();
path: server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java
user: login: artsiomkorzun
body: Where do you check that the upstream is still present in the config?
created_at: 2025-03-14T13:43:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,194 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final Set<Object> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // embedding request
+            return null;
+        }
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                batch.getMap(key, REDIS_MAP_CODEC).getAllAsync(ALL_FIELDS);
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < count; j++) {
+                Map<String, String> map = (Map<String, String>) responses.get(j);
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: We support volatile cache policy, that means that Redis could return stale entries with expired ttl.
created_at: 2025-03-14T14:36:03Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,194 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final Set<Object> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // embedding request
+            return null;
+        }
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                batch.getMap(key, REDIS_MAP_CODEC).getAllAsync(ALL_FIELDS);
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < count; j++) {
+                Map<String, String> map = (Map<String, String>) responses.get(j);
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: fixed
created_at: 2025-03-14T14:50:25Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,194 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final Set<Object> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // embedding request
+            return null;
+        }
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                batch.getMap(key, REDIS_MAP_CODEC).getAllAsync(ALL_FIELDS);
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < count; j++) {
+                Map<String, String> map = (Map<String, String>) responses.get(j);
+                if (!map.isEmpty()) {
+                    return new CachedUpstreamEntry(map.get(UPSTREAM_ENDPOINT_FIELD), map.get(PREFIX_PATH_FIELD));
+                }
+            }
+        }
+        // take the last breakpoint
+        String prefixPath = breakpoints.get(breakpoints.size() - 1);
+        return new CachedUpstreamEntry(null, prefixPath);
+    }
+
+    public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, String expireAtStr) {
+        String key = getEntryKey(model.getName(), hash);
+        Map<String, String> fields = new HashMap<>();
+        fields.put(UPSTREAM_ENDPOINT_FIELD, entry.endpoint());
+        fields.put(PREFIX_PATH_FIELD, entry.prefixPath());
+        try (var ignore = lockService.lock(key)) {
+
+            RMap<String, String> map = redisClient.getMap(key);
+            boolean exists = map.isExists();
+
+            map.putAll(fields);
+
+            Instant expireAt = expireAt(expireAtStr);
+            if (expireAt == null && exists) {
+                // adapter didn't return expireAt for a new cache entry
+                expireAt = Instant.now().plus(DEFAULT_TTL);
+            }
+            if (expireAt != null) {
+                map.expire(expireAt);
+            }
+        }
+    }
+
+    private static Instant expireAt(String val) {
+        if (val == null) {
+            return null;
+        }
+        try {
+            return Instant.ofEpochSecond(Long.parseLong(val));
+        } catch (NumberFormatException e) {
+            log.error("Invalid expireAt datetime format: " + val);
+            return null;
+        }
+    }
+
+    private static String toString(byte[] digest) {
+        StringBuilder hexString = new StringBuilder();
+        for (byte b : digest) {
+            hexString.append(String.format("%02x", b));
+        }
+        return hexString.toString();
+    }
+
+    private String getEntryKey(String modelName, String hash) {
+        return prefix + ":" + modelName + ":" + hash;
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: fixed
created_at: 2025-03-14T14:51:43Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,219 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final String EXPIRED_AT_FIELD = "expired_at";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD, EXPIRED_AT_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // model doesn't support caching
+            return null;
+        }
+        long currentTime = clock.getAsLong();
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                RMapAsync<String, String> rmap = batch.getMap(key, REDIS_MAP_CODEC);
+                rmap.isExistsAsync();
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < responses.size(); j++) {
+                Boolean exists = (Boolean) responses.get(j);
+                if (exists) {
+                    int index = i + count - j;
+                    String prefixPath = breakpoints.get(index);
+                    String hash = prefixToHash.get(prefixPath);
+                    String key = getEntryKey(model.getName(), hash);
+                    RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+                    Map<String, String> fields = map.getAll(ALL_FIELDS);
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: It could be already missing, it might be evicted between two calls 
created_at: 2025-03-17T12:30:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,219 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final String EXPIRED_AT_FIELD = "expired_at";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD, EXPIRED_AT_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // model doesn't support caching
+            return null;
+        }
+        long currentTime = clock.getAsLong();
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                RMapAsync<String, String> rmap = batch.getMap(key, REDIS_MAP_CODEC);
+                rmap.isExistsAsync();
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < responses.size(); j++) {
+                Boolean exists = (Boolean) responses.get(j);
+                if (exists) {
+                    int index = i + count - j;
+                    String prefixPath = breakpoints.get(index);
+                    String hash = prefixToHash.get(prefixPath);
+                    String key = getEntryKey(model.getName(), hash);
+                    RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+                    Map<String, String> fields = map.getAll(ALL_FIELDS);
+                    long expiredAt = Long.parseLong(fields.get(EXPIRED_AT_FIELD));
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: Will fail with null pointer
created_at: 2025-03-17T12:31:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,219 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final String EXPIRED_AT_FIELD = "expired_at";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD, EXPIRED_AT_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // model doesn't support caching
+            return null;
+        }
+        long currentTime = clock.getAsLong();
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                RMapAsync<String, String> rmap = batch.getMap(key, REDIS_MAP_CODEC);
+                rmap.isExistsAsync();
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < responses.size(); j++) {
+                Boolean exists = (Boolean) responses.get(j);
+                if (exists) {
+                    int index = i + count - j;
+                    String prefixPath = breakpoints.get(index);
+                    String hash = prefixToHash.get(prefixPath);
+                    String key = getEntryKey(model.getName(), hash);
+                    RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+                    Map<String, String> fields = map.getAll(ALL_FIELDS);
+                    long expiredAt = Long.parseLong(fields.get(EXPIRED_AT_FIELD));
+                    if (expiredAt > currentTime) {
+                        return new CachedUpstreamEntry(fields.get(UPSTREAM_ENDPOINT_FIELD), fields.get(PREFIX_PATH_FIELD));
+                    }
+                }
+            }
+        }
+        // take the last breakpoint
+        String prefixPath = breakpoints.get(breakpoints.size() - 1);
+        return new CachedUpstreamEntry(null, prefixPath);
+    }
+
+    public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, String expireAtStr) {
+        String key = getEntryKey(model.getName(), hash);
+        Map<String, String> fields = new HashMap<>();
+        fields.put(UPSTREAM_ENDPOINT_FIELD, entry.endpoint());
+        fields.put(PREFIX_PATH_FIELD, entry.prefixPath());
+        try (var ignore = lockService.lock(key)) {
+
+            RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+            boolean exists = map.isExists();
+
+            Instant expireAt = expireAt(expireAtStr);
+            if (expireAt == null && !exists) {
+                // adapter didn't return expireAt for a new cache entry
+                expireAt = Instant.ofEpochMilli(clock.getAsLong()).plus(DEFAULT_TTL);
+            }
+
+            if (expireAt != null) {
+                fields.put(EXPIRED_AT_FIELD, Long.toString(expireAt.toEpochMilli()));
+            }
+
+            map.putAll(fields);
+
+            if (expireAt != null) {
+                map.expire(expireAt);
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: It could be already evicted, and you do not set expiration
created_at: 2025-03-17T12:32:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,219 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+    private static final String EXPIRED_AT_FIELD = "expired_at";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD, EXPIRED_AT_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
+                    }
+                }
+                String hash = toString(messageDigest.digest());
+                String prefix = field + "[" + index + "]";
+                if (autoCaching || (objectNode.has(CUSTOM_FIELDS_NODE) && objectNode.get(CUSTOM_FIELDS_NODE).has(CACHE_BREAKPOINT_NODE))) {
+                    breakpoints.add(prefix);
+                }
+                prefixToHash.put(prefix, hash);
+            }
+
+        }
+        return new CacheBreakpointContext(breakpoints, prefixToHash, policy);
+    }
+
+    public CachedUpstreamEntry getCacheEntry(CacheBreakpointContext cacheBreakpointContext, Model model) {
+        List<String> breakpoints = cacheBreakpointContext.breakpoints();
+        if (breakpoints.isEmpty()) {
+            // model doesn't support caching
+            return null;
+        }
+        long currentTime = clock.getAsLong();
+        Map<String, String> prefixToHash = cacheBreakpointContext.prefixToHash();
+        for (int i = breakpoints.size() - 1; i >= 0;) {
+            RBatch batch = redisClient.createBatch();
+            int count = 0;
+            for (; i >= 0 && count < BATCH_SIZE; count++, i--) {
+                String prefixPath = breakpoints.get(i);
+                String hash = prefixToHash.get(prefixPath);
+                String key = getEntryKey(model.getName(), hash);
+                RMapAsync<String, String> rmap = batch.getMap(key, REDIS_MAP_CODEC);
+                rmap.isExistsAsync();
+            }
+            BatchResult<?> result = batch.execute();
+            List<?> responses = result.getResponses();
+            for (int j = 0; j < responses.size(); j++) {
+                Boolean exists = (Boolean) responses.get(j);
+                if (exists) {
+                    int index = i + count - j;
+                    String prefixPath = breakpoints.get(index);
+                    String hash = prefixToHash.get(prefixPath);
+                    String key = getEntryKey(model.getName(), hash);
+                    RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+                    Map<String, String> fields = map.getAll(ALL_FIELDS);
+                    long expiredAt = Long.parseLong(fields.get(EXPIRED_AT_FIELD));
+                    if (expiredAt > currentTime) {
+                        return new CachedUpstreamEntry(fields.get(UPSTREAM_ENDPOINT_FIELD), fields.get(PREFIX_PATH_FIELD));
+                    }
+                }
+            }
+        }
+        // take the last breakpoint
+        String prefixPath = breakpoints.get(breakpoints.size() - 1);
+        return new CachedUpstreamEntry(null, prefixPath);
+    }
+
+    public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, String expireAtStr) {
+        String key = getEntryKey(model.getName(), hash);
+        Map<String, String> fields = new HashMap<>();
+        fields.put(UPSTREAM_ENDPOINT_FIELD, entry.endpoint());
+        fields.put(PREFIX_PATH_FIELD, entry.prefixPath());
+        try (var ignore = lockService.lock(key)) {
+
+            RMap<String, String> map = redisClient.getMap(key, REDIS_MAP_CODEC);
+            boolean exists = map.isExists();
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: artsiomkorzun
body: Check for expiration field?
created_at: 2025-03-17T12:45:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,37 @@
+package com.epam.aidial.core.server.data.cache;
+
+public enum CachePolicy {
+    /**
+     * Prioritize service availability over cache hits. We will try to route his request to same upstream, that will provide caching feature.
+     * But in case, when cache upstream in not available, we will fallback to another available upstream, and will create cache there.
+     */
+    AVAILABILITY_PRIORITY("availability-priority"),
+    /**
+     * Prioritize cache hits over service availability. We will route request with cache flag to the same upstream.
+     * Even if cache upstream is not available, we will retry request to the same upstream.
+     */
+    CACHE_PRIORITY("cache-priority"),
+    /**
+     * Try to create automatic cache, where it's possible. We will route request that is cached to same upstream.
+     * But in case, when cache upstream is not available, we will fallback to another available upstream, and will create cache there.
+     * <p>
+     *  <b>Note</b>. Core should calculate hash for all prefixes for each request, and check their existence in Redis.
+     * </p>
+     */
+    AUTO_CACHING("auto-caching");
+
+    private final String val;
+    CachePolicy(String val) {
+        this.val = val;
+    }
+
+    public static CachePolicy fromString(String val) {
+        for (CachePolicy policy : CachePolicy.values()) {
+            if (policy.val.equals(val)) {
+                return policy;
+            }
+        }
+        // fallback
+        return AVAILABILITY_PRIORITY;
path: server/src/main/java/com/epam/aidial/core/server/data/cache/CachePolicy.java
user: login: adubovik
body: I don't think that we should tolerate invalid `X-DIAL-CACHE-POLICY` headers by falling back to `AVAILABILITY_PRIORITY`.

Imagine, someone made a typo in the header: `X-DIAL-CACHE-POLICY: cache-priorty`
It will be treated as `AVAILABILITY_PRIORITY`, which is very surprising for the client.

Why not reject invalid requests with 400?
created_at: 2025-03-18T10:03:25Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,211 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: adubovik
body: How do you ensure that the order in which JSON dictionary fields are enumerated is fixed?
created_at: 2025-03-18T10:24:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,211 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: adubovik
body: How do you ensure that `.toString()` serializes the fields of the same JSON dictionary in the same order?

https://stackoverflow.com/questions/27577701/jackson-objectmapper-specify-serialization-order-of-object-properties
created_at: 2025-03-18T10:25:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -114,10 +136,46 @@ void fail(HttpStatus status, long retryAfterSeconds) {
     }
 
     public void succeed() {
+        succeed(null, null);
+    }
+
+    public void succeed(HttpClientResponse proxyResponse, Deployment deployment) {
         verifyCurrentUpstream();
+        if (deployment instanceof Model model) {
+            updateUpstreamCache(proxyResponse, model);
+        }
         balancer.succeed(upstream);
     }
 
+    private void updateUpstreamCache(HttpClientResponse proxyResponse, Model model) {
+        String breakpointPath = proxyResponse.getHeader(Proxy.HEADER_CACHE_BREAKPOINT_PATH);
+        if (breakpointPath == null) {
+            // no cache
+            return;
+        }
+        String hash = upstreamCacheContext.getPrefixToHash().get(breakpointPath);
+        if (hash == null) {
+            if (model.getType() != ModelType.EMBEDDING) {
path: server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java
user: login: adubovik
body: `if (model.getType() == ModelType.CHAT) {`

because we only support caching for chat deployments.
created_at: 2025-03-18T11:25:02Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,37 @@
+package com.epam.aidial.core.server.data.cache;
+
+public enum CachePolicy {
+    /**
+     * Prioritize service availability over cache hits. We will try to route his request to same upstream, that will provide caching feature.
+     * But in case, when cache upstream in not available, we will fallback to another available upstream, and will create cache there.
+     */
+    AVAILABILITY_PRIORITY("availability-priority"),
+    /**
+     * Prioritize cache hits over service availability. We will route request with cache flag to the same upstream.
+     * Even if cache upstream is not available, we will retry request to the same upstream.
+     */
+    CACHE_PRIORITY("cache-priority"),
+    /**
+     * Try to create automatic cache, where it's possible. We will route request that is cached to same upstream.
+     * But in case, when cache upstream is not available, we will fallback to another available upstream, and will create cache there.
+     * <p>
+     *  <b>Note</b>. Core should calculate hash for all prefixes for each request, and check their existence in Redis.
+     * </p>
+     */
+    AUTO_CACHING("auto-caching");
+
+    private final String val;
+    CachePolicy(String val) {
+        this.val = val;
+    }
+
+    public static CachePolicy fromString(String val) {
+        for (CachePolicy policy : CachePolicy.values()) {
+            if (policy.val.equals(val)) {
+                return policy;
+            }
+        }
+        // fallback
+        return AVAILABILITY_PRIORITY;
path: server/src/main/java/com/epam/aidial/core/server/data/cache/CachePolicy.java
user: login: astsiapanay
body: fixed
created_at: 2025-03-18T12:48:06Z
author_association: COLLABORATOR
diff_hunk: @@ -114,10 +136,46 @@ void fail(HttpStatus status, long retryAfterSeconds) {
     }
 
     public void succeed() {
+        succeed(null, null);
+    }
+
+    public void succeed(HttpClientResponse proxyResponse, Deployment deployment) {
         verifyCurrentUpstream();
+        if (deployment instanceof Model model) {
+            updateUpstreamCache(proxyResponse, model);
+        }
         balancer.succeed(upstream);
     }
 
+    private void updateUpstreamCache(HttpClientResponse proxyResponse, Model model) {
+        String breakpointPath = proxyResponse.getHeader(Proxy.HEADER_CACHE_BREAKPOINT_PATH);
+        if (breakpointPath == null) {
+            // no cache
+            return;
+        }
+        String hash = upstreamCacheContext.getPrefixToHash().get(breakpointPath);
+        if (hash == null) {
+            if (model.getType() != ModelType.EMBEDDING) {
path: server/src/main/java/com/epam/aidial/core/server/upstream/UpstreamRoute.java
user: login: astsiapanay
body: fixed
created_at: 2025-03-18T12:48:54Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,211 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: According to javadoc:

`Feature that defines default property serialization order used for POJO properties. If enabled, default ordering is alphabetic (similar to how com.fasterxml.jackson.annotation.JsonPropertyOrder.alphabetic() works); if disabled, order is unspecified (based on what JDK gives us, which may be declaration order, but is not guaranteed).`

The service take JsonNode is an arbitrary JSON object. So I had to write own serializer.
created_at: 2025-03-18T15:02:00Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,211 @@
+package com.epam.aidial.core.server.service;
+
+import com.epam.aidial.core.config.Model;
+import com.epam.aidial.core.server.data.cache.CacheBreakpointContext;
+import com.epam.aidial.core.server.data.cache.CachePolicy;
+import com.epam.aidial.core.server.data.cache.CachedUpstreamEntry;
+import com.epam.aidial.core.storage.blobstore.BlobStorageUtil;
+import com.epam.aidial.core.storage.service.LockService;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import lombok.SneakyThrows;
+import lombok.extern.slf4j.Slf4j;
+import org.redisson.api.BatchResult;
+import org.redisson.api.RBatch;
+import org.redisson.api.RMap;
+import org.redisson.api.RMapAsync;
+import org.redisson.api.RedissonClient;
+import org.redisson.client.codec.Codec;
+import org.redisson.client.codec.StringCodec;
+import org.redisson.codec.CompositeCodec;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.LongSupplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Slf4j
+public class UpstreamCacheService {
+
+    private static final Pattern PREFIX_PATH = Pattern.compile("^prefix\\.body\\.(?<nodeName>(tools|messages))$");
+    private static final Duration DEFAULT_TTL = Duration.ofMinutes(10);
+
+    private static final String CUSTOM_FIELDS_NODE = "custom_fields";
+    private static final String CACHE_BREAKPOINT_NODE = "cache_breakpoint";
+    private static final String UPSTREAM_ENDPOINT_FIELD = "upstream_endpoint";
+    private static final String PREFIX_PATH_FIELD = "prefix_path";
+
+
+    private static final Set<String> ALL_FIELDS = Set.of(UPSTREAM_ENDPOINT_FIELD, PREFIX_PATH_FIELD);
+
+    private static final int BATCH_SIZE = 64;
+
+    private static final Codec REDIS_MAP_CODEC = new CompositeCodec(
+            StringCodec.INSTANCE,
+            StringCodec.INSTANCE);
+
+    private final RedissonClient redisClient;
+
+    private final LockService lockService;
+
+    private final LongSupplier clock;
+
+    private final String prefix;
+
+    public UpstreamCacheService(RedissonClient redisClient, LockService lockService, LongSupplier clock, String prefix) {
+        this.redisClient = redisClient;
+        this.lockService = lockService;
+        this.clock = clock;
+        this.prefix = prefix;
+    }
+
+    public CacheBreakpointContext buildCacheBreakpointContext(ObjectNode body, CachePolicy policy, Model model) {
+        boolean autoCaching = policy == CachePolicy.AUTO_CACHING;
+        List<String> fieldsOrder = model.getFieldsHashingOrder();
+        MessageDigest messageDigest = createMessageDigest();
+        List<String> breakpoints = new ArrayList<>();
+        Map<String, String> prefixToHash = new HashMap<>();
+        for (String field : fieldsOrder) {
+            Matcher matcher = PREFIX_PATH.matcher(field);
+            if (!matcher.matches()) {
+                log.warn("Unsupported prefix path: {}", field);
+                continue;
+            }
+            String nodeName = matcher.group("nodeName");
+            JsonNode node = body.get(nodeName);
+            if (node == null || !node.isArray()) {
+                // embedding request is not supported yet
+                continue;
+            }
+            ArrayNode arrayNode = (ArrayNode) node;
+            for (int index = 0; index < arrayNode.size(); index++) {
+                ObjectNode objectNode = (ObjectNode) arrayNode.get(index);
+                for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext(); ) {
+                    Map.Entry<String, JsonNode> entry = it.next();
+                    if (entry.getKey().equals(CUSTOM_FIELDS_NODE)) {
+                        continue;
+                    }
+                    if (entry.getKey().equals("custom_content")) {
+                        // include attachments only
+                        if (entry.getValue().has("attachments")) {
+                            messageDigest.update(entry.getValue().get("attachments").toString().getBytes(StandardCharsets.UTF_8));
+                        }
+                    } else {
+                        messageDigest.update(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: fixed
created_at: 2025-03-18T15:02:09Z
author_association: COLLABORATOR
diff_hunk: @@ -182,6 +174,54 @@ public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, Str
         }
     }
 
+    @VisibleForTesting
+    static void traverse(JsonNode node, StringBuilder content) {
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: adubovik
body: Why not sort the dictionary by default?

https://github.com/epam/ai-dial-core/blob/ecc3c355a08d3c5f98bb53de2489470d80cd010e/server/src/main/java/com/epam/aidial/core/server/util/ProxyUtil.java#L35-L37

```diff
+ import com.fasterxml.jackson.databind.SerializationFeature;

    public static final JsonMapper MAPPER = JsonMapper.builder()
            .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS)
+           .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS)
            .build();
```
created_at: 2025-03-18T15:44:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -182,6 +174,54 @@ public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, Str
         }
     }
 
+    @VisibleForTesting
+    static void traverse(JsonNode node, StringBuilder content) {
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: it doesn't work for JsonNode. See https://stackoverflow.com/a/36465506

created_at: 2025-03-18T16:27:33Z
author_association: COLLABORATOR
diff_hunk: @@ -182,6 +174,54 @@ public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, Str
         }
     }
 
+    @VisibleForTesting
+    static void traverse(JsonNode node, StringBuilder content) {
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: This approach https://cowtowncoder.medium.com/jackson-tips-sorting-json-using-jsonnode-ce4476e37aee should work 
created_at: 2025-03-18T16:49:05Z
author_association: COLLABORATOR
diff_hunk: @@ -182,6 +174,54 @@ public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, Str
         }
     }
 
+    @VisibleForTesting
+    static void traverse(JsonNode node, StringBuilder content) {
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: There are 3 options to make the fixed order of object properties:

1. Make a single ObjectMapper with the ordered object factory for all functions. That implies that the service depends on a caller. The caller must provide JSON object with ordered properties. We don't need this for all cases.
2. Change the method's signature by passing byte array instead of Json object. That makes the method to parse the entire request body one more time.
3. Sort object properties recursively in place. The approach is more light weight than the 2nd option and more reliable than the 1st one.
created_at: 2025-03-19T08:44:08Z
author_association: COLLABORATOR
diff_hunk: @@ -182,6 +174,54 @@ public void updateEntry(String hash, CachedUpstreamEntry entry, Model model, Str
         }
     }
 
+    @VisibleForTesting
+    static void traverse(JsonNode node, StringBuilder content) {
path: server/src/main/java/com/epam/aidial/core/server/service/UpstreamCacheService.java
user: login: astsiapanay
body: fixed
created_at: 2025-03-19T08:45:43Z
author_association: COLLABORATOR

PR comments:
user: login: ai-dial-actions
created_at: 2025-03-13T09:49:24Z
body: <h1>Dependency Review</h1>
✅ No vulnerabilities or license issues or OpenSSF Scorecard issues found.<h2>OpenSSF Scorecard</h2>
<table><tr><th>Package</th><th>Version</th><th>Score</th><th>Details</th></tr>
</table><h2>Scanned Files</h2>
<ul>
<!-- Sticky Pull Request Commentdependency-analysis -->
author_association: CONTRIBUTOR
user: login: astsiapanay
created_at: 2025-03-19T10:42:27Z
body: /deploy-review
>Environment: [review-environment](https://chat-ai-dial-core-pr-731.nightly-test.deltixhub.io) | [pipeline](https://gitlab.deltixhub.com/Clients/deltix/nightly-test/dial/review/-/pipelines/583236)
>Overlay E2E tests status: [success](https://gitlab.deltixhub.com/Clients/deltix/nightly-test/dial/review/-/jobs/1716723)
>Chat E2E tests status: [success](https://gitlab.deltixhub.com/Clients/deltix/nightly-test/dial/review/-/jobs/1716722)
author_association: COLLABORATOR