Repository: elastic/kibana
Pull request #216519: [Expression] Cached expression can keep their own side effects
Author: login: dej611
Labels: name: release_note:fix
name: Feature:ExpressionLanguage
description: Interpreter expression language (aka canvas pipeline)
name: Team:Visualizations
description: Team label for Lens, elastic-charts, Graph, legacy editors (TSVB, Visualize, Timelion) t//
name: v9.1.0
name: v8.19.0
name: v8.18.1
name: v9.0.1
Created: 2025-03-31T16:48:58Z | Merged: 2025-04-11T12:50:47Z

## Summary

Fixes the #207204 

This PR introduces a new complementary function for an Expression definition named `sideEffects`, this goes together with the other `fn` function and it is used to restore any side effect when the caching system kicks in.

![side_effects_cache](https://github.com/user-attachments/assets/74b1ddff-a45c-4983-ac09-57559155fba8)

I haven't found how to programmatically test this.
Will add an FTR if it can be reliable to reproduce an expression caching scenario.

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

### Release notes

The request inspector now shows the correct request and response in any successful scenario.

<!--ONMERGE {"backportTargets":["8.16","8.17","8.18","8.x","9.0"]} ONMERGE-->


Diff:
diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml
index 6de583177707d..f5cb790aa999c 100644
--- a/packages/kbn-optimizer/limits.yml
+++ b/packages/kbn-optimizer/limits.yml
@@ -85,7 +85,7 @@ pageLoadAssetSize:
   infra: 184320
   ingestPipelines: 58003
   inputControlVis: 172675
-  inspector: 17600
+  inspector: 17740
   interactiveSetup: 80000
   inventory: 27430
   kibanaOverview: 56279
diff --git a/src/platform/plugins/shared/data/common/search/aggs/agg_configs.ts b/src/platform/plugins/shared/data/common/search/aggs/agg_configs.ts
index 770ba5521cb6d..6956bedbd80e5 100644
--- a/src/platform/plugins/shared/data/common/search/aggs/agg_configs.ts
+++ b/src/platform/plugins/shared/data/common/search/aggs/agg_configs.ts
@@ -472,8 +472,9 @@ export class AggConfigs {
     if (!this.hasTimeShifts()) {
       return response;
     }
-    const transformedRawResponse = cloneDeep(response.rawResponse);
-    if (!transformedRawResponse.aggregations) {
+    let transformedRawResponse = response.rawResponse;
+    if (!response.rawResponse.aggregations) {
+      transformedRawResponse = cloneDeep(response.rawResponse);
       transformedRawResponse.aggregations = {
         doc_count: response.rawResponse.hits?.total as estypes.AggregationsAggregate,
       };
diff --git a/src/platform/plugins/shared/data/common/search/expressions/esaggs/esaggs_fn.ts b/src/platform/plugins/shared/data/common/search/expressions/esaggs/esaggs_fn.ts
index e1d7af72884cb..57ad0c5e210bc 100644
--- a/src/platform/plugins/shared/data/common/search/expressions/esaggs/esaggs_fn.ts
+++ b/src/platform/plugins/shared/data/common/search/expressions/esaggs/esaggs_fn.ts
@@ -58,7 +58,6 @@ export const getEsaggsMeta: () => Omit<EsaggsExpressionFunctionDefinition, 'fn'>
   name,
   type: 'datatable',
   inputTypes: ['kibana_context', 'null'],
-  allowCache: true,
   help: i18n.translate('data.functions.esaggs.help', {
     defaultMessage: 'Run AggConfig aggregation',
   }),
diff --git a/src/platform/plugins/shared/data/common/search/expressions/esql.ts b/src/platform/plugins/shared/data/common/search/expressions/esql.ts
index 6d39182f8a2cb..b2c6c15ad2426 100644
--- a/src/platform/plugins/shared/data/common/search/expressions/esql.ts
+++ b/src/platform/plugins/shared/data/common/search/expressions/esql.ts
@@ -33,6 +33,7 @@ import { getTime } from '../../query';
 import {
   ESQL_ASYNC_SEARCH_STRATEGY,
   ESQL_TABLE_TYPE,
+  getSideEffectFunction,
   isRunningResponse,
   type KibanaContext,
 } from '..';
@@ -98,7 +99,6 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
     name: 'esql',
     type: 'datatable',
     inputTypes: ['kibana_context', 'null'],
-    allowCache: true,
     help: i18n.translate('data.search.esql.help', {
       defaultMessage: 'Queries Elasticsearch using ES|QL.',
     }),
@@ -155,6 +155,11 @@ export const getEsqlFn = ({ getStartDependencies }: EsqlFnArguments) => {
         }),
       },
     },
+    allowCache: {
+      withSideEffects: (_, { inspectorAdapters }) => {
+        return getSideEffectFunction(inspectorAdapters);
+      },
+    },
     fn(
       input,
       {
diff --git a/src/platform/plugins/shared/data/common/search/expressions/utils/index.ts b/src/platform/plugins/shared/data/common/search/expressions/utils/index.ts
index 357876ed6a8cc..ed3379450b9a8 100644
--- a/src/platform/plugins/shared/data/common/search/expressions/utils/index.ts
+++ b/src/platform/plugins/shared/data/common/search/expressions/utils/index.ts
@@ -9,3 +9,4 @@
 
 export * from './function_wrapper';
 export { adaptToExpressionValueFilter } from './filters_adapter';
+export { getSideEffectFunction } from './requests_side_effects';
diff --git a/src/platform/plugins/shared/data/common/search/expressions/utils/requests_side_effects.ts b/src/platform/plugins/shared/data/common/search/expressions/utils/requests_side_effects.ts
new file mode 100644
index 0000000000000..c1227743acf54
--- /dev/null
+++ b/src/platform/plugins/shared/data/common/search/expressions/utils/requests_side_effects.ts
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+import { Adapters } from '@kbn/inspector-plugin/common';
+
+const collectSideEffectsData = (adapters: Adapters) => {
+  return adapters.requests?.getRequestEntries();
+};
+
+export const getSideEffectFunction = (adapters: Adapters) => {
+  const requestsWithResponses = collectSideEffectsData(adapters);
+  return () => {
+    if (!requestsWithResponses || requestsWithResponses.length === 0) {
+      return;
+    }
+    const requestsMap = new Map(requestsWithResponses.map(([request]) => [request.id, request]));
+    const responsesMap = new WeakMap(requestsWithResponses);
+    adapters.requests?.loadFromEntries(requestsMap, responsesMap);
+  };
+};
diff --git a/src/platform/plugins/shared/data/public/search/expressions/esaggs.ts b/src/platform/plugins/shared/data/public/search/expressions/esaggs.ts
index ec42b18452b06..7e70e2d5d86e1 100644
--- a/src/platform/plugins/shared/data/public/search/expressions/esaggs.ts
+++ b/src/platform/plugins/shared/data/public/search/expressions/esaggs.ts
@@ -15,6 +15,7 @@ import {
   EsaggsExpressionFunctionDefinition,
   EsaggsStartDependencies,
   getEsaggsMeta,
+  getSideEffectFunction,
 } from '../../../common/search/expressions';
 import { DataPublicPluginStart, DataStartDependencies } from '../../types';
 
@@ -37,13 +38,19 @@ export function getFunctionDefinition({
 }) {
   return (): EsaggsExpressionFunctionDefinition => ({
     ...getEsaggsMeta(),
+    allowCache: {
+      withSideEffects: (_, { inspectorAdapters }) => {
+        return getSideEffectFunction(inspectorAdapters);
+      },
+    },
     fn(
       input,
       args,
       { inspectorAdapters, abortSignal, getSearchSessionId, getExecutionContext, getSearchContext }
     ) {
       return defer(async () => {
-        const { aggs, indexPatterns, searchSource, getNow } = await getStartDependencies();
+        const [{ aggs, indexPatterns, searchSource, getNow }, { handleEsaggsRequest }] =
+          await Promise.all([getStartDependencies(), import('../../../common/search/expressions')]);
 
         const indexPattern = await indexPatterns.create(args.index.value, true);
         const aggConfigs = aggs.createAggConfigs(
@@ -57,8 +64,6 @@ export function getFunctionDefinition({
           }
         );
 
-        const { handleEsaggsRequest } = await import('../../../common/search/expressions');
-
         return { aggConfigs, indexPattern, searchSource, getNow, handleEsaggsRequest };
       }).pipe(
         switchMap(({ aggConfigs, indexPattern, searchSource, getNow, handleEsaggsRequest }) => {
diff --git a/src/platform/plugins/shared/expressions/common/execution/execution.ts b/src/platform/plugins/shared/expressions/common/execution/execution.ts
index 9d0e9450271da..d969f9f95e887 100644
--- a/src/platform/plugins/shared/expressions/common/execution/execution.ts
+++ b/src/platform/plugins/shared/expressions/common/execution/execution.ts
@@ -59,6 +59,7 @@ type UnwrapReturnType<Function extends (...args: any[]) => unknown> =
 export interface FunctionCacheItem {
   value: unknown;
   time: number;
+  sideEffectFn?: () => void;
 }
 /**
  * The result returned after an expression function execution.
@@ -475,21 +476,19 @@ export class Execution<
       .pipe(
         map((currentInput) => this.cast(currentInput, fn.inputTypes)),
         switchMap((normalizedInput) => {
-          if (fn.allowCache && this.context.allowCache) {
-            hash = calculateObjectHash([
-              fn.name,
-              normalizedInput,
-              args,
-              this.context.getSearchContext(),
-            ]);
+          const {
+            hash: fnHash,
+            value: cachedValue,
+            valid: cacheValid,
+          } = this.#canUseCachedResult(fn, normalizedInput, args);
+          hash = fnHash;
+          if (cacheValid) {
+            cachedValue.sideEffectFn?.();
+            return of(cachedValue.value);
           }
-          if (hash && this.functionCache.has(hash)) {
-            const cached = this.functionCache.get(hash);
-            if (cached && Date.now() - cached.time < this.cacheTimeout) {
-              return of(cached.value);
-            }
-          }
-          return of(fn.fn(normalizedInput, args, this.context));
+          const output = fn.fn(normalizedInput, args, this.context);
+
+          return of(output);
         }),
         switchMap((fnResult) => {
           return (
@@ -524,10 +523,15 @@ export class Execution<
         }),
         finalize(() => {
           if (completionFlag && hash) {
+            const sideEffectResult = this.#getSideEffectFn(fn, args);
             while (this.functionCache.size >= maxCacheSize) {
               this.functionCache.delete(this.functionCache.keys().next().value);
             }
-            this.functionCache.set(hash, { value: lastValue, time: Date.now() });
+            this.functionCache.set(hash, {
+              value: lastValue,
+              time: Date.now(),
+              sideEffectFn: sideEffectResult,
+            });
           }
         })
       )
@@ -714,4 +718,41 @@ export class Execution<
         return throwError(new Error(`Unknown AST object: ${JSON.stringify(ast)}`));
     }
   }
+
+  #canUseCachedResult<Fn extends ExpressionFunction>(
+    fn: Fn,
+    input: unknown,
+    args: Record<string, unknown>
+  ):
+    | { hash: string; value: FunctionCacheItem; valid: boolean }
+    | { hash: string | undefined; value: undefined; valid: false } {
+    if (!fn.allowCache || !this.context.allowCache) {
+      return { hash: undefined, value: undefined, valid: false };
+    }
+    const hash = calculateObjectHash([fn.name, input, args, this.context.getSearchContext()]);
+
+    const cached = this.functionCache.get(hash);
+    if (hash && cached) {
+      return {
+        hash,
+        value: cached,
+        valid: Boolean(cached && Date.now() - cached.time < this.cacheTimeout),
+      };
+    }
+    return {
+      hash,
+      value: undefined,
+      valid: false,
+    };
+  }
+
+  #getSideEffectFn<Fn extends ExpressionFunction>(
+    fn: Fn,
+    args: Record<string, unknown>
+  ): undefined | (() => void) {
+    if (!fn.allowCache || typeof fn.allowCache === 'boolean') {
+      return undefined;
+    }
+    return fn.allowCache.withSideEffects?.(args, this.context);
+  }
 }
diff --git a/src/platform/plugins/shared/expressions/common/expression_functions/expression_function.ts b/src/platform/plugins/shared/expressions/common/expression_functions/expression_function.ts
index e9d91566b3dc3..45b6e11e2ff85 100644
--- a/src/platform/plugins/shared/expressions/common/expression_functions/expression_function.ts
+++ b/src/platform/plugins/shared/expressions/common/expression_functions/expression_function.ts
@@ -42,7 +42,9 @@ export class ExpressionFunction implements PersistableState<ExpressionAstFunctio
   /**
    * Opt-in to caching this function. By default function outputs are cached and given the same inputs cached result is returned.
    */
-  allowCache: boolean;
+  allowCache:
+    | boolean
+    | { withSideEffects: (params: Record<string, unknown>, handlers: object) => () => void };
 
   /**
    * Function to run function (context, args)
@@ -116,7 +118,10 @@ export class ExpressionFunction implements PersistableState<ExpressionAstFunctio
     this.fn = fn as ExpressionFunction['fn'];
     this.help = help || '';
     this.inputTypes = inputTypes || context?.types;
-    this.allowCache = !!allowCache;
+    this.allowCache =
+      allowCache && typeof allowCache !== 'boolean'
+        ? (allowCache as ExpressionFunction['allowCache'])
+        : Boolean(allowCache);
     this.disabled = disabled || false;
     this.deprecated = !!deprecated;
     this.telemetry = telemetry || ((s, c) => c);
diff --git a/src/platform/plugins/shared/expressions/common/expression_functions/types.ts b/src/platform/plugins/shared/expressions/common/expression_functions/types.ts
index 3237139b9e984..2652b7dfed81a 100644
--- a/src/platform/plugins/shared/expressions/common/expression_functions/types.ts
+++ b/src/platform/plugins/shared/expressions/common/expression_functions/types.ts
@@ -60,8 +60,19 @@ export interface ExpressionFunctionDefinition<
 
   /**
    * Opt-in to caching this function. By default function outputs are cached and given the same inputs cached result is returned.
+   *
+   * It is possible to collect side effects produced by the function
+   * (e.g. logging, sending events to the server, etc.) and return a
+   * handler to reproduce such side effects when the function cache is used
+   * instead of the original function implementation.
+   * @param args Parameters set for this function in expression.
+   * @param context Object with functions to perform side effects. This object
+   *     is created for the duration of the execution of expression and is the
+   *     same for all functions in expression chain.
+   * @returns A handler to be called to reproduce side effects when the function cache is used.
+   *
    */
-  allowCache?: boolean;
+  allowCache?: boolean | { withSideEffects(args: Arguments, context: Context): () => void };
 
   /**
    * List of allowed type names for input value of this function. If this
diff --git a/src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts b/src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
index 0c3298f4aa899..72e9944d8961e 100644
--- a/src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
+++ b/src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
@@ -21,10 +21,12 @@ import { Request, RequestParams, RequestStatus } from './types';
  */
 export class RequestAdapter extends EventEmitter {
   private requests: Map<string, Request>;
+  private responses: WeakMap<Request, RequestResponder>;
 
   constructor() {
     super();
     this.requests = new Map();
+    this.responses = new WeakMap();
   }
 
   /**
@@ -52,7 +54,17 @@ export class RequestAdapter extends EventEmitter {
     };
     this.requests.set(req.id, req);
     this._onChange();
-    return new RequestResponder(req, () => this._onChange());
+    const responder = new RequestResponder(req, () => this._onChange());
+    this.responses.set(req, responder);
+    return responder;
+  }
+
+  public loadFromEntries(
+    requests: Map<string, Request>,
+    responses: WeakMap<Request, RequestResponder>
+  ) {
+    this.requests = requests;
+    this.responses = responses;
   }
 
   public reset(): void {
@@ -61,7 +73,11 @@ export class RequestAdapter extends EventEmitter {
   }
 
   public resetRequest(id: string): void {
+    const req = this.requests.get(id);
     this.requests.delete(id);
+    if (req) {
+      this.responses.delete(req);
+    }
     this._onChange();
   }
 
@@ -69,6 +85,12 @@ export class RequestAdapter extends EventEmitter {
     return Array.from(this.requests.values());
   }
 
+  public getRequestEntries(): Array<[Request, RequestResponder]> {
+    return this.getRequests()
+      .map((req) => [req, this.responses.get(req)] as [Request, RequestResponder])
+      .filter(([_req, responder]) => responder != null);
+  }
+
   private _onChange(): void {
     this.emit('change');
   }
diff --git a/src/platform/plugins/shared/unified_histogram/public/container/hooks/use_state_props.test.ts b/src/platform/plugins/shared/unified_histogram/public/container/hooks/use_state_props.test.ts
index f57392998d1f5..94912430db480 100644
--- a/src/platform/plugins/shared/unified_histogram/public/container/hooks/use_state_props.test.ts
+++ b/src/platform/plugins/shared/unified_histogram/public/container/hooks/use_state_props.test.ts
@@ -129,6 +129,7 @@ describe('useStateProps', () => {
             "_eventsCount": 0,
             "_maxListeners": undefined,
             "requests": Map {},
+            "responses": WeakMap {},
             Symbol(shapeMode): false,
             Symbol(kCapture): false,
           },
@@ -217,6 +218,7 @@ describe('useStateProps', () => {
             "_eventsCount": 0,
             "_maxListeners": undefined,
             "requests": Map {},
+            "responses": WeakMap {},
             Symbol(shapeMode): false,
             Symbol(kCapture): false,
           },
@@ -415,6 +417,7 @@ describe('useStateProps', () => {
             "_eventsCount": 0,
             "_maxListeners": undefined,
             "requests": Map {},
+            "responses": WeakMap {},
             Symbol(shapeMode): false,
             Symbol(kCapture): false,
           },
@@ -498,6 +501,7 @@ describe('useStateProps', () => {
             "_eventsCount": 0,
             "_maxListeners": undefined,
             "requests": Map {},
+            "responses": WeakMap {},
             Symbol(shapeMode): false,
             Symbol(kCapture): false,
           },


Reviews:
user: login: dej611
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-02T08:24:58Z
user: login: dej611
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T08:52:20Z
user: login: jbudz
body: packages/kbn-optimizer/limits.yml
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-03T17:24:49Z
user: login: nreese
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T14:38:15Z
user: login: nreese
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T14:39:46Z
user: login: dej611
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T15:11:30Z
user: login: dej611
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T15:13:10Z
user: login: nreese
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T16:53:56Z
user: login: nreese
body: kibana-presentation changes LGTM
code review only
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T16:54:15Z
user: login: lukasolson
body: Tested and the behavior seems to be fixed (for Lens visualizations using classic mode).

The bug still happens for ES|QL visualizations on a dashboard using inline editing, so we will need to make the same changes to the `esql` function.

I wanted to ask a couple of quick questions regarding the approach:
* Could this be fixed by moving the `this.context.inspectorAdapters.requests?.reset();` call into the `esaggs` function (prior to making the search request)? It seems odd we're clearing the inspector adapters in the executor but then utilizing them inside the functions themselves.
* If we decide this approach is best, can we somehow tie these `sideEffects` into the same place we register the `allowCache: true` so that it's clear they're related? Right now it's not clear if you support caching that you have to also worry about side effects.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-04T21:03:07Z
user: login: lukasolson
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-09T16:30:55Z
user: login: markov00
body: Tested locally, works as expected
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T10:33:40Z
user: login: dej611
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-11T11:14:33Z

Review comments:
diff_hunk: @@ -472,8 +472,9 @@ export class AggConfigs {
     if (!this.hasTimeShifts()) {
       return response;
     }
-    const transformedRawResponse = cloneDeep(response.rawResponse);
-    if (!transformedRawResponse.aggregations) {
+    let transformedRawResponse = response.rawResponse;
path: src/platform/plugins/shared/data/common/search/aggs/agg_configs.ts
user: login: dej611
body: Just a tiny perf improvement
created_at: 2025-04-02T08:24:57Z
author_association: CONTRIBUTOR
diff_hunk: @@ -37,13 +37,28 @@ export function getFunctionDefinition({
 }) {
   return (): EsaggsExpressionFunctionDefinition => ({
     ...getEsaggsMeta(),
+    sideEffects: (
+      args,
+      { inspectorAdapters, abortSignal, getSearchSessionId, getExecutionContext, getSearchContext }
+    ) => {
+      const requests = inspectorAdapters.requests?.getRequestEntries();
+      return () => {
+        if (!requests || requests.length === 0) {
+          return;
+        }
+        const requestsMap = new Map(requests.map(([request]) => [request.id, request]));
+        const responsesMap = new WeakMap(requests);
+        inspectorAdapters.requests?.loadFromEntries(requestsMap, responsesMap);
+      };
+    },
     fn(
       input,
       args,
       { inspectorAdapters, abortSignal, getSearchSessionId, getExecutionContext, getSearchContext }
     ) {
       return defer(async () => {
-        const { aggs, indexPatterns, searchSource, getNow } = await getStartDependencies();
+        const [{ aggs, indexPatterns, searchSource, getNow }, { handleEsaggsRequest }] =
path: src/platform/plugins/shared/data/public/search/expressions/esaggs.ts
user: login: dej611
body: Another tiny perf improvement
created_at: 2025-04-03T08:52:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -52,7 +54,17 @@ export class RequestAdapter extends EventEmitter {
     };
     this.requests.set(req.id, req);
     this._onChange();
-    return new RequestResponder(req, () => this._onChange());
+    const responder = new RequestResponder(req, () => this._onChange());
+    this.responses.set(req, responder);
+    return responder;
+  }
+
+  public loadFromEntries(
+    requests: Map<string, Request>,
+    responses: WeakMap<Request, RequestResponder>
+  ) {
+    this.requests = requests;
+    this.responses = responses;
   }
 
   public reset(): void {
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: nreese
body: Should `reset` also reset `this.responses`?
created_at: 2025-04-04T14:38:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -21,10 +21,12 @@ import { Request, RequestParams, RequestStatus } from './types';
  */
 export class RequestAdapter extends EventEmitter {
   private requests: Map<string, Request>;
+  private responses: WeakMap<Request, RequestResponder>;
 
   constructor() {
     super();
     this.requests = new Map();
+    this.responses = new WeakMap();
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: nreese
body: Why is responses a WeakMap instead of just a map keyed to the request id?
created_at: 2025-04-04T14:39:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -52,7 +54,17 @@ export class RequestAdapter extends EventEmitter {
     };
     this.requests.set(req.id, req);
     this._onChange();
-    return new RequestResponder(req, () => this._onChange());
+    const responder = new RequestResponder(req, () => this._onChange());
+    this.responses.set(req, responder);
+    return responder;
+  }
+
+  public loadFromEntries(
+    requests: Map<string, Request>,
+    responses: WeakMap<Request, RequestResponder>
+  ) {
+    this.requests = requests;
+    this.responses = responses;
   }
 
   public reset(): void {
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: dej611
body: Due to the `WeakMap` nature the `responses` should be empty as soon as the `requests` get cleared.
created_at: 2025-04-04T15:11:30Z
author_association: CONTRIBUTOR
diff_hunk: @@ -21,10 +21,12 @@ import { Request, RequestParams, RequestStatus } from './types';
  */
 export class RequestAdapter extends EventEmitter {
   private requests: Map<string, Request>;
+  private responses: WeakMap<Request, RequestResponder>;
 
   constructor() {
     super();
     this.requests = new Map();
+    this.responses = new WeakMap();
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: dej611
body: I thought about that, but:
* there's no need to list the entries
* using the `request` itself saves a `map` cycle altogether
* it could `reset` as side effect automatically

Given the use cases it makes no specific difference between the two, other than no providing the iterator API.
created_at: 2025-04-04T15:13:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -21,10 +21,12 @@ import { Request, RequestParams, RequestStatus } from './types';
  */
 export class RequestAdapter extends EventEmitter {
   private requests: Map<string, Request>;
+  private responses: WeakMap<Request, RequestResponder>;
 
   constructor() {
     super();
     this.requests = new Map();
+    this.responses = new WeakMap();
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: nreese
body: thanks for the explanation
created_at: 2025-04-04T16:53:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -61,14 +73,24 @@ export class RequestAdapter extends EventEmitter {
   }
 
   public resetRequest(id: string): void {
+    const req = this.requests.get(id);
     this.requests.delete(id);
+    if (req) {
+      this.responses.delete(req);
+    }
     this._onChange();
   }
 
   public getRequests(): Request[] {
     return Array.from(this.requests.values());
   }
 
+  public getRequestEntries(): Array<[Request, RequestResponder]> {
+    return this.getRequests()
+      .map((req) => [req, this.responses.get(req)] as [Request, RequestResponder])
+      .filter(([_req, responder]) => responder != null);
+  }
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: markov00
body: nit: you can move the `as [Request, RequestResponder]` in the return of the filter, because at the casting this in the map callback doesn't make much sense (is filter that produce an array with both defined values, not the map.

You can also use a flatMap instead without the need of a cast:
```
return this.getRequests().flatMap((req) => {
      const responder = this.responses.get(req);
      return responder ? [[req, responder]] : [];
    });
```
created_at: 2025-04-11T10:33:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -61,14 +73,24 @@ export class RequestAdapter extends EventEmitter {
   }
 
   public resetRequest(id: string): void {
+    const req = this.requests.get(id);
     this.requests.delete(id);
+    if (req) {
+      this.responses.delete(req);
+    }
     this._onChange();
   }
 
   public getRequests(): Request[] {
     return Array.from(this.requests.values());
   }
 
+  public getRequestEntries(): Array<[Request, RequestResponder]> {
+    return this.getRequests()
+      .map((req) => [req, this.responses.get(req)] as [Request, RequestResponder])
+      .filter(([_req, responder]) => responder != null);
+  }
path: src/platform/plugins/shared/inspector/common/adapters/request/request_adapter.ts
user: login: dej611
body: The `filter` is there to prevent partial or failed request to land on the adapter.
As for the `flatMap` approach, it's a subjective topic.
I prefer the `filter` as it pushed less memory pressure in this case ( 1 extra array generated ) vs the `flatMap` ( N arrays generated ).
Not a big deal, but if it's not strictly necessarily I would avoid it.
created_at: 2025-04-11T11:14:33Z
author_association: CONTRIBUTOR

PR comments:
user: login: elasticmachine
created_at: 2025-04-03T14:21:56Z
body: Pinging @elastic/kibana-visualizations (Team:Visualizations)
author_association: CONTRIBUTOR
user: login: dej611
created_at: 2025-04-09T09:10:38Z
body: > Tested and the behavior seems to be fixed (for Lens visualizations using classic mode).
> 
> The bug still happens for ES|QL visualizations on a dashboard using inline editing, so we will need to make the same changes to the `esql` function.

Great catch! On it...

> I wanted to ask a couple of quick questions regarding the approach:
> 
> * Could this be fixed by moving the `this.context.inspectorAdapters.requests?.reset();` call into the `esaggs` function (prior to making the search request)? It seems odd we're clearing the inspector adapters in the executor but then utilizing them inside the functions themselves.
> * If we decide this approach is best, can we somehow tie these `sideEffects` into the same place we register the `allowCache: true` so that it's clear they're related? Right now it's not clear if you support caching that you have to also worry about side effects.

Makes sense. I'll try 1) or improve the current code with 2) if it doesn't work.
author_association: CONTRIBUTOR
user: login: dej611
created_at: 2025-04-09T13:29:53Z
body: @lukasolson fixed the bug on ES|QL panels with 47cd9a7 .
As of the alternative approach moving the `reset`call in the `esaggs` function, I've faced few issues with multiple layers charts (i.e. multiple data layers, reference lines, etc...). So I've merged it with the `allowCache` type, adding a new `{withSideEffect: ...}` option
author_association: CONTRIBUTOR
user: login: elasticmachine
created_at: 2025-04-09T15:47:22Z
body: ## :green_heart: Build Succeeded
* [Buildkite Build](https://buildkite.com/elastic/kibana-pull-request/builds/291593)
* Commit: 7529e81b0f1e381772017697571e57aa724949d4

### Metrics [[docs](https://www.elastic.co/guide/en/kibana/master/ci-metrics.html)]
#### Module Count
> Fewer modules leads to a faster build time


| id | [before](https://github.com/elastic/kibana/commit/f833e09121d6235a22ec377ab17aa2b23b6191fa) | [after](https://github.com/elastic/kibana/commit/7529e81b0f1e381772017697571e57aa724949d4) | diff |
| --- | --- | --- | --- |
| `data` | 534 | 535 | +1 |

#### Public APIs missing comments
> Total count of every public API that lacks a comment. Target amount is 0. Run `node scripts/build_api_docs --plugin [yourplugin] --stats comments` for more detailed information.


| id | [before](https://github.com/elastic/kibana/commit/f833e09121d6235a22ec377ab17aa2b23b6191fa) | [after](https://github.com/elastic/kibana/commit/7529e81b0f1e381772017697571e57aa724949d4) | diff |
| --- | --- | --- | --- |
| `expressions` | 1774 | 1795 | +21 |
| `inspector` | 101 | 109 | +8 |
| total |  |  | +29 |

#### Page load bundle
> Size of the bundles that are downloaded on every page load. Target size is below 100kb


| id | [before](https://github.com/elastic/kibana/commit/f833e09121d6235a22ec377ab17aa2b23b6191fa) | [after](https://github.com/elastic/kibana/commit/7529e81b0f1e381772017697571e57aa724949d4) | diff |
| --- | --- | --- | --- |
| `data` | 398.0KB | 399.8KB | +1.8KB |
| `expressions` | 92.6KB | 93.4KB | +813.0B |
| `inspector` | 17.0KB | 17.3KB | +320.0B |
| total |  |  | +2.9KB |

<details><summary>Unknown metric groups</summary>

#### API count
| id | [before](https://github.com/elastic/kibana/commit/f833e09121d6235a22ec377ab17aa2b23b6191fa) | [after](https://github.com/elastic/kibana/commit/7529e81b0f1e381772017697571e57aa724949d4) | diff |
| --- | --- | --- | --- |
| `expressions` | 2246 | 2267 | +21 |
| `inspector` | 128 | 136 | +8 |
| total |  |  | +29 |


</details>


### History
* :broken_heart: [Build #291557](https://buildkite.com/elastic/kibana-pull-request/builds/291557) failed 017799f5ddaba24b50d66d6ffc777c0bf7c83076
* :green_heart: [Build #289848](https://buildkite.com/elastic/kibana-pull-request/builds/289848) succeeded d464b93e6276242c5802ccc275f254791932336a
* :broken_heart: [Build #289829](https://buildkite.com/elastic/kibana-pull-request/builds/289829) failed 13910a834cfe99d573a4abb1d225ac1c161f8c00
* :broken_heart: [Build #289797](https://buildkite.com/elastic/kibana-pull-request/builds/289797) failed 2d457dc282cacf53567b069a994d67c67b2212ac
* :broken_heart: [Build #289357](https://buildkite.com/elastic/kibana-pull-request/builds/289357) failed 597ebf9e76f0340eea353749c66e146c4aaaa99d

<!--buildkite-pr-comment-kibana-pull-request
{"builds":[{"buildStatus":{"state":"passed","success":true,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/291593","number":291593,"commit":"7529e81b0f1e381772017697571e57aa724949d4"},{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/291557","number":291557,"commit":"017799f5ddaba24b50d66d6ffc777c0bf7c83076"},{"buildStatus":{"state":"passed","success":true,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/289848","number":289848,"commit":"d464b93e6276242c5802ccc275f254791932336a"},{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/289829","number":289829,"commit":"13910a834cfe99d573a4abb1d225ac1c161f8c00"},{"buildStatus":{"state":"failing","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/289797","number":289797,"commit":"2d457dc282cacf53567b069a994d67c67b2212ac"},{"buildStatus":{"state":"failing","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/289357","number":289357,"commit":"597ebf9e76f0340eea353749c66e146c4aaaa99d"}],"number":291593}
buildkite-pr-comment-->
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-11T12:51:04Z
body: Starting backport for target branches: 8.16, 8.17, 8.18, 8.x, 9.0

https://github.com/elastic/kibana/actions/runs/14403537896
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-11T13:02:09Z
body: ## 💔 All backports failed

| Status | Branch | Result |
|:------:|:------:|:------|
|❌|8.16|Backport failed because of merge conflicts|
|❌|8.17|Backport failed because of merge conflicts|
|❌|8.18|Backport failed because of merge conflicts|
|❌|8.x|Backport failed because of merge conflicts|
|❌|9.0|Backport failed because of merge conflicts|

### Manual backport
To create the backport manually run:
```
node scripts/backport --pr 216519
```

### Questions ?
Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)

<!--- Backport version: 9.6.6 -->
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-15T13:48:44Z
body: Friendly reminder: Looks like this PR hasn’t been backported yet.
To create automatically backports add a `backport:*` label or prevent reminders by adding the `backport:skip` label.
You can also create backports manually by running `node scripts/backport --pr 216519` locally
author_association: CONTRIBUTOR
user: login: dej611
created_at: 2025-04-16T12:13:01Z
body: ## 💚 All backports created successfully

| Status | Branch | Result |
|:------:|:------:|:------|
|✅|9.0|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/kibana/218419">](https://github.com/elastic/kibana/pull/218419)|
|✅|8.x|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/kibana/218422">](https://github.com/elastic/kibana/pull/218422)|
|✅|8.18|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/kibana/218425">](https://github.com/elastic/kibana/pull/218425)|

Note: Successful backport PRs will be merged automatically after passing CI.

### Questions ?
Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)

<!--- Backport version: 9.6.6 -->
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-16T14:50:47Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-17T15:48:53Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-18T16:48:41Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-21T16:48:57Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-22T17:48:43Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-04-23T17:49:04Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR