Repository: elastic/kibana
Pull request #214721: add API title_conversation tests
Author: login: arturoliduena
Labels: name: release_note:skip
description: Skip the PR/issue when compiling release notes
name: Team:Obs AI Assistant
description: Observability AI Assistant
name: backport:version
description: Backport to applied version labels
name: v9.1.0
name: v8.19.0
Created: 2025-03-17T11:47:36Z | Merged: 2025-03-25T16:10:02Z

Related: https://github.com/elastic/kibana/issues/180787

- Adds test for `title_conversation` function

<!--ONMERGE {"backportTargets":["8.x","9.0"]} ONMERGE-->


Diff:
diff --git a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/operators/get_generated_title.ts b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/operators/get_generated_title.ts
index 89a43d62af20d..b5cb3fce7fcbc 100644
--- a/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/operators/get_generated_title.ts
+++ b/x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/operators/get_generated_title.ts
@@ -13,7 +13,8 @@ import { Message, MessageRole } from '../../../../common';
 import { LangTracer } from '../instrumentation/lang_tracer';
 
 export const TITLE_CONVERSATION_FUNCTION_NAME = 'title_conversation';
-
+export const TITLE_SYSTEM_MESSAGE =
+  'You are a helpful assistant for Elastic Observability. Assume the following message is the start of a conversation between you and a user; give this conversation a title based on the content below. DO NOT UNDER ANY CIRCUMSTANCES wrap this title in single or double quotes. This title is shown in a list of conversations to the user, so title it for the user, not for you.';
 type ChatFunctionWithoutConnectorAndTokenCount = (
   name: string,
   params: Omit<
@@ -35,8 +36,7 @@ export function getGeneratedTitle({
 }): Observable<string> {
   return from(
     chat('generate_title', {
-      systemMessage:
-        'You are a helpful assistant for Elastic Observability. Assume the following message is the start of a conversation between you and a user; give this conversation a title based on the content below. DO NOT UNDER ANY CIRCUMSTANCES wrap this title in single or double quotes. This title is shown in a list of conversations to the user, so title it for the user, not for you.',
+      systemMessage: TITLE_SYSTEM_MESSAGE,
       messages: [
         {
           '@timestamp': new Date().toISOString(),
diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
new file mode 100644
index 0000000000000..086c017cd9d38
--- /dev/null
+++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
@@ -0,0 +1,103 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';
+import {
+  TITLE_CONVERSATION_FUNCTION_NAME,
+  TITLE_SYSTEM_MESSAGE,
+} from '@kbn/observability-ai-assistant-plugin/server/service/client/operators/get_generated_title';
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { chatComplete } from '../../utils/conversation';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const log = getService('log');
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
+
+  describe('when calling the title_conversation function', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
+    let llmProxy: LlmProxy;
+    let connectorId: string;
+
+    before(async () => {
+      llmProxy = await createLlmProxy(log);
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: llmProxy.getPort(),
+      });
+    });
+
+    after(async () => {
+      llmProxy.close();
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
+    });
+
+    // Calling `title_conversation` via the chat/complete endpoint
+    describe('POST /internal/observability_ai_assistant/chat/complete', function () {
+      let titleRequestBody: ChatCompletionStreamParams;
+
+      const TITLE = 'Question about color of the sky';
+      let conversationId: string;
+
+      before(async () => {
+        void llmProxy.interceptTitle(TITLE);
+        void llmProxy.interceptConversation('The sky is blue because of Rayleigh scattering.');
+
+        const res = await chatComplete({
+          userPrompt: 'Why the sky is blue?',
+          connectorId,
+          persist: true,
+          observabilityAIAssistantAPIClient,
+        });
+        conversationId = res.conversationCreateEvent?.conversation.id || '';
+
+        await llmProxy.waitForAllInterceptorsToHaveBeenCalled();
+
+        titleRequestBody = llmProxy.interceptedRequests[0].requestBody;
+      });
+
+      it('makes 2 requests to the LLM', () => {
+        expect(llmProxy.interceptedRequests.length).to.be(2);
+      });
+
+      it('sends the correct system message to the LLM for the title', () => {
+        expect(
+          titleRequestBody.messages.find((message) => message.role === 'system')?.content
+        ).to.be(TITLE_SYSTEM_MESSAGE);
+      });
+
+      it('sends the correct user message to the LLM for the title', () => {
+        expect(
+          titleRequestBody.messages.find((message) => message.role === 'user')?.content
+        ).to.contain('Why the sky is blue?');
+      });
+
+      it('sends the correct function call to the LLM for the title', () => {
+        expect(titleRequestBody.tools?.[0].function.name).to.be(TITLE_CONVERSATION_FUNCTION_NAME);
+      });
+
+      it('stores the generated title in the conversation', async () => {
+        const { status, body } = await observabilityAIAssistantAPIClient.editor({
+          endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}',
+          params: {
+            path: {
+              conversationId,
+            },
+          },
+        });
+        expect(status).to.be(200);
+        expect(body.conversation.title).to.be(TITLE);
+      });
+    });
+  });
+}
diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts
index c152e82f6f883..bec363472c968 100644
--- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/index.ts
@@ -24,6 +24,7 @@ export default function aiAssistantApiIntegrationTests({
     loadTestFile(require.resolve('./complete/functions/summarize.spec.ts'));
     loadTestFile(require.resolve('./complete/functions/recall.spec.ts'));
     loadTestFile(require.resolve('./complete/functions/context.spec.ts'));
+    loadTestFile(require.resolve('./complete/functions/title_conversation.spec.ts'));
     loadTestFile(require.resolve('./public_complete/public_complete.spec.ts'));
     loadTestFile(require.resolve('./knowledge_base/knowledge_base_setup.spec.ts'));
     loadTestFile(
diff --git a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/conversation.ts b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/conversation.ts
index 437297e7d82a6..d16abb88d6632 100644
--- a/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/conversation.ts
+++ b/x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/utils/conversation.ts
@@ -84,11 +84,13 @@ export async function chatComplete({
   userPrompt,
   screenContexts = [],
   connectorId,
+  persist = false,
   observabilityAIAssistantAPIClient,
 }: {
   userPrompt: string;
   screenContexts?: ObservabilityAIAssistantScreenContextRequest[];
   connectorId: string;
+  persist?: boolean;
   observabilityAIAssistantAPIClient: ObservabilityAIAssistantApiClient;
 }) {
   const { status, body } = await observabilityAIAssistantAPIClient.editor({
@@ -105,7 +107,7 @@ export async function chatComplete({
           },
         ],
         connectorId,
-        persist: false,
+        persist,
         screenContexts,
         scopes: ['observability' as const],
       },
@@ -113,9 +115,10 @@ export async function chatComplete({
   });
 
   expect(status).to.be(200);
+  const messageEvents = decodeEvents(body);
   const messageAddedEvents = getMessageAddedEvents(body);
-
-  return { messageAddedEvents, body, status };
+  const conversationCreateEvent = getConversationCreatedEvent(body);
+  return { messageAddedEvents, conversationCreateEvent, messageEvents, status };
 }
 
 // order of instructions can vary, so we sort to compare them
@@ -160,12 +163,6 @@ export function getConversationCreatedEvent(body: Readable | string) {
     (event) => event.type === StreamingChatResponseEventType.ConversationCreate
   ) as ConversationCreateEvent;
 
-  if (!conversationCreatedEvent) {
-    throw new Error(
-      `No conversation created event found: ${JSON.stringify(decodedEvents, null, 2)}`
-    );
-  }
-
   return conversationCreatedEvent;
 }
 


Reviews:
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T11:58:10Z
user: login: arturoliduena
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T09:50:45Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:47:07Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:48:37Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:50:58Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:52:04Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:52:23Z
user: login: sorenlouv
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T10:52:44Z
user: login: arturoliduena
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T15:05:46Z
user: login: arturoliduena
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T15:20:27Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T19:23:36Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T19:24:02Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T19:25:17Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-18T19:26:46Z
user: login: arturoliduena
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T21:20:20Z
user: login: arturoliduena
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T21:27:40Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T22:23:21Z
user: login: sorenlouv
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T22:24:30Z
user: login: sorenlouv
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-19T22:25:50Z

Review comments:
diff_hunk: @@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MessageAddEvent } from '@kbn/observability-ai-assistant-plugin/common';
+import expect from '@kbn/expect';
+import { ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';
+import {
+  TITLE_CONVERSATION_FUNCTION_NAME,
+  TITLE_SYSTEM_MESSAGE,
+} from '@kbn/observability-ai-assistant-plugin/server/service/client/operators/get_generated_title';
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { chatComplete, getMessageAddedEvents } from './helpers';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const log = getService('log');
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
+
+  describe('when calling the title_conversation function', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
+    let llmProxy: LlmProxy;
+    let connectorId: string;
+
+    before(async () => {
+      llmProxy = await createLlmProxy(log);
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: llmProxy.getPort(),
+      });
+    });
+
+    after(async () => {
+      llmProxy.close();
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
+    });
+
+    // Calling `title_conversation` via the chat/complete endpoint
+    describe('POST /internal/observability_ai_assistant/chat/complete', function () {
+      let messageAddedEvents: MessageAddEvent[];
+      let titleRequestBody: ChatCompletionStreamParams;
+
+      const USER_MESSAGE = 'Why the sky is blue?';
+
+      before(async () => {
+        void llmProxy.interceptTitle('title');
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
user: login: sorenlouv
body: Can you add a more unique title and verify that it is actually stored as the title in ES?

```suggestion
        void llmProxy.interceptTitle('Question about color of the sky');
```
created_at: 2025-03-17T11:58:10Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { MessageAddEvent } from '@kbn/observability-ai-assistant-plugin/common';
+import expect from '@kbn/expect';
+import { ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';
+import {
+  TITLE_CONVERSATION_FUNCTION_NAME,
+  TITLE_SYSTEM_MESSAGE,
+} from '@kbn/observability-ai-assistant-plugin/server/service/client/operators/get_generated_title';
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { chatComplete, getMessageAddedEvents } from './helpers';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const log = getService('log');
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
+
+  describe('when calling the title_conversation function', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
+    let llmProxy: LlmProxy;
+    let connectorId: string;
+
+    before(async () => {
+      llmProxy = await createLlmProxy(log);
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: llmProxy.getPort(),
+      });
+    });
+
+    after(async () => {
+      llmProxy.close();
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
+    });
+
+    // Calling `title_conversation` via the chat/complete endpoint
+    describe('POST /internal/observability_ai_assistant/chat/complete', function () {
+      let messageAddedEvents: MessageAddEvent[];
+      let titleRequestBody: ChatCompletionStreamParams;
+
+      const USER_MESSAGE = 'Why the sky is blue?';
+
+      before(async () => {
+        void llmProxy.interceptTitle('title');
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
user: login: arturoliduena
body: agreed, I've added a test to cover `stores the generated title in the conversation`
created_at: 2025-03-18T09:50:45Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,7 +71,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
 
       await proxy.waitForAllInterceptorsToHaveBeenCalled();
 
-      events = getMessageAddedEvents(responseBody);
+      events = getMessageAddedEvents(decodeEvents(responseBody));
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/elasticsearch.spec.ts
user: login: sorenlouv
body: I think `getMessageAddedEvents` already calls `decodeEvents`
created_at: 2025-03-18T10:47:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,7 +71,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
 
       await proxy.waitForAllInterceptorsToHaveBeenCalled();
 
-      events = getMessageAddedEvents(responseBody);
+      events = getMessageAddedEvents(decodeEvents(responseBody));
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/elasticsearch.spec.ts
user: login: sorenlouv
body: I see you moved decoding out. Why?
created_at: 2025-03-18T10:48:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -98,17 +105,18 @@ export async function chatComplete({
           },
         ],
         connectorId,
-        persist: false,
+        persist,
         screenContexts: [],
         scopes: ['observability' as const],
       },
     },
   });
 
   expect(status).to.be(200);
-  const messageAddedEvents = getMessageAddedEvents(body);
-
-  return { messageAddedEvents, body, status };
+  const decodedBody = decodeEvents(body);
+  const messageAddedEvents = getMessageAddedEvents(decodedBody);
+  const conversation = getConversationCreateEvent(decodedBody);
+  return { messageAddedEvents, conversation, body: decodedBody, status };
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: sorenlouv
body: ```suggestion
  const messageEvents = decodeEvents(body);
  const messageAddedEvents = getMessageAddedEvents(decodedBody);
  const conversation = getConversationCreateEvent(decodedBody);
  return { messageAddedEvents, conversation, messageEvents, status };
```
created_at: 2025-03-18T10:50:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';
+import {
+  TITLE_CONVERSATION_FUNCTION_NAME,
+  TITLE_SYSTEM_MESSAGE,
+} from '@kbn/observability-ai-assistant-plugin/server/service/client/operators/get_generated_title';
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { chatComplete } from './helpers';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const log = getService('log');
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
+
+  describe('when calling the title_conversation function', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
+    let llmProxy: LlmProxy;
+    let connectorId: string;
+
+    before(async () => {
+      llmProxy = await createLlmProxy(log);
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: llmProxy.getPort(),
+      });
+    });
+
+    after(async () => {
+      llmProxy.close();
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
+    });
+
+    // Calling `title_conversation` via the chat/complete endpoint
+    describe('POST /internal/observability_ai_assistant/chat/complete', function () {
+      let titleRequestBody: ChatCompletionStreamParams;
+
+      const USER_MESSAGE = 'Why the sky is blue?';
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
user: login: sorenlouv
body: Unless this is re-used I think it's fine to inline.
created_at: 2025-03-18T10:52:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';
+import {
+  TITLE_CONVERSATION_FUNCTION_NAME,
+  TITLE_SYSTEM_MESSAGE,
+} from '@kbn/observability-ai-assistant-plugin/server/service/client/operators/get_generated_title';
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { chatComplete } from './helpers';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const log = getService('log');
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
+
+  describe('when calling the title_conversation function', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
+    let llmProxy: LlmProxy;
+    let connectorId: string;
+
+    before(async () => {
+      llmProxy = await createLlmProxy(log);
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: llmProxy.getPort(),
+      });
+    });
+
+    after(async () => {
+      llmProxy.close();
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
+    });
+
+    // Calling `title_conversation` via the chat/complete endpoint
+    describe('POST /internal/observability_ai_assistant/chat/complete', function () {
+      let titleRequestBody: ChatCompletionStreamParams;
+
+      const USER_MESSAGE = 'Why the sky is blue?';
+      const TITLE = 'Question about color of the sky';
+      let conversationId: string;
+
+      before(async () => {
+        void llmProxy.interceptTitle(TITLE);
+        void llmProxy.interceptConversation('The sky is blue because of Rayleigh scattering.');
+
+        const res = await chatComplete({
+          userPrompt: USER_MESSAGE,
+          connectorId,
+          persist: true,
+          observabilityAIAssistantAPIClient,
+        });
+        conversationId = res.conversation?.conversation.id || '';
+
+        await llmProxy.waitForAllInterceptorsToHaveBeenCalled();
+
+        titleRequestBody = llmProxy.interceptedRequests[0].requestBody;
+      });
+
+      it('makes 2 requests to the LLM', () => {
+        expect(llmProxy.interceptedRequests.length).to.be(2);
+      });
+
+      it('sends the correct system message to the LLM for the title', () => {
+        expect(
+          titleRequestBody.messages.find((message) => message.role === 'system')?.content
+        ).to.be(TITLE_SYSTEM_MESSAGE);
+      });
+
+      it('sends the correct user message to the LLM for the title', () => {
+        expect(
+          titleRequestBody.messages.find((message) => message.role === 'user')?.content
+        ).to.contain('Why the sky is blue?');
+      });
+
+      it('sends the correct function call to the LLM for the title', () => {
+        expect(titleRequestBody.tools?.[0].function.name).to.be(TITLE_CONVERSATION_FUNCTION_NAME);
+      });
+
+      it('stores the generated title in the conversation', async () => {
+        const { status, body } = await observabilityAIAssistantAPIClient.editor({
+          endpoint: 'GET /internal/observability_ai_assistant/conversation/{conversationId}',
+          params: {
+            path: {
+              conversationId,
+            },
+          },
+        });
+        expect(status).to.be(200);
+        expect(body.conversation.title).to.be(TITLE);
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
user: login: sorenlouv
body: Thanks!
created_at: 2025-03-18T10:52:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -67,7 +71,7 @@ export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderCon
 
       await proxy.waitForAllInterceptorsToHaveBeenCalled();
 
-      events = getMessageAddedEvents(responseBody);
+      events = getMessageAddedEvents(decodeEvents(responseBody));
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/elasticsearch.spec.ts
user: login: arturoliduena
body: bc I added a `getConversationCreateEvent` that also uses decode, I moved the decoding outside of each function instead of doing it individually within them. let me move it back in. 
created_at: 2025-03-18T15:05:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { ChatCompletionStreamParams } from 'openai/lib/ChatCompletionStream';
+import {
+  TITLE_CONVERSATION_FUNCTION_NAME,
+  TITLE_SYSTEM_MESSAGE,
+} from '@kbn/observability-ai-assistant-plugin/server/service/client/operators/get_generated_title';
+import {
+  LlmProxy,
+  createLlmProxy,
+} from '../../../../../../../observability_ai_assistant_api_integration/common/create_llm_proxy';
+import { chatComplete } from './helpers';
+import type { DeploymentAgnosticFtrProviderContext } from '../../../../../ftr_provider_context';
+
+export default function ApiTest({ getService }: DeploymentAgnosticFtrProviderContext) {
+  const log = getService('log');
+  const observabilityAIAssistantAPIClient = getService('observabilityAIAssistantApi');
+
+  describe('when calling the title_conversation function', function () {
+    // Fails on MKI: https://github.com/elastic/kibana/issues/205581
+    this.tags(['failsOnMKI']);
+    let llmProxy: LlmProxy;
+    let connectorId: string;
+
+    before(async () => {
+      llmProxy = await createLlmProxy(log);
+      connectorId = await observabilityAIAssistantAPIClient.createProxyActionConnector({
+        port: llmProxy.getPort(),
+      });
+    });
+
+    after(async () => {
+      llmProxy.close();
+      await observabilityAIAssistantAPIClient.deleteActionConnector({
+        actionId: connectorId,
+      });
+    });
+
+    // Calling `title_conversation` via the chat/complete endpoint
+    describe('POST /internal/observability_ai_assistant/chat/complete', function () {
+      let titleRequestBody: ChatCompletionStreamParams;
+
+      const USER_MESSAGE = 'Why the sky is blue?';
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/title_conversation.spec.ts
user: login: arturoliduena
body: done
created_at: 2025-03-18T15:20:27Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,6 +31,12 @@ export function getMessageAddedEvents(body: Readable | string) {
   );
 }
 
+export function getConversationCreateEvent(body: Readable | string) {
+  return decodeEvents(body).find(
+    (event): event is ConversationCreateEvent => event.type === 'conversationCreate'
+  );
+}
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: sorenlouv
body: I think this file has been moved to the utils folder.
created_at: 2025-03-18T19:23:36Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,6 +31,12 @@ export function getMessageAddedEvents(body: Readable | string) {
   );
 }
 
+export function getConversationCreateEvent(body: Readable | string) {
+  return decodeEvents(body).find(
+    (event): event is ConversationCreateEvent => event.type === 'conversationCreate'
+  );
+}
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: sorenlouv
body: Try pulling from main to get the latest
created_at: 2025-03-18T19:24:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,6 +31,12 @@ export function getMessageAddedEvents(body: Readable | string) {
   );
 }
 
+export function getConversationCreateEvent(body: Readable | string) {
+  return decodeEvents(body).find(
+    (event): event is ConversationCreateEvent => event.type === 'conversationCreate'
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: sorenlouv
body: nit: can you use the enum?

```suggestion
    (event): event is ConversationCreateEvent => event.type === StreamingChatResponseEventType.ConversationCreate
```
created_at: 2025-03-18T19:25:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -98,17 +107,18 @@ export async function chatComplete({
           },
         ],
         connectorId,
-        persist: false,
+        persist,
         screenContexts: [],
         scopes: ['observability' as const],
       },
     },
   });
 
   expect(status).to.be(200);
+  const messageEvents = decodeEvents(body);
   const messageAddedEvents = getMessageAddedEvents(body);
-
-  return { messageAddedEvents, body, status };
+  const conversation = getConversationCreateEvent(body);
+  return { messageAddedEvents, conversation, messageEvents, status };
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: sorenlouv
body: ```suggestion
  const conversationCreateEvent = getConversationCreateEvent(body);
  return { messageAddedEvents, conversationCreateEvent, messageEvents, status };
```
created_at: 2025-03-18T19:26:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -98,17 +107,18 @@ export async function chatComplete({
           },
         ],
         connectorId,
-        persist: false,
+        persist,
         screenContexts: [],
         scopes: ['observability' as const],
       },
     },
   });
 
   expect(status).to.be(200);
+  const messageEvents = decodeEvents(body);
   const messageAddedEvents = getMessageAddedEvents(body);
-
-  return { messageAddedEvents, body, status };
+  const conversation = getConversationCreateEvent(body);
+  return { messageAddedEvents, conversation, messageEvents, status };
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: arturoliduena
body: changed
created_at: 2025-03-19T21:20:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -30,6 +31,12 @@ export function getMessageAddedEvents(body: Readable | string) {
   );
 }
 
+export function getConversationCreateEvent(body: Readable | string) {
+  return decodeEvents(body).find(
+    (event): event is ConversationCreateEvent => event.type === 'conversationCreate'
path: x-pack/test/api_integration/deployment_agnostic/apis/observability/ai_assistant/complete/functions/helpers.ts
user: login: arturoliduena
body: I've deleted this `getConversationCreateEvent` function, It was duplicated. 
created_at: 2025-03-19T21:27:40Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,7 +13,8 @@ import { Message, MessageRole } from '../../../../common';
 import { LangTracer } from '../instrumentation/lang_tracer';
path: x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/operators/get_generated_title.ts
user: login: sorenlouv
body: Shouldn't this file be in https://github.com/elastic/kibana/tree/main/x-pack/platform/plugins/shared/observability_ai_assistant/server/functions ?
created_at: 2025-03-19T22:23:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -13,7 +13,8 @@ import { Message, MessageRole } from '../../../../common';
 import { LangTracer } from '../instrumentation/lang_tracer';
path: x-pack/platform/plugins/shared/observability_ai_assistant/server/service/client/operators/get_generated_title.ts
user: login: sorenlouv
body: Same with the score function. I suggest extracting the score function from `score_suggestions`.

https://github.com/elastic/kibana/blob/81f69713f37dc4f355744327ee9ae84a54d9f147/x-pack/platform/plugins/shared/observability_ai_assistant/server/utils/recall/score_suggestions.ts#L93-L122
created_at: 2025-03-19T22:24:30Z
author_association: CONTRIBUTOR

PR comments:
user: login: elasticmachine
created_at: 2025-03-17T11:47:42Z
body: Pinging @elastic/obs-ai-assistant (Team:Obs AI Assistant)
author_association: CONTRIBUTOR
user: login: elasticmachine
created_at: 2025-03-25T14:15:26Z
body: ## :hourglass_flowing_sand: Build in-progress, with failures
* [Buildkite Build](https://buildkite.com/elastic/kibana-pull-request/builds/286915)
* Commit: 6ba58097bca6b398e6e809abc71968f84afa18e7

### Failed CI Steps
* [x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts](https://buildkite.com/elastic/kibana-pull-request/builds/286915#0195cd66-6538-481e-8b83-23d924ea93b8)
* [FTR Configs #16](https://buildkite.com/elastic/kibana-pull-request/builds/286915#0195cd66-6545-4f19-8aa5-32b3890daefc)
* [FTR Configs #16](https://buildkite.com/elastic/kibana-pull-request/builds/286915#0195cd89-6c18-4d4b-bb67-ddd586f67869)

### Test Failures
* [[job]](https://buildkite.com/elastic/kibana-pull-request/builds/286915#0195cd66-6538-481e-8b83-23d924ea93b8) [[logs]](https://buildkite.com/organizations/elastic/pipelines/kibana-pull-request/builds/286915/jobs/0195cd66-6538-481e-8b83-23d924ea93b8/artifacts/0195cd8d-ad0c-4807-a144-cced3f556384) x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts / Serverless Observability - Deployment-agnostic API integration tests observability AI Assistant get_alerts_dataset_info "before all" hook in "get_alerts_dataset_info"
* [[job]](https://buildkite.com/elastic/kibana-pull-request/builds/286915#0195cd66-6545-4f19-8aa5-32b3890daefc) [[logs]](https://buildkite.com/organizations/elastic/pipelines/kibana-pull-request/builds/286915/jobs/0195cd66-6545-4f19-8aa5-32b3890daefc/artifacts/0195cd89-417e-4934-8fcb-c292c5dc6290) FTR Configs #<span></span>16 / Stateful Observability - Deployment-agnostic API integration tests observability AI Assistant get_alerts_dataset_info "before all" hook in "get_alerts_dataset_info"
* [[job]](https://buildkite.com/elastic/kibana-pull-request/builds/286915#0195cd89-6c18-4d4b-bb67-ddd586f67869) [[logs]](https://buildkite.com/organizations/elastic/pipelines/kibana-pull-request/builds/286915/jobs/0195cd89-6c18-4d4b-bb67-ddd586f67869/artifacts/0195cda8-4e30-4aee-8d5e-fdf89d0991ff) FTR Configs #<span></span>16 / Stateful Observability - Deployment-agnostic API integration tests observability AI Assistant get_alerts_dataset_info "before all" hook in "get_alerts_dataset_info"

### History
* :broken_heart: [Build #285906](https://buildkite.com/elastic/kibana-pull-request/builds/285906) failed bc21b62a9be55eef94eac6d6db934d2a580d5ac6
* :broken_heart: [Build #285586](https://buildkite.com/elastic/kibana-pull-request/builds/285586) failed 2178933bff8e5ccd537f862b851cea605cb2dcb8
* :broken_heart: [Build #285530](https://buildkite.com/elastic/kibana-pull-request/builds/285530) failed f177e429826ee0c94b7c8334bf9a29febc259755
* :broken_heart: [Build #285451](https://buildkite.com/elastic/kibana-pull-request/builds/285451) failed 7bbfddf6d748157f4d33a3715fafdcaa5d0e54a0
* :broken_heart: [Build #285044](https://buildkite.com/elastic/kibana-pull-request/builds/285044) failed aadf4a2d3b4e037ae7bbaaac26623f5943d7fc36

<!--buildkite-pr-comment-kibana-pull-request
{"builds":[{"buildStatus":{"state":"failing","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/286915","number":286915,"commit":"6ba58097bca6b398e6e809abc71968f84afa18e7"},{"buildStatus":{"state":"failed","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/285906","number":285906,"commit":"bc21b62a9be55eef94eac6d6db934d2a580d5ac6"},{"buildStatus":{"state":"failed","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/285586","number":285586,"commit":"2178933bff8e5ccd537f862b851cea605cb2dcb8"},{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/285530","number":285530,"commit":"f177e429826ee0c94b7c8334bf9a29febc259755"},{"buildStatus":{"state":"failed","success":false,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/285451","number":285451,"commit":"7bbfddf6d748157f4d33a3715fafdcaa5d0e54a0"},{"buildStatus":{"state":"failed","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/kibana-pull-request/builds/285044","number":285044,"commit":"aadf4a2d3b4e037ae7bbaaac26623f5943d7fc36"}],"number":286915}
buildkite-pr-comment-->
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-03-25T16:10:21Z
body: Starting backport for target branches: 8.x, 9.0

https://github.com/elastic/kibana/actions/runs/14064589937
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-03-25T16:17:08Z
body: ## 💔 All backports failed

| Status | Branch | Result |
|:------:|:------:|:------|
|❌|8.x|**Backport failed because of merge conflicts**<br><br>You might need to backport the following PRs to 8.x:<br> - [[Obs AI Assistant] Fix contextual insights scoring (#214259)](https://github.com/elastic/kibana/pull/214259)|
|❌|9.0|Backport failed because of merge conflicts|

### Manual backport
To create the backport manually run:
```
node scripts/backport --pr 214721
```

### Questions ?
Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)

<!--- Backport version: 9.6.6 -->
author_association: CONTRIBUTOR
user: login: arturoliduena
created_at: 2025-03-26T07:59:41Z
body: ## 💚 All backports created successfully

| Status | Branch | Result |
|:------:|:------:|:------|
|✅|9.0|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/kibana/215992">](https://github.com/elastic/kibana/pull/215992)|
|✅|8.x|[<img src="https://img.shields.io/github/pulls/detail/state/elastic/kibana/215993">](https://github.com/elastic/kibana/pull/215993)|

Note: Successful backport PRs will be merged automatically after passing CI.

### Questions ?
Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)

<!--- Backport version: 9.6.6 -->
author_association: CONTRIBUTOR
user: login: kibanamachine
created_at: 2025-03-26T16:49:13Z
body: Looks like this PR has backport PRs but they still haven't been merged. Please merge them ASAP to keep the branches relatively in sync.
author_association: CONTRIBUTOR
user: login: arturoliduena
created_at: 2025-03-26T22:46:01Z
body: No need to backport to 9.0.
author_association: CONTRIBUTOR