Repository: meilisearch/meilisearch-go

Issue:
[v1.16.0] Add support for multi-modal search
## Description

We want to update this SDK to allow using **indexing and search fragments** to enable **multi-modal search**. This feature is introduced in [Meilisearch 1.16](https://www.meilisearch.com/blog/meilisearch-1-16?utm_campaign=oss&utm_medium=integrations).

For more information, see the [embedder settings API](https://www.meilisearch.com/docs/reference/api/settings#update-embedder-settings).

NB: As of Meilisearch 1.16, this feature is experimental. This might require updating the API related to experimental features.

## Tasks

- [ ] Update settings methods to allow configuring `indexingFraments` and `searchFragments`
- [ ] Add new tests cases
- [ ] Add code sample in [.code-samples.meilisearch.yaml](../blob/main/.code-samples.meilisearch.yaml) under the `search_parameter_reference_media_1` key. The example should be similar to [this CURL example](https://github.com/meilisearch/documentation/blob/301cf97a0bd3059c9b99abda475c927e4caa32c7/.code-samples.meilisearch.yaml#L1548-L1563)

Discussion:

Solution patch:
diff --git a/.code-samples.meilisearch.yaml b/.code-samples.meilisearch.yaml
index 1258752..e08ed64 100644
--- a/.code-samples.meilisearch.yaml
+++ b/.code-samples.meilisearch.yaml
@@ -1003,3 +1003,16 @@ export_2: |-
       }
     }
   });
+search_parameter_reference_media_1: |-
+  client.Index("INDEX_NAME").Search("", &meilisearch.SearchRequest{
+    Hybrid: &meilisearch.SearchRequestHybrid{
+      Embedder: "EMBEDDER_NAME",
+    },
+    Media: map[string]any{
+      "FIELD_A": "VALUE_A",
+      "FIELD_B": map[string]any{
+        "FIELD_C": "VALUE_B",
+        "FIELD_D": "VALUE_C",
+      },
+    },
+  });
diff --git a/features.go b/features.go
index 0738ed9..8e61403 100644
--- a/features.go
+++ b/features.go
@@ -49,6 +49,11 @@ func (ef *ExperimentalFeatures) SetChatCompletions(enable bool) *ExperimentalFea
 	return ef
 }
 
+func (ef *ExperimentalFeatures) SetMultiModal(enable bool) *ExperimentalFeatures {
+	ef.MultiModal = &enable
+	return ef
+}
+
 func (ef *ExperimentalFeatures) Get() (*ExperimentalFeaturesResult, error) {
 	return ef.GetWithContext(context.Background())
 }
@@ -85,6 +90,7 @@ func (ef *ExperimentalFeatures) UpdateWithContext(ctx context.Context) (*Experim
 		Network:                 ef.Network,
 		CompositeEmbedders:      ef.CompositeEmbedders,
 		ChatCompletions:         ef.ChatCompletions,
+		MultiModal:              ef.MultiModal,
 	}
 	resp := new(ExperimentalFeaturesResult)
 	req := &internalRequest{
diff --git a/types.go b/types.go
index 079d3e4..1fcf2ab 100644
--- a/types.go
+++ b/types.go
@@ -170,6 +170,12 @@ type Embedder struct {
 	Pooling                  EmbedderPooling        `json:"pooling,omitempty"`
 	IndexingEmbedder         *Embedder              `json:"indexingEmbedder,omitempty"` // For Composite
 	SearchEmbedder           *Embedder              `json:"searchEmbedder,omitempty"`   // For Composite
+	IndexingFragments        map[string]Fragment    `json:"indexingFragments,omitempty"`
+	SearchFragments          map[string]Fragment    `json:"searchFragments,omitempty"`
+}
+
+type Fragment struct {
+	Value map[string]any `json:"value,omitempty"`
 }
 
 // Distribution represents a statistical distribution with mean and standard deviation (sigma).
@@ -416,6 +422,7 @@ type SearchRequest struct {
 	RankingScoreThreshold   float64                  `json:"rankingScoreThreshold,omitempty"`
 	FederationOptions       *SearchFederationOptions `json:"federationOptions,omitempty"`
 	Locales                 []string                 `json:"locales,omitempty"`
+	Media                   map[string]any           `json:"media,omitempty"`
 }
 
 type SearchFederationOptions struct {
@@ -556,6 +563,7 @@ type ExperimentalFeaturesBase struct {
 	Network                 *bool `json:"network,omitempty"`
 	CompositeEmbedders      *bool `json:"compositeEmbedders,omitempty"`
 	ChatCompletions         *bool `json:"chatCompletions,omitempty"`
+	MultiModal              *bool `json:"multimodal,omitempty"`
 }
 
 type ExperimentalFeaturesResult struct {
@@ -566,6 +574,7 @@ type ExperimentalFeaturesResult struct {
 	Network                 bool `json:"network"`
 	CompositeEmbedders      bool `json:"compositeEmbedders"`
 	ChatCompletions         bool `json:"chatCompletions"`
+	MultiModal              bool `json:"multimodal,omitempty"`
 }
 
 type SwapIndexesParams struct {



Test patch:
diff --git a/integration/features_test.go b/integration/features_test.go
index 8106896..eb0bf82 100644
--- a/integration/features_test.go
+++ b/integration/features_test.go
@@ -68,6 +68,8 @@ func TestUpdate_ExperimentalFeatures(t *testing.T) {
 			ef.SetNetwork(true)
 			ef.SetCompositeEmbedders(true)
 			ef.SetChatCompletions(true)
+			ef.SetMultiModal(true)
+			
 			gotResp, err := ef.Update()
 			require.NoError(t, err)
 
@@ -78,6 +80,7 @@ func TestUpdate_ExperimentalFeatures(t *testing.T) {
 			require.Equal(t, true, gotResp.Network, "ExperimentalFeatures.Update() should return network as true")
 			require.Equal(t, true, gotResp.CompositeEmbedders, "ExperimentalFeatures.Update() should return composite embedders as true")
 			require.Equal(t, true, gotResp.ChatCompletions, "ExperimentalFeatures.Update() should return chat completions as true")
+			require.Equal(t, true, gotResp.MultiModal, "ExperimentalFeatures.Update() should return multi modal as true")
 		})
 	}
 }
diff --git a/integration/index_settings_test.go b/integration/index_settings_test.go
index 85ffa30..c0e03ba 100644
--- a/integration/index_settings_test.go
+++ b/integration/index_settings_test.go
@@ -4000,43 +4000,167 @@ func TestIndex_GetEmbedders(t *testing.T) {
 }
 
 func TestIndex_UpdateEmbedders(t *testing.T) {
-	c := setup(t, "")
-	t.Cleanup(cleanup(c))
-
-	indexID := "newIndexUID"
-	i := c.Index(indexID)
-	taskInfo, err := c.CreateIndex(&meilisearch.IndexConfig{Uid: indexID})
-	require.NoError(t, err)
-	testWaitForTask(t, i, taskInfo)
-
-	embedders := map[string]meilisearch.Embedder{
-		"someEmbbeder": {
-			Source:     "userProvided",
-			Dimensions: 3,
+	tests := []struct {
+		name         string
+		indexID      string
+		experimental bool
+		initialEmbed map[string]meilisearch.Embedder
+		updatedEmbed map[string]meilisearch.Embedder
+	}{
+		{
+			name:         "update existing embedder dimensions",
+			indexID:      "newIndexUID",
+			experimental: false,
+			initialEmbed: map[string]meilisearch.Embedder{
+				"someEmbbeder": {
+					Source:     "userProvided",
+					Dimensions: 3,
+				},
+			},
+			updatedEmbed: map[string]meilisearch.Embedder{
+				"someEmbbeder": {
+					Source:     "userProvided",
+					Dimensions: 5,
+				},
+			},
 		},
-	}
-	taskInfo, err = i.UpdateSettings(&meilisearch.Settings{
-		Embedders: embedders,
-	})
-	require.NoError(t, err)
-	testWaitForTask(t, i, taskInfo)
-
-	updated := map[string]meilisearch.Embedder{
-		"someEmbbeder": {
-			Source:     "userProvided",
-			Dimensions: 5,
+		{
+			name:         "update indexingFragments & searchFragments",
+			indexID:      "newIndexUID",
+			experimental: true,
+			initialEmbed: map[string]meilisearch.Embedder{
+				"voyageAI": {
+					Source: "rest",
+					URL:    "https://api.voyageai.com/v1/multimodalembeddings",
+					Request: map[string]any{
+						"inputs": []string{
+							"{{fragment}}",
+							"{{..}}",
+						},
+						"model": "voyage-multimodal-3",
+					},
+					Response: map[string]any{
+						"response": map[string]any{
+							"data": []any{
+								map[string]any{
+									"embedding": "{{embedding}}",
+								},
+								"{{..}}",
+							},
+						},
+					},
+					Dimensions: 5,
+					IndexingFragments: map[string]meilisearch.Fragment{
+						"text": {
+							Value: map[string]any{
+								"content": []any{
+									map[string]any{
+										"type": "text",
+										"text": "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.",
+									},
+								},
+							},
+						},
+					},
+					SearchFragments: map[string]meilisearch.Fragment{
+						"text": {
+							Value: map[string]any{
+								"content": []any{
+									map[string]any{
+										"type": "text",
+										"text": "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			updatedEmbed: map[string]meilisearch.Embedder{
+				"voyageAI": {
+					Source:  "rest",
+					URL:     "https://api.voyageai.com/v1/multimodalembeddings",
+					Headers: map[string]string{},
+					Request: map[string]any{
+						"inputs": []any{
+							"{{fragment}}",
+							"{{..}}",
+						},
+						"model": "voyage-multimodal-3",
+					},
+					Response: map[string]any{
+						"response": map[string]any{
+							"data": []any{
+								map[string]any{
+									"embedding": "{{embedding}}",
+								},
+								"{{..}}",
+							},
+						},
+					},
+					Dimensions: 5,
+					IndexingFragments: map[string]meilisearch.Fragment{
+						"text": {
+							Value: map[string]any{
+								"content": []any{
+									map[string]any{
+										"type": "text",
+										"text": "A movie titled {{doc.title}} whose description starts with {{doc.overview|truncatewords:20}}.",
+									},
+								},
+							},
+						},
+					},
+					SearchFragments: map[string]meilisearch.Fragment{
+						"text": {
+							Value: map[string]any{
+								"content": []any{
+									map[string]any{
+										"type": "text",
+										"text": "{{q}}",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
 		},
 	}
 
-	taskInfo, err = i.UpdateEmbedders(updated)
-	require.NoError(t, err)
-	task, err := i.WaitForTask(taskInfo.TaskUID, 0)
-	require.NoError(t, err)
-	require.Equal(t, meilisearch.TaskStatusSucceeded, task.Status)
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := setup(t, "")
+			t.Cleanup(cleanup(c))
 
-	got, err := i.GetEmbedders()
-	require.NoError(t, err)
-	require.Equal(t, updated, got)
+			i := c.Index(tt.indexID)
+			taskInfo, err := c.CreateIndex(&meilisearch.IndexConfig{Uid: tt.indexID})
+			require.NoError(t, err)
+			testWaitForTask(t, i, taskInfo)
+
+			if tt.experimental {
+				result, err := c.ExperimentalFeatures().SetMultiModal(true).Update()
+				require.NoError(t, err)
+				require.True(t, result.MultiModal)
+			}
+
+			taskInfo, err = i.UpdateSettings(&meilisearch.Settings{
+				Embedders: tt.initialEmbed,
+			})
+			require.NoError(t, err)
+			testWaitForTask(t, i, taskInfo)
+
+			taskInfo, err = i.UpdateEmbedders(tt.updatedEmbed)
+			require.NoError(t, err)
+			task, err := i.WaitForTask(taskInfo.TaskUID, 0)
+			require.NoError(t, err)
+			require.Equal(t, meilisearch.TaskStatusSucceeded, task.Status)
+
+			got, err := i.GetEmbedders()
+			require.NoError(t, err)
+			require.Equal(t, tt.updatedEmbed, got)
+		})
+	}
 }
 
 func TestIndex_ResetEmbedders(t *testing.T) {