Merge #593

593: Add AI-powered search changes related to v1.10 r=curquiza a=Ja7ad # Pull Request ## Related issue Fixes #576 ## What does this PR do? - Update the embedders types + tests according to v1.10 - Fix name convation for "APIKey" instead "ApiKey". ## PR checklist Please check if your PR fulfills the following requirements: - [x] Does this PR fix an existing issue, or have you listed the changes applied in the PR description (and why they are needed)? - [x] Have you read the contributing guidelines? - [x] Have you made sure that the title is accurate and descriptive of the changes? Thank you so much for contributing to Meilisearch! Co-authored-by: Javad <[email protected]> Co-authored-by: Javad Rajabzadeh <[email protected]>
meilisearch · Dec 30, 2024 · add5268 · add5268
2 parents c167439 + 13a25b1
commit add5268
Show file tree

Hide file tree

Showing 3 changed files with 489 additions and 200 deletions.
diff --git a/index_settings_test.go b/index_settings_test.go
@@ -3421,7 +3421,7 @@ func TestIndex_UpdateSettingsEmbedders(t *testing.T) {
 					Embedders: map[string]Embedder{
 						"default": {
 							Source:           "openAi",
-							ApiKey:           "xxx",
+							APIKey:           "xxx",
 							Model:            "text-embedding-3-small",
 							DocumentTemplate: "A movie titled '{{doc.title}}'",
 						},
@@ -3452,6 +3452,47 @@ func TestIndex_UpdateSettingsEmbedders(t *testing.T) {
 				TaskUID: 1,
 			},
 		},
+		{
+			name: "TestIndexUpdateSettingsEmbeddersWithRestSource",
+			args: args{
+				newIndex: true,
+				UID:      "newIndexUID",
+				client:   meili,
+				request: Settings{
+					Embedders: map[string]Embedder{
+						"default": {
+							Source:           "rest",
+							URL:              "https://api.openai.com/v1/embeddings",
+							APIKey:           "<your-openai-api-key>",
+							Dimensions:       1536,
+							DocumentTemplate: "A movie titled '{{doc.title}}' whose description starts with {{doc.overview|truncatewords: 20}}",
+							Distribution: &Distribution{
+								Mean:  0.7,
+								Sigma: 0.3,
+							},
+							Request: map[string]interface{}{
+								"model": "text-embedding-3-small",
+								"input": []string{"{{text}}", "{{..}}"},
+							},
+							Response: map[string]interface{}{
+								"data": []interface{}{
+									map[string]interface{}{
+										"embedding": "{{embedding}}",
+									},
+									"{{..}}",
+								},
+							},
+							Headers: map[string]string{
+								"Custom-Header": "CustomValue",
+							},
+						},
+					},
+				},
+			},
+			wantTask: &TaskInfo{
+				TaskUID: 1,
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/types.go b/types.go
@@ -120,13 +120,25 @@ type Faceting struct {
 	SortFacetValuesBy map[string]SortFacetType `json:"sortFacetValuesBy"`
 }
 
+// Embedder represents a unified configuration for various embedder types.
 type Embedder struct {
-	Source           string `json:"source"`
-	URL              string `json:"url,omitempty"`
-	ApiKey           string `json:"apiKey,omitempty"`
-	Model            string `json:"model,omitempty"`
-	Dimensions       int    `json:"dimensions,omitempty"`
-	DocumentTemplate string `json:"documentTemplate,omitempty"`
+	Source           string                 `json:"source"`                     // The type of embedder: "openAi", "huggingFace", "userProvided", "rest", "ollama"
+	Model            string                 `json:"model,omitempty"`            // Optional for "openAi", "huggingFace", "ollama"
+	APIKey           string                 `json:"apiKey,omitempty"`           // Optional for "openAi", "rest", "ollama"
+	DocumentTemplate string                 `json:"documentTemplate,omitempty"` // Optional for most embedders
+	Dimensions       int                    `json:"dimensions,omitempty"`       // Optional for "openAi", "rest", "userProvided", "ollama"
+	Distribution     *Distribution          `json:"distribution,omitempty"`     // Optional for all embedders
+	URL              string                 `json:"url,omitempty"`              // Optional for "openAi", "rest", "ollama"
+	Revision         string                 `json:"revision,omitempty"`         // Optional for "huggingFace"
+	Request          map[string]interface{} `json:"request,omitempty"`          // Optional for "rest"
+	Response         map[string]interface{} `json:"response,omitempty"`         // Optional for "rest"
+	Headers          map[string]string      `json:"headers,omitempty"`          // Optional for "rest"
+}
+
+// Distribution represents a statistical distribution with mean and standard deviation (sigma).
+type Distribution struct {
+	Mean  float64 `json:"mean"`  // Mean of the distribution
+	Sigma float64 `json:"sigma"` // Sigma (standard deviation) of the distribution
 }
 
 // Version is the type that represents the versions in meilisearch