diff --git a/src/types/types.ts b/src/types/types.ts index b6feccdd6..9d9a5e312 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -98,7 +98,7 @@ export type SearchForFacetValuesResponse = { }; export type HybridSearch = { - embedder?: string; + embedder: string; semanticRatio?: number; }; @@ -389,6 +389,8 @@ export type OpenAiEmbedder = { dimensions?: number; distribution?: Distribution; url?: string; + documentTemplateMaxBytes?: number; + binaryQuantized?: boolean; }; export type HuggingFaceEmbedder = { @@ -397,12 +399,15 @@ export type HuggingFaceEmbedder = { revision?: string; documentTemplate?: string; distribution?: Distribution; + documentTemplateMaxBytes?: number; + binaryQuantized?: boolean; }; export type UserProvidedEmbedder = { source: "userProvided"; dimensions: number; distribution?: Distribution; + binaryQuantized?: boolean; }; export type RestEmbedder = { @@ -415,6 +420,8 @@ export type RestEmbedder = { request: Record; response: Record; headers?: Record; + documentTemplateMaxBytes?: number; + binaryQuantized?: boolean; }; export type OllamaEmbedder = { @@ -425,6 +432,8 @@ export type OllamaEmbedder = { documentTemplate?: string; distribution?: Distribution; dimensions?: number; + documentTemplateMaxBytes?: number; + binaryQuantized?: boolean; }; export type Embedder = diff --git a/tests/__snapshots__/settings.test.ts.snap b/tests/__snapshots__/settings.test.ts.snap index e05695111..069ab0055 100644 --- a/tests/__snapshots__/settings.test.ts.snap +++ b/tests/__snapshots__/settings.test.ts.snap @@ -249,8 +249,9 @@ exports[`Test on settings > Admin key: Update embedders settings 1`] = ` "distinctAttribute": null, "embedders": { "default": { - "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }} -{% endfor %}", + "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }} +{% endif %}{% endfor %}", + "documentTemplateMaxBytes": 400, "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "source": "huggingFace", }, @@ -804,8 +805,9 @@ exports[`Test on settings > Master key: Update embedders settings 1`] = ` "distinctAttribute": null, "embedders": { "default": { - "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }} -{% endfor %}", + "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }} +{% endif %}{% endfor %}", + "documentTemplateMaxBytes": 400, "model": "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", "source": "huggingFace", }, diff --git a/tests/embedders.test.ts b/tests/embedders.test.ts index 3bbffe265..02c061301 100644 --- a/tests/embedders.test.ts +++ b/tests/embedders.test.ts @@ -90,6 +90,7 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( mean: 0.7, sigma: 0.3, }, + binaryQuantized: false, }, }; const task: EnqueuedTask = await client @@ -101,6 +102,7 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( const response: Embedders = await client.index(index.uid).getEmbedders(); expect(response).toEqual(newEmbedder); + expect(response).not.toHaveProperty("documentTemplateMaxBytes"); }); test(`${permission} key: Update embedders with 'openAi' source`, async () => { @@ -118,6 +120,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( sigma: 0.3, }, url: "https://api.openai.com/v1/embeddings", + documentTemplateMaxBytes: 500, + binaryQuantized: false, }, }; const task: EnqueuedTask = await client @@ -147,6 +151,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( mean: 0.7, sigma: 0.3, }, + documentTemplateMaxBytes: 500, + binaryQuantized: false, }, }; const task: EnqueuedTask = await client @@ -188,6 +194,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( headers: { "Custom-Header": "CustomValue", }, + documentTemplateMaxBytes: 500, + binaryQuantized: false, }, }; const task: EnqueuedTask = await client @@ -219,6 +227,8 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( sigma: 0.3, }, dimensions: 512, + documentTemplateMaxBytes: 500, + binaryQuantized: false, }, }; const task: EnqueuedTask = await client @@ -266,6 +276,58 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( expect(response).toEqual(null); }); + test(`${permission} key: search (POST) with vectors`, async () => { + const client = await getClient(permission); + + const { taskUid } = await client.index(index.uid).updateEmbedders({ + default: { + source: "userProvided", + dimensions: 1, + }, + }); + await client.waitForTask(taskUid); + + const response = await client.index(index.uid).search("", { + vector: [1], + hybrid: { + embedder: "default", + semanticRatio: 1.0, + }, + }); + + expect(response).toHaveProperty("hits"); + expect(response).toHaveProperty("semanticHitCount"); + // Those fields are no longer returned by the search response + // We want to ensure that they don't appear in it anymore + expect(response).not.toHaveProperty("vector"); + expect(response).not.toHaveProperty("_semanticScore"); + }); + + test(`${permission} key: search (GET) with vectors`, async () => { + const client = await getClient(permission); + + const { taskUid } = await client.index(index.uid).updateEmbedders({ + default: { + source: "userProvided", + dimensions: 1, + }, + }); + await client.waitForTask(taskUid); + + const response = await client.index(index.uid).searchGet("", { + vector: [1], + hybridEmbedder: "default", + hybridSemanticRatio: 1.0, + }); + + expect(response).toHaveProperty("hits"); + expect(response).toHaveProperty("semanticHitCount"); + // Those fields are no longer returned by the search response + // We want to ensure that they don't appear in it anymore + expect(response).not.toHaveProperty("vector"); + expect(response).not.toHaveProperty("_semanticScore"); + }); + test(`${permission} key: search for similar documents`, async () => { const client = await getClient(permission); @@ -288,6 +350,7 @@ describe.each([{ permission: "Master" }, { permission: "Admin" }])( await client.waitForTask(documentAdditionTask); const response = await client.index(index.uid).searchSimilarDocuments({ + embedder: "manual", id: "143", }); diff --git a/tests/get_search.test.ts b/tests/get_search.test.ts index f5834c355..241c72539 100644 --- a/tests/get_search.test.ts +++ b/tests/get_search.test.ts @@ -457,41 +457,6 @@ describe.each([ "The filter query parameter should be in string format when using searchGet", ); }); - test(`${permission} key: search with vectors`, async () => { - const client = await getClient(permission); - const adminClient = await getClient("Admin"); - const adminKey = await getKey("Admin"); - - await fetch(`${HOST}/experimental-features`, { - body: JSON.stringify({ vectorStore: true }), - headers: { - Authorization: `Bearer ${adminKey}`, - "Content-Type": "application/json", - }, - method: "PATCH", - }); - - const { taskUid } = await adminClient - .index(emptyIndex.uid) - .updateEmbedders({ - default: { - source: "userProvided", - dimensions: 1, - }, - }); - await adminClient.waitForTask(taskUid); - - const response = await client - .index(emptyIndex.uid) - .searchGet("", { vector: [1], hybridSemanticRatio: 1.0 }); - - expect(response).toHaveProperty("hits"); - expect(response).toHaveProperty("semanticHitCount"); - // Those fields are no longer returned by the search response - // We want to ensure that they don't appear in it anymore - expect(response).not.toHaveProperty("vector"); - expect(response).not.toHaveProperty("_semanticScore"); - }); test(`${permission} key: search without vectors`, async () => { const client = await getClient(permission); diff --git a/tests/search.test.ts b/tests/search.test.ts index f4cf6658b..df007036c 100644 --- a/tests/search.test.ts +++ b/tests/search.test.ts @@ -936,45 +936,6 @@ describe.each([ expect(response.hits.length).toEqual(0); }); - test(`${permission} key: search with vectors`, async () => { - const client = await getClient(permission); - const adminClient = await getClient("Admin"); - const adminKey = await getKey("Admin"); - - await fetch(`${HOST}/experimental-features`, { - body: JSON.stringify({ vectorStore: true }), - headers: { - Authorization: `Bearer ${adminKey}`, - "Content-Type": "application/json", - }, - method: "PATCH", - }); - - const { taskUid } = await adminClient - .index(emptyIndex.uid) - .updateEmbedders({ - default: { - source: "userProvided", - dimensions: 1, - }, - }); - await adminClient.waitForTask(taskUid); - - const response = await client.index(emptyIndex.uid).search("", { - vector: [1], - hybrid: { - semanticRatio: 1.0, - }, - }); - - expect(response).toHaveProperty("hits"); - expect(response).toHaveProperty("semanticHitCount"); - // Those fields are no longer returned by the search response - // We want to ensure that they don't appear in it anymore - expect(response).not.toHaveProperty("vector"); - expect(response).not.toHaveProperty("_semanticScore"); - }); - test(`${permission} key: search without vectors`, async () => { const client = await getClient(permission); const response = await client.index(index.uid).search("prince", {});