From c6ca5d222410cec7bf2ee61cc6dd6978f7f8ec2b Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 8 Jan 2025 17:32:41 +1000 Subject: [PATCH 01/11] Fix index name in bitmap filtering docs Signed-off-by: Russ Cam Signed-off-by: Russ Cam --- _query-dsl/term/terms.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_query-dsl/term/terms.md b/_query-dsl/term/terms.md index 2de0b71bd6..f80ab8b4a1 100644 --- a/_query-dsl/term/terms.md +++ b/_query-dsl/term/terms.md @@ -277,7 +277,7 @@ PUT /products Next, index three documents that correspond to products: ```json -PUT students/_doc/1 +PUT products/_doc/1 { "name": "Product 1", "product_id" : "111" @@ -286,7 +286,7 @@ PUT students/_doc/1 {% include copy-curl.html %} ```json -PUT students/_doc/2 +PUT products/_doc/2 { "name": "Product 2", "product_id" : "222" @@ -295,7 +295,7 @@ PUT students/_doc/2 {% include copy-curl.html %} ```json -PUT students/_doc/3 +PUT products/_doc/3 { "name": "Product 3", "product_id" : "333" From b11e2fe9f2883ae4f98a693b4954dbdc48c47efd Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 9 Jan 2025 11:43:45 +1000 Subject: [PATCH 02/11] Apply changes from code review Add forward slashes to URLs Signed-off-by: Russ Cam Signed-off-by: Russ Cam --- _query-dsl/term/terms.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/_query-dsl/term/terms.md b/_query-dsl/term/terms.md index f80ab8b4a1..089fb16601 100644 --- a/_query-dsl/term/terms.md +++ b/_query-dsl/term/terms.md @@ -22,6 +22,7 @@ GET shakespeare/_search } } ``` + {% include copy-curl.html %} A document is returned if it matches any of the terms in the array. @@ -63,6 +64,7 @@ PUT students } } ``` + {% include copy-curl.html %} Next, index three documents that correspond to students: @@ -74,6 +76,7 @@ PUT students/_doc/1 "student_id" : "111" } ``` + {% include copy-curl.html %} ```json @@ -83,6 +86,7 @@ PUT students/_doc/2 "student_id" : "222" } ``` + {% include copy-curl.html %} ```json @@ -92,6 +96,7 @@ PUT students/_doc/3 "student_id" : "333" } ``` + {% include copy-curl.html %} Create a separate index that contains class information, including the class name and an array of student IDs corresponding to the students enrolled in the class: @@ -103,6 +108,7 @@ PUT classes/_doc/101 "enrolled" : ["111" , "222"] } ``` + {% include copy-curl.html %} To search for students enrolled in the `CS101` class, specify the document ID of the document that corresponds to the class, the index of that document, and the path of the field in which the terms are located: @@ -121,6 +127,7 @@ GET students/_search } } ``` + {% include copy-curl.html %} The response contains the documents in the `students` index for every student whose ID matches one of the values in the `enrolled` array: @@ -178,6 +185,7 @@ PUT classes/_doc/102 } } ``` + {% include copy-curl.html %} To search for students enrolled in `CS102`, use the dot path notation to specify the full path to the field in the `path` parameter: @@ -196,6 +204,7 @@ GET students/_search } } ``` + {% include copy-curl.html %} The response contains the matching documents: @@ -253,12 +262,13 @@ Parameter | Data type | Description `boost` | Floating-point | A floating-point value that specifies the weight of this field toward the relevance score. Values above 1.0 increase the field’s relevance. Values between 0.0 and 1.0 decrease the field’s relevance. Default is 1.0. ## Bitmap filtering + **Introduced 2.17** {: .label .label-purple } -The `terms` query can filter for multiple terms simultaneously. However, when the number of terms in the input filter increases to a large value (around 10,000), the resulting network and memory overhead can become significant, making the query inefficient. In such cases, consider encoding your large terms filter using a [roaring bitmap](https://github.com/RoaringBitmap/RoaringBitmap) for more efficient filtering. +The `terms` query can filter for multiple terms simultaneously. However, when the number of terms in the input filter increases to a large value (around 10,000), the resulting network and memory overhead can become significant, making the query inefficient. In such cases, consider encoding your large terms filter using a [roaring bitmap](https://github.com/RoaringBitmap/RoaringBitmap) for more efficient filtering. -The following example assumes that you have two indexes: a `products` index, which contains all the products sold by a company, and a `customers` index, which stores filters representing customers who own specific products. +The following example assumes that you have two indexes: a `products` index, which contains all the products sold by a company, and a `customers` index, which stores filters representing customers who own specific products. First, create a `products` index and map `product_id` as a `keyword`: @@ -272,35 +282,39 @@ PUT /products } } ``` + {% include copy-curl.html %} Next, index three documents that correspond to products: ```json -PUT products/_doc/1 +PUT /products/_doc/1 { "name": "Product 1", "product_id" : "111" } ``` + {% include copy-curl.html %} ```json -PUT products/_doc/2 +PUT /products/_doc/2 { "name": "Product 2", "product_id" : "222" } ``` + {% include copy-curl.html %} ```json -PUT products/_doc/3 +PUT /products/_doc/3 { "name": "Product 3", "product_id" : "333" } ``` + {% include copy-curl.html %} To store customer bitmap filters, you'll create a `customer_filter` [binary field](https://opensearch.org/docs/latest/field-types/supported-field-types/binary/) in the `customers` index. Specify `store` as `true` to store the field: @@ -318,6 +332,7 @@ PUT /customers } } ``` + {% include copy-curl.html %} For each customer, you need to generate a bitmap that represents the product IDs of the products the customer owns. This bitmap effectively encodes the filter criteria for that customer. In this example, you'll create a `terms` filter for a customer whose ID is `customer123` and who owns products `111`, `222`, and `333`. @@ -338,6 +353,7 @@ encoded_bm_str = encoded.decode('utf-8') # Print the encoded bitmap print(f"Encoded Bitmap: {encoded_bm_str}") ``` + {% include copy.html %} Next, index the customer filter into the `customers` index. The document ID for the filter is the same as the ID for the corresponding customer (in this example, `customer123`). The `customer_filter` field contains the bitmap you generated for this customer: @@ -348,6 +364,7 @@ POST customers/_doc/customer123 "customer_filter": "OjAAAAEAAAAAAAIAEAAAAG8A3gBNAQ==" } ``` + {% include copy-curl.html %} Now you can run a `terms` query on the `products` index to look up a specific customer in the `customers` index. Because you're looking up a stored field instead of `_source`, set `store` to `true`. In the `value_type` field, specify the data type of the `terms` input as `bitmap`: @@ -368,6 +385,7 @@ POST /products/_search } } ``` + {% include copy-curl.html %} You can also directly pass the bitmap to the `terms` query. In this example, the `product_id` field contains the customer filter bitmap for the customer whose ID is `customer123`: @@ -383,4 +401,5 @@ POST /products/_search } } ``` + {% include copy-curl.html %} From 11bbbf23279fbfec6a2613efe0e4367c395b1318 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:40:30 -0500 Subject: [PATCH 03/11] Apply suggestions from code review Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Russ Cam --- _query-dsl/term/terms.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/_query-dsl/term/terms.md b/_query-dsl/term/terms.md index 089fb16601..5141a0d91c 100644 --- a/_query-dsl/term/terms.md +++ b/_query-dsl/term/terms.md @@ -22,7 +22,6 @@ GET shakespeare/_search } } ``` - {% include copy-curl.html %} A document is returned if it matches any of the terms in the array. @@ -64,7 +63,6 @@ PUT students } } ``` - {% include copy-curl.html %} Next, index three documents that correspond to students: @@ -76,7 +74,6 @@ PUT students/_doc/1 "student_id" : "111" } ``` - {% include copy-curl.html %} ```json @@ -86,7 +83,6 @@ PUT students/_doc/2 "student_id" : "222" } ``` - {% include copy-curl.html %} ```json @@ -96,7 +92,6 @@ PUT students/_doc/3 "student_id" : "333" } ``` - {% include copy-curl.html %} Create a separate index that contains class information, including the class name and an array of student IDs corresponding to the students enrolled in the class: @@ -108,7 +103,6 @@ PUT classes/_doc/101 "enrolled" : ["111" , "222"] } ``` - {% include copy-curl.html %} To search for students enrolled in the `CS101` class, specify the document ID of the document that corresponds to the class, the index of that document, and the path of the field in which the terms are located: @@ -127,7 +121,6 @@ GET students/_search } } ``` - {% include copy-curl.html %} The response contains the documents in the `students` index for every student whose ID matches one of the values in the `enrolled` array: @@ -185,7 +178,6 @@ PUT classes/_doc/102 } } ``` - {% include copy-curl.html %} To search for students enrolled in `CS102`, use the dot path notation to specify the full path to the field in the `path` parameter: @@ -204,7 +196,6 @@ GET students/_search } } ``` - {% include copy-curl.html %} The response contains the matching documents: @@ -262,7 +253,6 @@ Parameter | Data type | Description `boost` | Floating-point | A floating-point value that specifies the weight of this field toward the relevance score. Values above 1.0 increase the field’s relevance. Values between 0.0 and 1.0 decrease the field’s relevance. Default is 1.0. ## Bitmap filtering - **Introduced 2.17** {: .label .label-purple } @@ -282,7 +272,6 @@ PUT /products } } ``` - {% include copy-curl.html %} Next, index three documents that correspond to products: @@ -294,7 +283,6 @@ PUT /products/_doc/1 "product_id" : "111" } ``` - {% include copy-curl.html %} ```json @@ -304,7 +292,6 @@ PUT /products/_doc/2 "product_id" : "222" } ``` - {% include copy-curl.html %} ```json @@ -314,7 +301,6 @@ PUT /products/_doc/3 "product_id" : "333" } ``` - {% include copy-curl.html %} To store customer bitmap filters, you'll create a `customer_filter` [binary field](https://opensearch.org/docs/latest/field-types/supported-field-types/binary/) in the `customers` index. Specify `store` as `true` to store the field: @@ -332,7 +318,6 @@ PUT /customers } } ``` - {% include copy-curl.html %} For each customer, you need to generate a bitmap that represents the product IDs of the products the customer owns. This bitmap effectively encodes the filter criteria for that customer. In this example, you'll create a `terms` filter for a customer whose ID is `customer123` and who owns products `111`, `222`, and `333`. @@ -353,7 +338,6 @@ encoded_bm_str = encoded.decode('utf-8') # Print the encoded bitmap print(f"Encoded Bitmap: {encoded_bm_str}") ``` - {% include copy.html %} Next, index the customer filter into the `customers` index. The document ID for the filter is the same as the ID for the corresponding customer (in this example, `customer123`). The `customer_filter` field contains the bitmap you generated for this customer: @@ -364,7 +348,6 @@ POST customers/_doc/customer123 "customer_filter": "OjAAAAEAAAAAAAIAEAAAAG8A3gBNAQ==" } ``` - {% include copy-curl.html %} Now you can run a `terms` query on the `products` index to look up a specific customer in the `customers` index. Because you're looking up a stored field instead of `_source`, set `store` to `true`. In the `value_type` field, specify the data type of the `terms` input as `bitmap`: @@ -385,7 +368,6 @@ POST /products/_search } } ``` - {% include copy-curl.html %} You can also directly pass the bitmap to the `terms` query. In this example, the `product_id` field contains the customer filter bitmap for the customer whose ID is `customer123`: @@ -401,5 +383,4 @@ POST /products/_search } } ``` - {% include copy-curl.html %} From d8cbf7b4dea5aa4f85801b88105ec4bb5adb126a Mon Sep 17 00:00:00 2001 From: Saliha <49085460+Saliha067@users.noreply.github.com> Date: Wed, 8 Jan 2025 23:45:55 +0900 Subject: [PATCH 04/11] Update permissions.md (#9031) Signed-off-by: Saliha <49085460+Saliha067@users.noreply.github.com> Signed-off-by: Russ Cam --- _security/access-control/permissions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_security/access-control/permissions.md b/_security/access-control/permissions.md index 5a75a0a5a7..bacc49fe20 100644 --- a/_security/access-control/permissions.md +++ b/_security/access-control/permissions.md @@ -528,7 +528,7 @@ These permissions apply to an index or index pattern. You might want a user to h | `indices:monitor/data_stream/stats` | Permission to stream stats. | | `indices:monitor/recovery` | Permission to access recovery stats. | | `indices:monitor/segments` | Permission to access segment stats. | -| `indices:monitor/settings/get` | Permission to get mointor settings. | +| `indices:monitor/settings/get` | Permission to get monitor settings. | | `indices:monitor/shard_stores` | Permission to access shard store stats. | | `indices:monitor/stats` | Permission to access monitoring stats. | | `indices:monitor/upgrade` | Permission to access upgrade stats. | From 73d239e9d39df97c83d1aebcfe5e0e9bcdefe4f5 Mon Sep 17 00:00:00 2001 From: Shawshark Date: Wed, 8 Jan 2025 07:33:48 -0800 Subject: [PATCH 05/11] Update knn-vector-quantization.md (#8985) Signed-off-by: Shawshark Signed-off-by: Russ Cam --- _search-plugins/knn/knn-vector-quantization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_search-plugins/knn/knn-vector-quantization.md b/_search-plugins/knn/knn-vector-quantization.md index a911dc91c9..86984972ee 100644 --- a/_search-plugins/knn/knn-vector-quantization.md +++ b/_search-plugins/knn/knn-vector-quantization.md @@ -359,7 +359,7 @@ PUT my-vector-index "mode": "on_disk", "compression_level": "16x", "method": { - "params": { + "parameters": { "ef_construction": 16 } } From 5b259d2feece335ef3aceee9c5578b9c4ee56085 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:43:36 -0500 Subject: [PATCH 06/11] Update knn-vector-quantization.md (#9035) Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Russ Cam --- _search-plugins/knn/knn-vector-quantization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_search-plugins/knn/knn-vector-quantization.md b/_search-plugins/knn/knn-vector-quantization.md index 86984972ee..2e516b0b8d 100644 --- a/_search-plugins/knn/knn-vector-quantization.md +++ b/_search-plugins/knn/knn-vector-quantization.md @@ -384,7 +384,7 @@ PUT my-vector-index "name": "hnsw", "engine": "faiss", "space_type": "l2", - "params": { + "parameters": { "m": 16, "ef_construction": 512, "encoder": { From dd2c9eca5936ecc9d9bad3eeb8a1b0184a202df2 Mon Sep 17 00:00:00 2001 From: Rithin Pullela Date: Wed, 8 Jan 2025 08:44:41 -0800 Subject: [PATCH 07/11] docs: add validation requirement for message fields (#9000) * docs: add validation requirement for message fields Add note clarifying that at least one field must be non-null and non-empty for successful message creation/update operations Signed-off-by: rithin-pullela-aws * Update _ml-commons-plugin/api/memory-apis/create-message.md Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Rithin Pullela --------- Signed-off-by: rithin-pullela-aws Signed-off-by: Rithin Pullela Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Russ Cam --- .../api/memory-apis/create-message.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/_ml-commons-plugin/api/memory-apis/create-message.md b/_ml-commons-plugin/api/memory-apis/create-message.md index 561e321624..d1272e3e54 100644 --- a/_ml-commons-plugin/api/memory-apis/create-message.md +++ b/_ml-commons-plugin/api/memory-apis/create-message.md @@ -44,11 +44,14 @@ The following table lists the available request fields. Field | Data type | Required/Optional | Updatable | Description :--- | :--- | :--- | :--- | :--- -| `input` | String | Optional | No | The question (human input) in the message. | -| `prompt_template` | String | Optional | No | The prompt template that was used for the message. The template may contain instructions or examples that were sent to the large language model. | -| `response` | String | Optional | No | The answer (generative AI output) to the question. | -| `origin` | String | Optional | No | The name of the AI or other system that generated the response. | -| `additional_info` | Object | Optional | Yes | Any other information that was sent to the `origin`. | +`input` | String | Optional | No | The question (human input) in the message. | +`prompt_template` | String | Optional | No | The prompt template that was used for the message. The template may contain instructions or examples that were sent to the large language model. | +`response` | String | Optional | No | The answer (generative AI output) to the question. | +`origin` | String | Optional | No | The name of the AI or other system that generated the response. | +`additional_info` | Object | Optional | Yes | Any other information that was sent to the `origin`. | + +To create or update a message successfully, you must provide at least one of the preceding fields. The provided field(s) cannot be null or empty. +{: .note} #### Example request: Create a message From ff16e9fb3927f91f196db5c4db940af7b30a8bcb Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:56:03 -0600 Subject: [PATCH 08/11] Add editorial review from 2.18 (#9038) Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Signed-off-by: Russ Cam --- .../is-migration-assistant-right-for-you.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_migration-assistant/is-migration-assistant-right-for-you.md b/_migration-assistant/is-migration-assistant-right-for-you.md index 77f3c6a806..2d8895ec30 100644 --- a/_migration-assistant/is-migration-assistant-right-for-you.md +++ b/_migration-assistant/is-migration-assistant-right-for-you.md @@ -36,9 +36,9 @@ There are also tools available for migrating cluster configuration, templates, a The tooling is designed to work with other cloud provider platforms, but it is not officially tested with these other platforms. If you would like to add support, please contact one of the maintainers on [GitHub](https://github.com/opensearch-project/opensearch-migrations/blob/main/MAINTAINERS.md). -### Supported AWS regions +### Supported AWS Regions -Migration Assistant supports the following AWS regions: +Migration Assistant supports the following AWS Regions: - US East (N. Virginia) - US East (Ohio) @@ -53,7 +53,7 @@ Migration Assistant supports the following AWS regions: - AWS GovCloud (US-East)[^1] - AWS GovCloud (US-West)[^1] -[^1]: GovCloud does not support `reindex-from-snapshot` (RFS) shard sizes above 80GiB. Ensure your shard sizes are within this limit when planning migrations with RFS in the listed GovCloud regions. +[^1]: GovCloud does not support `reindex-from-snapshot` (RFS) shard sizes above 80 GiB. Ensure your shard sizes are within this limit when planning migrations with RFS in the listed GovCloud regions. ### Future migration paths @@ -74,4 +74,4 @@ Before starting a migration, consider the scope of the components involved. The | **Index State Management (ISM) policies** | Expected in 2025 | Manually migrate using an API. | | **Elasticsearch Kibana dashboards** | Expected in 2025 | This tool is only needed when used to migrate Elasticsearch Kibana Dashboards to OpenSearch Dashboards. To start, export JSON files from Kibana and import them into OpenSearch Dashboards; before importing, use the [`dashboardsSanitizer`](https://github.com/opensearch-project/opensearch-migrations/tree/main/dashboardsSanitizer) tool on X-Pack visualizations like Canvas and Lens in Kibana Dashboards, as they may require recreation for compatibility with OpenSearch. | | **Security constructs** | No | Configure roles and permissions based on cloud provider recommendations. For example, if using AWS, leverage AWS Identity and Access Management (IAM) for enhanced security management. | -| **Plugins** | No | Check plugin compatibility; some Elasticsearch plugins may not have direct equivalents in OpenSearch. | +| **Plugins** | No | Check plugin compatibility; some Elasticsearch plugins may not have direct OpenSearch equivalents. | From 219ec783c43cb9c0531a23f7456ab4dab01e393e Mon Sep 17 00:00:00 2001 From: Rithin Pullela Date: Thu, 9 Jan 2025 05:42:23 -0800 Subject: [PATCH 09/11] Update Linear and Logistic Regression Parameters & Improve Documentation (#8982) * Update Linear and Logistic Regression Parameters & Improve Documentation - Add comprehensive documentation for supported: - Optimizers (SIMPLE_SGD, LINEAR_DECAY_SGD, etc.) - Objective types (ABSOLUTE_LOSS, HUBER, SQUARED_LOSS) - Momentum types (STANDARD, NESTEROV) - Fix parameter name typos Signed-off-by: rithin-pullela-aws * Update _ml-commons-plugin/algorithms.md Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Rithin Pullela * Added Tribuo documentation links to optimizers, Objective types, and momentum type. Signed-off-by: rithin-pullela-aws * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --------- Signed-off-by: rithin-pullela-aws Signed-off-by: Rithin Pullela Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Co-authored-by: Nathan Bower Signed-off-by: Russ Cam --- _ml-commons-plugin/algorithms.md | 43 +++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/_ml-commons-plugin/algorithms.md b/_ml-commons-plugin/algorithms.md index d7809d51b2..a5a173b358 100644 --- a/_ml-commons-plugin/algorithms.md +++ b/_ml-commons-plugin/algorithms.md @@ -59,20 +59,31 @@ The training process supports multithreading, but the number of threads must be ## Linear regression -Linear regression maps the linear relationship between inputs and outputs. In ML Commons, the linear regression algorithm is adopted from the public machine learning library [Tribuo](https://tribuo.org/), which offers multidimensional linear regression models. The model supports the linear optimizer in training, including popular approaches like Linear Decay, SQRT_DECAY, [ADA](https://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf), [ADAM](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/Adam.html), and [RMS_DROP](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/RMSProp.html). +Linear regression maps the linear relationship between inputs and outputs. In ML Commons, the linear regression algorithm is adopted from the public machine learning library [Tribuo](https://tribuo.org/), which offers multidimensional linear regression models. The model supports the linear optimizer in training, including popular approaches like Linear Decay, SQRT_DECAY, [ADA](https://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf), [ADAM](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/Adam.html), and [RMS_PROP](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/RMSProp.html). + +**Optimizers supported:** [SIMPLE_SGD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.html#:~:text=learning%20rate%20SGD.-,getSimpleSGD,-public%20static%C2%A0), [LINEAR_DECAY_SGD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.html#:~:text=linear%20decay%20SGD.-,getLinearDecaySGD,-public%20static%C2%A0), [SQRT_DECAY_SGD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.html#:~:text=sqrt%20decay%20SGD.-,getSqrtDecaySGD,-public%20static%C2%A0), [ADA_GRAD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/AdaGrad.html), [ADA_DELTA](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/AdaDelta.html), [ADAM](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/Adam.html), and [RMS_PROP](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/RMSProp.html). +**Objectives supported:** [ABSOLUTE_LOSS](https://tribuo.org/learn/4.2/javadoc/org/tribuo/regression/sgd/objectives/AbsoluteLoss.html), [HUBER](https://tribuo.org/learn/4.2/javadoc/org/tribuo/regression/sgd/objectives/Huber.html), and [SQUARED_LOSS](https://tribuo.org/learn/4.2/javadoc/org/tribuo/regression/sgd/objectives/SquaredLoss.html). +**momentum_type supported:** [STANDARD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.Momentum.html#STANDARD:~:text=No%20momentum.-,STANDARD,-public%20static%20final) and [NESTEROV](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.Momentum.html#STANDARD:~:text=Standard%20momentum.-,NESTEROV,-public%20static%20final). ### Parameters Parameter | Type | Description | Default value :--- |:--- | :--- | :--- -`learningRate` | Double | The initial step size used in an iterative optimization algorithm. | `0.01` -`momentumFactor` | Double | The extra weight factors that accelerate the rate at which the weight is adjusted. This helps move the minimization routine out of local minima. | `0` +`target` | String | The name of the target variable to predict. Identifies which feature the model will learn to predict during training. | `NA` +`learning_rate` | Double | The initial step size used in an iterative optimization algorithm. | `0.01` +`momentum_factor` | Double | The extra weight factors that accelerate the rate at which the weight is adjusted. This helps move the minimization routine out of local minima. | `0` `epsilon` | Double | The value for stabilizing gradient inversion. | `1.00E-06` `beta1` | Double | The exponential decay rates for the moment estimates. | `0.9` `beta2` | Double | The exponential decay rates for the moment estimates. | `0.99` -`decayRate` | Double | The Root Mean Squared Propagation (RMSProp). | `0.9` -`momentumType` | String | The defined Stochastic Gradient Descent (SGD) momentum type that helps accelerate gradient vectors in the right directions, leading to a fast convergence.| `STANDARD` -`optimizerType` | String | The optimizer used in the model. | `SIMPLE_SGD` +`decay_rate` | Double | The Root Mean Squared Propagation (RMSProp). | `0.9` +`momentum_type` | String | The defined Stochastic Gradient Descent (SGD) momentum type that helps accelerate gradient vectors in the right directions, leading to a fast convergence.| `STANDARD` +`optimiser` | String | The optimizer used in the model. | `SIMPLE_SGD` +`objective` | String | The objective function used. | `SQUARED_LOSS` +`epochs` | Integer | The number of iterations. | `5`| +`batch_size` | Integer | The minimum batch size. | `1` +`logging_interval` | Integer | The frequency of logging during training iterations. Set to `-1` to disable logging. | `-1` +`seed` | Long | A random seed used for reproducible results. Controls the initialization of random number generators. | `12345` + ### Supported APIs @@ -412,23 +423,27 @@ The Localization algorithm can only be executed directly. Therefore, it cannot b A classification algorithm, logistic regression models the probability of a discrete outcome given an input variable. In ML Commons, these classifications include both binary and multi-class. The most common is the binary classification, which takes two values, such as "true/false" or "yes/no", and predicts the outcome based on the values specified. Alternatively, a multi-class output can categorize different inputs based on type. This makes logistic regression most useful for situations where you are trying to determine how your inputs fit best into a specified category. +**Optimizers supported:** [SIMPLE_SGD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.html#:~:text=learning%20rate%20SGD.-,getSimpleSGD,-public%20static%C2%A0), [LINEAR_DECAY_SGD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.html#:~:text=linear%20decay%20SGD.-,getLinearDecaySGD,-public%20static%C2%A0), [SQRT_DECAY_SGD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.html#:~:text=sqrt%20decay%20SGD.-,getSqrtDecaySGD,-public%20static%C2%A0), [ADA_GRAD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/AdaGrad.html), [ADA_DELTA](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/AdaDelta.html), [ADAM](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/Adam.html), and [RMS_PROP](https://tribuo.org/learn/4.1/javadoc/org/tribuo/math/optimisers/RMSProp.html). +**Objectives supported:** [HINGE](https://tribuo.org/learn/4.2/javadoc/org/tribuo/classification/sgd/objectives/Hinge.html) and [LOGMULTICLASS](https://tribuo.org/learn/4.2/javadoc/org/tribuo/classification/sgd/objectives/LogMulticlass.html). +**Momentum type supported:** [STANDARD](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.Momentum.html#STANDARD:~:text=No%20momentum.-,STANDARD,-public%20static%20final) and [NESTEROV](https://tribuo.org/learn/4.2/javadoc/org/tribuo/math/optimisers/SGD.Momentum.html#STANDARD:~:text=Standard%20momentum.-,NESTEROV,-public%20static%20final). + ### Parameters | Parameter | Type | Description | Default value | |---|---|---|---| -| `learningRate` | Double | The initial step size used in an iterative optimization algorithm. | `1` | -| `momentumFactor` | Double | The extra weight factors that accelerate the rate at which the weight is adjusted. This helps move the minimization routine out of local minima. | `0` | +| `learning_rate` | Double | The initial step size used in an iterative optimization algorithm. | `1` | +| `momentum_factor` | Double | The extra weight factors that accelerate the rate at which the weight is adjusted. This helps move the minimization routine out of local minima. | `0` | | `epsilon` | Double | The value for stabilizing gradient inversion. | `0.1` | | `beta1` | Double | The exponential decay rates for the moment estimates. | `0.9` | | `beta2` | Double | The exponential decay rates for the moment estimates. | `0.99` | -| `decayRate` | Double | The Root Mean Squared Propagation (RMSProp). | `0.9` | -| `momentumType` | String | The Stochastic Gradient Descent (SGD) momentum that helps accelerate gradient vectors in the right direction, leading to faster convergence between vectors. | `STANDARD` | -| `optimizerType` | String | The optimizer used in the model. | `AdaGrad` | +| `decay_rate` | Double | The Root Mean Squared Propagation (RMSProp). | `0.9` | +| `momentum_type` | String | The Stochastic Gradient Descent (SGD) momentum that helps accelerate gradient vectors in the right directions, leading to a fast convergence. | `STANDARD` | +| `optimiser` | String | The optimizer used in the model. | `ADA_GRAD` | | `target` | String | The target field. | null | -| `objectiveType` | String | The objective function type. | `LogMulticlass` | +| `objective` | String | The objective function type. | `LOGMULTICLASS` | | `epochs` | Integer | The number of iterations. | `5` | -| `batchSize` | Integer | The size of min batches. | `1` | -| `loggingInterval` | Integer | The interval of logs lost after many iterations. The interval is `1` if the algorithm contains no logs. | `1000` | +| `batch_size` | Integer | The minimum batch size. | `1` | +| `logging_interval` | Integer | The interval of logs lost after many iterations. The interval is `1` if the algorithm contains no logs. | `1000` | ### Supported APIs From becc3619fa9641985b6daad6af4bfd194f0784ce Mon Sep 17 00:00:00 2001 From: Takayuki Enomoto <4161768+tkykenmt@users.noreply.github.com> Date: Thu, 9 Jan 2025 22:43:16 +0900 Subject: [PATCH 10/11] Add support for Bedrock Rerank API #9027 (#9029) * Add support for Bedrock Rerank API #9027 Signed-off-by: tkykenmt * Doc review Signed-off-by: Fanit Kolchina * Change title Signed-off-by: Fanit Kolchina * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> * fix fieldname on Amazon Bedrock Rerank API output #9029 Signed-off-by: tkykenmt --------- Signed-off-by: tkykenmt Signed-off-by: Fanit Kolchina Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Co-authored-by: Fanit Kolchina Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Co-authored-by: Nathan Bower Signed-off-by: Russ Cam --- _ml-commons-plugin/tutorials/index.md | 1 + .../tutorials/reranking-bedrock.md | 731 ++++++++++++++++++ 2 files changed, 732 insertions(+) create mode 100644 _ml-commons-plugin/tutorials/reranking-bedrock.md diff --git a/_ml-commons-plugin/tutorials/index.md b/_ml-commons-plugin/tutorials/index.md index 070da3cae1..c039ecfd34 100644 --- a/_ml-commons-plugin/tutorials/index.md +++ b/_ml-commons-plugin/tutorials/index.md @@ -19,6 +19,7 @@ Using the OpenSearch machine learning (ML) framework, you can build various appl - **Reranking search results**: - [Reranking search results using the Cohere Rerank model]({{site.url}}{{site.baseurl}}/ml-commons-plugin/tutorials/reranking-cohere/) + - [Reranking search results using models hosted on Amazon Bedrock]({{site.url}}{{site.baseurl}}/ml-commons-plugin/tutorials/reranking-bedrock/) - [Reranking search results using the MS MARCO cross-encoder model]({{site.url}}{{site.baseurl}}/ml-commons-plugin/tutorials/reranking-cross-encoder/) - **Agents and tools**: diff --git a/_ml-commons-plugin/tutorials/reranking-bedrock.md b/_ml-commons-plugin/tutorials/reranking-bedrock.md new file mode 100644 index 0000000000..b46104f241 --- /dev/null +++ b/_ml-commons-plugin/tutorials/reranking-bedrock.md @@ -0,0 +1,731 @@ +--- +layout: default +title: Reranking search results with Amazon Bedrock models +parent: Tutorials +nav_order: 32 +--- + +# Reranking search results using models hosted on Amazon Bedrock + +A [reranking pipeline]({{site.url}}{{site.baseurl}}/search-plugins/search-relevance/reranking-search-results/) can rerank search results, providing a relevance score for each document in the search results with respect to the search query. The relevance score is calculated by a cross-encoder model. + +This tutorial illustrates using the [Amazon Bedrock Rerank API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_Rerank.html) to rerank search results using a model hosted on Amazon Bedrock. + +Replace the placeholders beginning with the prefix `your_` with your own values. +{: .note} + +## Prerequisite: Test the model on Amazon Bedrock + +Before using your model, test it on Amazon Bedrock. For supported reranker models, see [Supported Regions and models for reranking in Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/rerank-supported.html). For model IDs, see [Supported foundation models in Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/models-supported.html). To perform a reranking test, use the following code: + +```python +import json +import boto3 +bedrock_region = "your_bedrock_model_region_like_us-west-2" +bedrock_agent_runtime_client = boto3.client("bedrock-agent-runtime", region_name=bedrock_region) + +model_id = "amazon.rerank-v1:0" + +response = bedrock_agent_runtime_client.rerank( + queries=[ + { + "textQuery": { + "text": "What is the capital city of America?", + }, + "type": "TEXT" + } + ], + rerankingConfiguration={ + "bedrockRerankingConfiguration": { + "modelConfiguration": { + "modelArn": f"arn:aws:bedrock:{bedrock_region}::foundation-model/{model_id}" + }, + }, + "type": "BEDROCK_RERANKING_MODEL" + }, + sources=[ + { + "inlineDocumentSource": { + "textDocument": { + "text": "Carson City is the capital city of the American state of Nevada.", + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + { + "inlineDocumentSource": { + "textDocument": { + "text": "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + { + "inlineDocumentSource": { + "textDocument": { + "text": "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + { + "inlineDocumentSource": { + "textDocument": { + "text": "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + ] +) + +results = response["results"] +print(json.dumps(results, indent=2)) +``` +{% include copy.html %} + +The reranked results are ordered by the highest score: + +```json +[ + { + "index": 2, + "relevanceScore": 0.7711548805236816 + }, + { + "index": 0, + "relevanceScore": 0.0025114635936915874 + }, + { + "index": 1, + "relevanceScore": 2.4876489987946115e-05 + }, + { + "index": 3, + "relevanceScore": 6.339210358419223e-06 + } +] +``` + +To sort the results by index, use the following code: + +```python +print(json.dumps(sorted(results, key=lambda x: x['index']),indent=2)) +``` + +The following are the results sorted by index: + +```json +[ + { + "index": 0, + "relevanceScore": 0.0025114635936915874 + }, + { + "index": 1, + "relevanceScore": 2.4876489987946115e-05 + }, + { + "index": 2, + "relevanceScore": 0.7711548805236816 + }, + { + "index": 3, + "relevanceScore": 6.339210358419223e-06 + } +] +``` + +## Step 1: Create a connector and register the model + +To create a connector and register the model, use the following steps. + +### Step 1.1: Create a connector for the model + +First, create a connector for the model. + +If you are using self-managed OpenSearch, supply your AWS credentials: + +```json +POST /_plugins/_ml/connectors/_create +{ + "name": "Amazon Bedrock Rerank API", + "description": "Test connector for Amazon Bedrock Rerank API", + "version": 1, + "protocol": "aws_sigv4", + "credential": { + "access_key": "your_access_key", + "secret_key": "your_secret_key", + "session_token": "your_session_token" + }, + "parameters": { + "service_name": "bedrock", + "endpoint": "bedrock-agent-runtime", + "region": "your_bedrock_model_region_like_us-west-2", + "api_name": "rerank", + "model_id": "amazon.rerank-v1:0" + }, + "actions": [ + { + "action_type": "PREDICT", + "method": "POST", + "url": "https://${parameters.endpoint}.${parameters.region}.amazonaws.com/${parameters.api_name}", + "headers": { + "x-amz-content-sha256": "required", + "content-type": "application/json" + }, + "pre_process_function": "connector.pre_process.bedrock.rerank", + "request_body": """ + { + "queries": ${parameters.queries}, + "rerankingConfiguration": { + "bedrockRerankingConfiguration": { + "modelConfiguration": { + "modelArn": "arn:aws:bedrock:${parameters.region}::foundation-model/${parameters.model_id}" + } + }, + "type": "BEDROCK_RERANKING_MODEL" + }, + "sources": ${parameters.sources} + } + """, + "post_process_function": "connector.post_process.bedrock.rerank" + } + ] +} +``` +{% include copy-curl.html %} + +If you are using Amazon OpenSearch Service, you can provide an AWS Identity and Access Management (IAM) role Amazon Resource Name (ARN) that allows access to Amazon Bedrock. For more information, see the [AWS documentation](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/ml-amazon-connector.html). Use the following request to create a connector: + +```json +POST /_plugins/_ml/connectors/_create +{ + "name": "Amazon Bedrock Rerank API", + "description": "Test connector for Amazon Bedrock Rerank API", + "version": 1, + "protocol": "aws_sigv4", + "credential": { + "roleArn": "your_role_arn_which_allows_access_to_bedrock_agent_runtime_rerank_api" + }, + "parameters": { + "service_name": "bedrock", + "endpoint": "bedrock-agent-runtime", + "region": "your_bedrock_model_region_like_us-west-2", + "api_name": "rerank", + "model_id": "amazon.rerank-v1:0" + }, + "actions": [ + { + "action_type": "PREDICT", + "method": "POST", + "url": "https://${parameters.endpoint}.${parameters.region}.amazonaws.com/${parameters.api_name}", + "headers": { + "x-amz-content-sha256": "required", + "content-type": "application/json" + }, + "pre_process_function": "connector.pre_process.bedrock.rerank", + "request_body": """ + { + "queries": ${parameters.queries}, + "rerankingConfiguration": { + "bedrockRerankingConfiguration": { + "modelConfiguration": { + "modelArn": "arn:aws:bedrock:${parameters.region}::foundation-model/${parameters.model_id}" + } + }, + "type": "BEDROCK_RERANKING_MODEL" + }, + "sources": ${parameters.sources} + } + """, + "post_process_function": "connector.post_process.bedrock.rerank" + } + ] +} +``` +{% include copy-curl.html %} + +### Step 1.2: Register and deploy the model + +Use the connector ID from the response to register and deploy the model: + +```json +POST /_plugins/_ml/models/_register?deploy=true +{ + "name": "Amazon Bedrock Rerank API", + "function_name": "remote", + "description": "test Amazon Bedrock Rerank API", + "connector_id": "your_connector_id" +} +``` +{% include copy-curl.html %} + +Note the model ID in the response; you'll use it in the following steps. + +### Step 1.3: Test the model + +Test the model by using the Predict API: + +```json +POST _plugins/_ml/_predict/text_similarity/your_model_id +{ + "query_text": "What is the capital city of America?", + "text_docs": [ + "Carson City is the capital city of the American state of Nevada.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan.", + "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district.", + "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + ] +} +``` +{% include copy-curl.html %} + +Alternatively, you can test the model using the following query. This query bypasses the `pre_process_function` and calls the Rerank API directly: + +```json +POST _plugins/_ml/models/your_model_id/_predict +{ + "parameters": { + "queries": [ + { + "textQuery": { + "text": "What is the capital city of America?" + }, + "type": "TEXT" + } + ], + "sources": [ + { + "inlineDocumentSource": { + "textDocument": { + "text": "Carson City is the capital city of the American state of Nevada." + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + { + "inlineDocumentSource": { + "textDocument": { + "text": "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan." + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + { + "inlineDocumentSource": { + "textDocument": { + "text": "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district." + }, + "type": "TEXT" + }, + "type": "INLINE" + }, + { + "inlineDocumentSource": { + "textDocument": { + "text": "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + }, + "type": "TEXT" + }, + "type": "INLINE" + } + ] + } +} +``` +{% include copy-curl.html %} + +The connector `pre_process_function` transforms the input into the format required by the Predict API `parameters`. + +By default, the Amazon Bedrock Rerank API output is formatted as follows: + +```json +[ + { + "index": 2, + "relevanceScore": 0.7711548724998493 + }, + { + "index": 0, + "relevanceScore": 0.0025114635138098534 + }, + { + "index": 1, + "relevanceScore": 2.4876490010363496e-05 + }, + { + "index": 3, + "relevanceScore": 6.339210403977635e-06 + } +] +``` + +The connector `post_process_function` transforms the model's output into a format that the [Reranker processor](https://opensearch.org/docs/latest/search-plugins/search-pipelines/rerank-processor/) can interpret and orders the results by index. + +The response contains four `similarity` outputs. For each `similarity` output, the `data` array contains a relevance score for each document against the query. The `similarity` outputs are provided in the order of the input documents; the first similarity result pertains to the first document: + +```json +{ + "inference_results": [ + { + "output": [ + { + "name": "similarity", + "data_type": "FLOAT32", + "shape": [ + 1 + ], + "data": [ + 0.0025114636 + ] + }, + { + "name": "similarity", + "data_type": "FLOAT32", + "shape": [ + 1 + ], + "data": [ + 2.487649e-05 + ] + }, + { + "name": "similarity", + "data_type": "FLOAT32", + "shape": [ + 1 + ], + "data": [ + 0.7711549 + ] + }, + { + "name": "similarity", + "data_type": "FLOAT32", + "shape": [ + 1 + ], + "data": [ + 6.3392104e-06 + ] + } + ], + "status_code": 200 + } + ] +} +``` + +## Step 2: Create a reranking pipeline + +To create a reranking pipeline, use the following steps. + +### Step 2.1: Ingest test data + +Use the following request to ingest data into your index: + +```json +POST _bulk +{ "index": { "_index": "my-test-data" } } +{ "passage_text" : "Carson City is the capital city of the American state of Nevada." } +{ "index": { "_index": "my-test-data" } } +{ "passage_text" : "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan." } +{ "index": { "_index": "my-test-data" } } +{ "passage_text" : "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district." } +{ "index": { "_index": "my-test-data" } } +{ "passage_text" : "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." } +``` +{% include copy-curl.html %} + +### Step 2.2: Create a reranking pipeline + +Create a reranking pipeline using the Amazon Bedrock reranking model: + +```json +PUT /_search/pipeline/rerank_pipeline_bedrock +{ + "description": "Pipeline for reranking with Bedrock rerank model", + "response_processors": [ + { + "rerank": { + "ml_opensearch": { + "model_id": "your_model_id_created_in_step1" + }, + "context": { + "document_fields": ["passage_text"] + } + } + } + ] +} +``` +{% include copy-curl.html %} + +If you provide multiple field names in `document_fields`, the values of all fields are first concatenated, after which reranking is performed. +{: .note} + +### Step 2.3: Test reranking + +First, test the query without using the reranking pipeline: + +```json +POST my-test-data/_search +{ + "query": { + "match": { + "passage_text": "What is the capital city of America?" + } + }, + "highlight": { + "pre_tags": [""], + "post_tags": [""], + "fields": {"passage_text": {}} + }, + "_source": false, + "fields": ["passage_text"] +} +``` +{% include copy-curl.html %} + +The first document in the response is `Carson City is the capital city of the American state of Nevada`, which is incorrect: + +```json +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4, + "relation": "eq" + }, + "max_score": 2.5045562, + "hits": [ + { + "_index": "my-test-data", + "_id": "1", + "_score": 2.5045562, + "fields": { + "passage_text": [ + "Carson City is the capital city of the American state of Nevada." + ] + }, + "highlight": { + "passage_text": [ + "Carson City is the capital city of the American state of Nevada." + ] + } + }, + { + "_index": "my-test-data", + "_id": "2", + "_score": 0.5807494, + "fields": { + "passage_text": [ + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan." + ] + }, + "highlight": { + "passage_text": [ + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean.", + "Its capital is Saipan." + ] + } + }, + { + "_index": "my-test-data", + "_id": "3", + "_score": 0.5261191, + "fields": { + "passage_text": [ + "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district." + ] + }, + "highlight": { + "passage_text": [ + "(also known as simply Washington or D.C., and officially as the District of Columbia) is the capital", + "of the United States.", + "It is a federal district." + ] + } + }, + { + "_index": "my-test-data", + "_id": "4", + "_score": 0.5083029, + "fields": { + "passage_text": [ + "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + ] + }, + "highlight": { + "passage_text": [ + "Capital punishment (the death penalty) has existed in the United States since beforethe United States", + "As of 2017, capital punishment is legal in 30 of the 50 states." + ] + } + } + ] + } +} +``` + +Next, test the query using the reranking pipeline: + +```json +POST my-test-data/_search?search_pipeline=rerank_pipeline_bedrock +{ + "query": { + "match": { + "passage_text": "What is the capital city of America?" + } + }, + "ext": { + "rerank": { + "query_context": { + "query_text": "What is the capital city of America?" + } + } + }, + "highlight": { + "pre_tags": [""], + "post_tags": [""], + "fields": {"passage_text": {}} + }, + "_source": false, + "fields": ["passage_text"] +} +``` +{% include copy-curl.html %} + +The first document in the response is `"Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district."`, which is correct: + +```json +{ + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 4, + "relation": "eq" + }, + "max_score": 0.7711549, + "hits": [ + { + "_index": "my-test-data", + "_id": "3", + "_score": 0.7711549, + "fields": { + "passage_text": [ + "Washington, D.C. (also known as simply Washington or D.C., and officially as the District of Columbia) is the capital of the United States. It is a federal district." + ] + }, + "highlight": { + "passage_text": [ + "(also known as simply Washington or D.C., and officially as the District of Columbia) is the capital", + "of the United States.", + "It is a federal district." + ] + } + }, + { + "_index": "my-test-data", + "_id": "1", + "_score": 0.0025114636, + "fields": { + "passage_text": [ + "Carson City is the capital city of the American state of Nevada." + ] + }, + "highlight": { + "passage_text": [ + "Carson City is the capital city of the American state of Nevada." + ] + } + }, + { + "_index": "my-test-data", + "_id": "2", + "_score": 02.487649e-05, + "fields": { + "passage_text": [ + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean. Its capital is Saipan." + ] + }, + "highlight": { + "passage_text": [ + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean.", + "Its capital is Saipan." + ] + } + }, + { + "_index": "my-test-data", + "_id": "4", + "_score": 6.3392104e-06, + "fields": { + "passage_text": [ + "Capital punishment (the death penalty) has existed in the United States since beforethe United States was a country. As of 2017, capital punishment is legal in 30 of the 50 states." + ] + }, + "highlight": { + "passage_text": [ + "Capital punishment (the death penalty) has existed in the United States since beforethe United States", + "As of 2017, capital punishment is legal in 30 of the 50 states." + ] + } + } + ] + }, + "profile": { + "shards": [] + } +} +``` + +You can reuse the same query by specifying the `query_text_path` instead of `query_text`: + +```json +POST my-test-data/_search?search_pipeline=rerank_pipeline_bedrock +{ + "query": { + "match": { + "passage_text": "What is the capital city of America?" + } + }, + "ext": { + "rerank": { + "query_context": { + "query_text_path": "query.match.passage_text.query" + } + } + }, + "highlight": { + "pre_tags": [""], + "post_tags": [""], + "fields": {"passage_text": {}} + }, + "_source": false, + "fields": ["passage_text"] +} +``` +{% include copy-curl.html %} + From adebc972f98c700295bb334a3447ca68f34a4442 Mon Sep 17 00:00:00 2001 From: Peter Alfonsi Date: Thu, 9 Jan 2025 05:44:45 -0800 Subject: [PATCH 11/11] Fix incorrect documentation about query cache stats (#9021) * Fix incorrect documentation about query cache stats Signed-off-by: Peter Alfonsi * Update _api-reference/nodes-apis/nodes-stats.md Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Peter Alfonsi --------- Signed-off-by: Peter Alfonsi Signed-off-by: Peter Alfonsi Co-authored-by: Peter Alfonsi Co-authored-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Signed-off-by: Russ Cam --- _api-reference/nodes-apis/nodes-stats.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_api-reference/nodes-apis/nodes-stats.md b/_api-reference/nodes-apis/nodes-stats.md index 0a5ca00634..1b0552b538 100644 --- a/_api-reference/nodes-apis/nodes-stats.md +++ b/_api-reference/nodes-apis/nodes-stats.md @@ -943,12 +943,12 @@ warmer.total | Integer | The total number of index warming operations. warmer.total_time_in_millis | Integer | The total time for all index warming operations, in milliseconds. query_cache | Statistics about query cache operations for the node. query_cache.memory_size_in_bytes | Integer | The amount of memory used for the query cache for all shards in the node. -query_cache.total_count | Integer | The total number of hits, misses, and cached queries in the query cache. +query_cache.total_count | Integer | The total number of hits and misses in the query cache. query_cache.hit_count | Integer | The total number of hits in the query cache. query_cache.miss_count | Integer | The total number of misses in the query cache. -query_cache.cache_size | Integer | The size of the query cache, in bytes. -query_cache.cache_count | Integer | The number of queries in the query cache. -query_cache.evictions | Integer | The number of evictions in the query cache. +query_cache.cache_size | Integer | The number of queries currently in the query cache. +query_cache.cache_count | Integer | The total number of queries that have been added to the query cache, including those that have since been evicted. +query_cache.evictions | Integer | The number of evictions from the query cache. fielddata | Object | Statistics about the field data cache for all shards in the node. fielddata.memory_size_in_bytes | Integer | The total amount of memory used for the field data cache for all shards in the node. fielddata.evictions | Integer | The number of evictions in the field data cache.