From aa51ade15080adaec8d4d48456fc2128bf670afd Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Fri, 10 Jan 2025 11:51:20 -0800 Subject: [PATCH] Added memory threshold settings in Controller API tests. Signed-off-by: Nathalie Jonathan --- ...6ae7bbc6bd0a2ed2671a349103ce0d70344aaa.log | 335 ++++++++++++++++++ tests/plugins/ml/ml/controller/create.yaml | 6 + tests/plugins/ml/ml/controller/delete.yaml | 6 + tests/plugins/ml/ml/controller/get.yaml | 6 + tests/plugins/ml/ml/controller/update.yaml | 6 + 5 files changed, 359 insertions(+) create mode 100644 logs/test-spec-2.18.0-daeaa72ad2dd9c8f64703f44896ae7bbc6bd0a2ed2671a349103ce0d70344aaa.log diff --git a/logs/test-spec-2.18.0-daeaa72ad2dd9c8f64703f44896ae7bbc6bd0a2ed2671a349103ce0d70344aaa.log b/logs/test-spec-2.18.0-daeaa72ad2dd9c8f64703f44896ae7bbc6bd0a2ed2671a349103ce0d70344aaa.log new file mode 100644 index 00000000..5ede8db6 --- /dev/null +++ b/logs/test-spec-2.18.0-daeaa72ad2dd9c8f64703f44896ae7bbc6bd0a2ed2671a349103ce0d70344aaa.log @@ -0,0 +1,335 @@ +[INFO] Authenticating with admin ... +[INFO] Connecting to https://localhost:9200 ... (1/20) +[INFO] Evaluating ml/agents.yaml ... +[INFO] => POST /_plugins/_ml/agents/_register ({}) [application/json] { + "name": "Test_Agent_For_RAG", + "type": "flow", + "description": "this is a test agent", + "tools": [ + { + "type": "VectorDBTool", + "parameters": { + "model_id": "YOUR_TEXT_EMBEDDING_MODEL_ID", + "index": "my_test_data", + "embedding_field": "embedding", + "source_field": [ + "text" + ], + "input": "test_question" + } + }, + { + "type": "MLModelTool", + "description": "A general tool to answer any question", + "parameters": { + "model_id": "YOUR_LLM_MODEL_ID" + } + } + ] +} +[INFO] <= 200 (application/json; charset=UTF-8) | { + "agent_id": "KejBUZQBaqMO3uIYKp67" +} +[INFO] $ { + "outputs": { + "test_agent_id": "KejBUZQBaqMO3uIYKp67" + } +} +[INFO] => DELETE /_plugins/_ml/agents/KejBUZQBaqMO3uIYKp67 ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "_index": ".plugins-ml-agent", + "_id": "KejBUZQBaqMO3uIYKp67", + "_version": 2, + "result": "deleted", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 19, + "_primary_term": 6 +} +[INFO] => DELETE /_plugins/_ml/agents/KejBUZQBaqMO3uIYKp67 ({}) [application/json] +[INFO] <= 404 (application/json) | { + "error": { + "root_cause": [ + { + "type": "status_exception", + "reason": "Failed to find agent with the provided agent id: KejBUZQBaqMO3uIYKp67" + } + ], + "type": "status_exception", + "reason": "Failed to find agent with the provided agent id: KejBUZQBaqMO3uIYKp67" + }, + "status": 404 +} +[INFO] Evaluating ml/connectors.yaml ... +[INFO] => POST /_plugins/_ml/connectors/_create ({}) [application/json] { + "name": "OpenAI Chat Connector", + "description": "The connector to public OpenAI model service for GPT 3.5", + "version": 1, + "protocol": "http", + "parameters": { + "endpoint": "api.openai.com", + "model": "gpt-3.5-turbo" + }, + "credential": { + "openAI_key": "test_api_key" + }, + "actions": [ + { + "action_type": "predict", + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer Key" + }, + "request_body": "{ \"model\": \"model\", \"messages\": \"messages\" }" + } + ] +} +[INFO] <= 200 (application/json; charset=UTF-8) | { + "connector_id": "KujBUZQBaqMO3uIYK57K" +} +[INFO] $ { + "outputs": { + "test_connector_id": "KujBUZQBaqMO3uIYK57K" + } +} +[INFO] => GET /_plugins/_ml/connectors/KujBUZQBaqMO3uIYK57K ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "name": "OpenAI Chat Connector", + "version": "1", + "description": "The connector to public OpenAI model service for GPT 3.5", + "protocol": "http", + "parameters": { + "endpoint": "api.openai.com", + "model": "gpt-3.5-turbo" + }, + "actions": [ + { + "action_type": "PREDICT", + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Authorization": "Bearer Key" + }, + "request_body": "{ \"model\": \"model\", \"messages\": \"messages\" }" + } + ], + "created_time": 1736538401737, + "last_updated_time": 1736538401737 +} +[INFO] => DELETE /_plugins/_ml/connectors/KujBUZQBaqMO3uIYK57K ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "_index": ".plugins-ml-connector", + "_id": "KujBUZQBaqMO3uIYK57K", + "_version": 2, + "result": "deleted", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 40, + "_primary_term": 6 +} +[INFO] => DELETE /_plugins/_ml/connectors/KujBUZQBaqMO3uIYK57K ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "_index": ".plugins-ml-connector", + "_id": "KujBUZQBaqMO3uIYK57K", + "_version": 3, + "result": "not_found", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 41, + "_primary_term": 6 +} +[INFO] Evaluating ml/model_groups.yaml ... +[INFO] => POST /_plugins/_ml/model_groups/_register ({}) [application/json] { + "name": "NLP_Group", + "description": "Model group for NLP models." +} +[INFO] <= 200 (application/json; charset=UTF-8) | { + "model_group_id": "K-jBUZQBaqMO3uIYLJ5u", + "status": "CREATED" +} +[INFO] $ { + "outputs": { + "test_model_group_id": "K-jBUZQBaqMO3uIYLJ5u" + } +} +[INFO] => GET /_plugins/_ml/model_groups/K-jBUZQBaqMO3uIYLJ5u ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "name": "NLP_Group", + "latest_version": 0, + "description": "Model group for NLP models.", + "owner": { + "name": "admin", + "backend_roles": [ + "admin" + ], + "roles": [ + "own_index", + "all_access" + ], + "custom_attribute_names": [], + "user_requested_tenant": null + }, + "access": "private", + "created_time": 1736538401901, + "last_updated_time": 1736538401901 +} +[INFO] => DELETE /_plugins/_ml/model_groups/K-jBUZQBaqMO3uIYLJ5u ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "_index": ".plugins-ml-model-group", + "_id": "K-jBUZQBaqMO3uIYLJ5u", + "_version": 2, + "result": "deleted", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 236, + "_primary_term": 6 +} +[INFO] => DELETE /_plugins/_ml/model_groups/K-jBUZQBaqMO3uIYLJ5u ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "_index": ".plugins-ml-model-group", + "_id": "K-jBUZQBaqMO3uIYLJ5u", + "_version": 3, + "result": "not_found", + "forced_refresh": true, + "_shards": { + "total": 2, + "successful": 2, + "failed": 0 + }, + "_seq_no": 237, + "_primary_term": 6 +} +[INFO] Evaluating ml/models.yaml ... +[INFO] => POST /_plugins/_ml/models/_register ({}) [application/json] { + "name": "huggingface/sentence-transformers/msmarco-distilbert-base-tas-b", + "version": "1.0.1", + "model_format": "TORCH_SCRIPT" +} +[INFO] <= 200 (application/json; charset=UTF-8) | { + "task_id": "LOjBUZQBaqMO3uIYLJ6y", + "status": "CREATED" +} +[INFO] $ { + "outputs": { + "task_id": "LOjBUZQBaqMO3uIYLJ6y" + } +} +[INFO] => GET /_plugins/_ml/tasks/LOjBUZQBaqMO3uIYLJ6y ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "CREATED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538401970, + "is_async": true +} +[INFO] { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "CREATED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538401970, + "is_async": true +} +[INFO] Failed, retrying, 5 retries left ... +[INFO] => GET /_plugins/_ml/tasks/LOjBUZQBaqMO3uIYLJ6y ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "FAILED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538407919, + "error": "/usr/share/opensearch/data/ml_cache/models_cache/register/slnBUZQBdUaAr33ZL44-/57/huggingface/sentence-transformers/msmarco-distilbert-base-tas-b.zip", + "is_async": true +} +[INFO] { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "FAILED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538407919, + "error": "/usr/share/opensearch/data/ml_cache/models_cache/register/slnBUZQBdUaAr33ZL44-/57/huggingface/sentence-transformers/msmarco-distilbert-base-tas-b.zip", + "is_async": true +} +[INFO] Failed, retrying, 4 retries left ... +[INFO] => GET /_plugins/_ml/tasks/LOjBUZQBaqMO3uIYLJ6y ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "FAILED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538407919, + "error": "/usr/share/opensearch/data/ml_cache/models_cache/register/slnBUZQBdUaAr33ZL44-/57/huggingface/sentence-transformers/msmarco-distilbert-base-tas-b.zip", + "is_async": true +} +[INFO] { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "FAILED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538407919, + "error": "/usr/share/opensearch/data/ml_cache/models_cache/register/slnBUZQBdUaAr33ZL44-/57/huggingface/sentence-transformers/msmarco-distilbert-base-tas-b.zip", + "is_async": true +} +[INFO] Failed, retrying, 3 retries left ... +[INFO] => GET /_plugins/_ml/tasks/LOjBUZQBaqMO3uIYLJ6y ({}) [application/json] +[INFO] <= 200 (application/json; charset=UTF-8) | { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "FAILED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538407919, + "error": "/usr/share/opensearch/data/ml_cache/models_cache/register/slnBUZQBdUaAr33ZL44-/57/huggingface/sentence-transformers/msmarco-distilbert-base-tas-b.zip", + "is_async": true +} +[INFO] { + "task_type": "REGISTER_MODEL", + "function_name": "TEXT_EMBEDDING", + "state": "FAILED", + "worker_node": [ + "ZcCsgpwPQsOq347BXU79jQ" + ], + "create_time": 1736538401970, + "last_update_time": 1736538407919, + "error": "/usr/share/opensearch/data/ml_cache/models_cache/register/slnBUZQBdUaAr33ZL44-/57/huggingface/sentence-transformers/msmarco-distilbert-base-tas-b.zip", + "is_async": true +} +[INFO] Failed, retrying, 2 retries left ... diff --git a/tests/plugins/ml/ml/controller/create.yaml b/tests/plugins/ml/ml/controller/create.yaml index 2e1f6d30..40f7fcb8 100644 --- a/tests/plugins/ml/ml/controller/create.yaml +++ b/tests/plugins/ml/ml/controller/create.yaml @@ -3,6 +3,12 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test the creation of a controller. version: '>= 2.12' prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.native_memory_threshold: 100 - path: /_plugins/_ml/models/_register id: register_model method: POST diff --git a/tests/plugins/ml/ml/controller/delete.yaml b/tests/plugins/ml/ml/controller/delete.yaml index 9e7426aa..6187e5cf 100644 --- a/tests/plugins/ml/ml/controller/delete.yaml +++ b/tests/plugins/ml/ml/controller/delete.yaml @@ -3,6 +3,12 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test the deletion of a controller. version: '>= 2.12' prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.native_memory_threshold: 100 - path: /_plugins/_ml/models/_register id: register_model method: POST diff --git a/tests/plugins/ml/ml/controller/get.yaml b/tests/plugins/ml/ml/controller/get.yaml index 0707953c..3ea9d881 100644 --- a/tests/plugins/ml/ml/controller/get.yaml +++ b/tests/plugins/ml/ml/controller/get.yaml @@ -3,6 +3,12 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test the retrieval of a controller. version: '>= 2.12' prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.native_memory_threshold: 100 - path: /_plugins/_ml/models/_register id: register_model method: POST diff --git a/tests/plugins/ml/ml/controller/update.yaml b/tests/plugins/ml/ml/controller/update.yaml index a259d74b..557ca813 100644 --- a/tests/plugins/ml/ml/controller/update.yaml +++ b/tests/plugins/ml/ml/controller/update.yaml @@ -3,6 +3,12 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test updating a controller. version: '>= 2.12' prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.native_memory_threshold: 100 - path: /_plugins/_ml/models/_register id: register_model method: POST