From 8d9786bf1d7e5f8e06b59c323ed0488b607738f0 Mon Sep 17 00:00:00 2001
From: Daniel Widdis <widdis@gmail.com>
Date: Mon, 23 Sep 2024 09:59:38 -0700
Subject: [PATCH] [Feature/multi_tenancy] Delay context restoring for
 initializing master key and deleting agent (#2866)

* Delay context restoring for initializing master key and deleting agent

Signed-off-by: Daniel Widdis <widdis@gmail.com>

* Simplify restoring context with runBefore

Signed-off-by: Daniel Widdis <widdis@gmail.com>

---------

Signed-off-by: Daniel Widdis <widdis@gmail.com>
---
 .../ml/engine/encryptor/EncryptorImpl.java    |  2 +-
 .../agents/DeleteAgentTransportAction.java    | 35 +++++++++----------
 2 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/encryptor/EncryptorImpl.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/encryptor/EncryptorImpl.java
index 329459973e..5615236f95 100644
--- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/encryptor/EncryptorImpl.java
+++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/encryptor/EncryptorImpl.java
@@ -187,7 +187,6 @@ private void handleGetDataObjectResponse(
         AtomicReference<Exception> exceptionRef,
         CountDownLatch latch
     ) {
-        context.restore();
         log.debug("Completed Get MASTER_KEY Request, for tenant id:{}", tenantId);
 
         if (throwable != null) {
@@ -195,6 +194,7 @@ private void handleGetDataObjectResponse(
         } else {
             handleGetDataObjectSuccess(response, tenantId, exceptionRef, latch, context);
         }
+        context.restore();
     }
 
     private void handleGetDataObjectFailure(Throwable throwable, AtomicReference<Exception> exceptionRef, CountDownLatch latch) {
diff --git a/plugin/src/main/java/org/opensearch/ml/action/agents/DeleteAgentTransportAction.java b/plugin/src/main/java/org/opensearch/ml/action/agents/DeleteAgentTransportAction.java
index 8e702d7186..f9d3d8a469 100644
--- a/plugin/src/main/java/org/opensearch/ml/action/agents/DeleteAgentTransportAction.java
+++ b/plugin/src/main/java/org/opensearch/ml/action/agents/DeleteAgentTransportAction.java
@@ -95,20 +95,19 @@ protected void doExecute(Task task, ActionRequest request, ActionListener<Delete
             .build();
 
         try (ThreadContext.StoredContext context = client.threadPool().getThreadContext().stashContext()) {
-
+            ActionListener<DeleteResponse> wrappedListener = ActionListener.runBefore(actionListener, context::restore);
             sdkClient
                 .getDataObjectAsync(getDataObjectRequest, client.threadPool().executor(GENERAL_THREAD_POOL))
                 .whenComplete((r, throwable) -> {
-                    context.restore();
                     log.debug("Completed Get Agent Request, Agent id:{}", agentId);
                     if (throwable != null) {
                         Exception cause = SdkClientUtils.unwrapAndConvertToException(throwable);
                         if (cause instanceof IndexNotFoundException) {
                             log.info("Failed to get Agent index", cause);
-                            actionListener.onFailure(new OpenSearchStatusException("Failed to get agent index", RestStatus.NOT_FOUND));
+                            wrappedListener.onFailure(new OpenSearchStatusException("Failed to get agent index", RestStatus.NOT_FOUND));
                         } else {
                             log.error("Failed to get ML Agent {}", agentId, cause);
-                            actionListener.onFailure(cause);
+                            wrappedListener.onFailure(cause);
                         }
                     } else {
                         try {
@@ -122,9 +121,14 @@ protected void doExecute(Task task, ActionRequest request, ActionListener<Delete
                                     ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser);
                                     MLAgent mlAgent = MLAgent.parse(parser);
                                     if (TenantAwareHelper
-                                        .validateTenantResource(mlFeatureEnabledSetting, tenantId, mlAgent.getTenantId(), actionListener)) {
+                                        .validateTenantResource(
+                                            mlFeatureEnabledSetting,
+                                            tenantId,
+                                            mlAgent.getTenantId(),
+                                            wrappedListener
+                                        )) {
                                         if (mlAgent.getIsHidden() && !isSuperAdmin) {
-                                            actionListener
+                                            wrappedListener
                                                 .onFailure(
                                                     new OpenSearchStatusException(
                                                         "User doesn't have privilege to perform this operation on this agent",
@@ -144,30 +148,25 @@ protected void doExecute(Task task, ActionRequest request, ActionListener<Delete
                                                             .build(),
                                                         client.threadPool().executor(GENERAL_THREAD_POOL)
                                                     )
-                                                    .whenComplete(
-                                                        (response, delThrowable) -> handleDeleteResponse(
-                                                            response,
-                                                            delThrowable,
-                                                            tenantId,
-                                                            actionListener
-                                                        )
-                                                    );
+                                                    .whenComplete((response, delThrowable) -> {
+                                                        handleDeleteResponse(response, delThrowable, tenantId, wrappedListener);
+                                                    });
                                             } catch (Exception e) {
                                                 log.error("Failed to delete ML agent: {}", agentId, e);
-                                                actionListener.onFailure(e);
+                                                wrappedListener.onFailure(e);
                                             }
                                         }
                                     }
                                 } catch (Exception e) {
                                     log.error("Failed to parse ml agent {}", agentId, e);
-                                    actionListener.onFailure(e);
+                                    wrappedListener.onFailure(e);
                                 }
                             } else {
-                                actionListener.onFailure(new OpenSearchStatusException("Fail to find ml agent", RestStatus.NOT_FOUND));
+                                wrappedListener.onFailure(new OpenSearchStatusException("Fail to find ml agent", RestStatus.NOT_FOUND));
                             }
                         } catch (Exception e) {
                             log.error("Failed to delete ML agent: {}", agentId, e);
-                            actionListener.onFailure(e);
+                            wrappedListener.onFailure(e);
                         }
                     }
                 });