Tweak security system indexes refresh behavior for stateless (elastic…

…#98285)

This is an omnibus PR that:
 * toggles the fast_refresh option for the profiles index (to make it alike to the .kibana index)
 * makes API Key creation refresh policy default to IMMEDIATE from WAIT_UNTIL in serverless, to mitigate the long automatic refresh interval

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/AbstractCreateApiKeyRequest.java

@@ -21,17 +21,17 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 
 public abstract class AbstractCreateApiKeyRequest extends ActionRequest {
-    public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.WAIT_UNTIL;
     protected final String id;
     protected String name;
     protected TimeValue expiration;
     protected Map<String, Object> metadata;
     protected List<RoleDescriptor> roleDescriptors = Collections.emptyList();
-    protected WriteRequest.RefreshPolicy refreshPolicy = DEFAULT_REFRESH_POLICY;
+    protected WriteRequest.RefreshPolicy refreshPolicy;
 
     public AbstractCreateApiKeyRequest() {
         super();
@@ -68,6 +68,10 @@ public WriteRequest.RefreshPolicy getRefreshPolicy() {
         return refreshPolicy;
     }
 
+    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
+        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null");
+    }
+
     public Map<String, Object> getMetadata() {
         return metadata;
     }
@@ -94,6 +98,7 @@ public ActionRequestValidationException validate() {
                 validationException
             );
         }
+        assert refreshPolicy != null : "refresh policy is required";
         return validationException;
     }
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java

@@ -21,7 +21,6 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 
 /**
  * Request class used for the creation of an API key. The request requires a name to be provided
@@ -103,10 +102,6 @@ public void setRoleDescriptors(@Nullable List<RoleDescriptor> roleDescriptors) {
         this.roleDescriptors = (roleDescriptors == null) ? List.of() : List.copyOf(roleDescriptors);
     }
 
-    public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) {
-        this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null");
-    }
-
     public void setMetadata(Map<String, Object> metadata) {
         this.metadata = metadata;
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestTests.java

@@ -21,6 +21,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.containsStringIgnoringCase;
 import static org.hamcrest.Matchers.equalTo;
@@ -31,6 +34,7 @@ public class CreateApiKeyRequestTests extends ESTestCase {
     public void testNameValidation() {
         final String name = randomAlphaOfLengthBetween(1, 256);
         CreateApiKeyRequest request = new CreateApiKeyRequest();
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
 
         ActionRequestValidationException ve = request.validate();
         assertThat(ve.validationErrors().size(), is(1));
@@ -78,6 +82,7 @@ public void testNameValidation() {
     public void testMetadataKeyValidation() {
         final String name = randomAlphaOfLengthBetween(1, 256);
         CreateApiKeyRequest request = new CreateApiKeyRequest();
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         request.setName(name);
         request.setMetadata(Map.of("_foo", "bar"));
         final ActionRequestValidationException ve = request.validate();
@@ -112,6 +117,7 @@ public void testRoleDescriptorValidation() {
             ),
             null
         );
+        request1.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
         final ActionRequestValidationException ve1 = request1.validate();
         assertNotNull(ve1);
         assertThat(ve1.validationErrors().get(0), containsString("unknown cluster privilege"));

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequestTests.java

@@ -17,6 +17,10 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
+
 public class CreateCrossClusterApiKeyRequestTests extends AbstractWireSerializingTestCase<CreateCrossClusterApiKeyRequest> {
 
     private String access;
@@ -35,12 +39,14 @@ protected Writeable.Reader<CreateCrossClusterApiKeyRequest> instanceReader() {
 
     @Override
     protected CreateCrossClusterApiKeyRequest createTestInstance() {
-        return new CreateCrossClusterApiKeyRequest(
+        CreateCrossClusterApiKeyRequest request = new CreateCrossClusterApiKeyRequest(
             randomAlphaOfLengthBetween(3, 8),
             roleDescriptorBuilder,
             randomExpiration(),
             randomMetadata()
         );
+        request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE));
+        return request;
     }
 
     @Override

x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.idp.action;
 
 import org.elasticsearch.ElasticsearchException;
-import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.Response;
@@ -57,6 +56,9 @@
 import java.util.zip.Deflater;
 import java.util.zip.DeflaterOutputStream;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.hamcrest.Matchers.containsString;
@@ -405,10 +407,7 @@ private void registerServiceProvider(String entityId, String acsUrl) throws Exce
             )
         );
         spFields.put("privileges", Map.of("resource", entityId, "roles", Set.of("sso:(\\w+)")));
-        Request request = new Request(
-            "PUT",
-            "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue()
-        );
+        Request request = new Request("PUT", "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + IMMEDIATE.getValue());
         request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.map(spFields);
@@ -437,7 +436,7 @@ private void registerApplicationPrivileges() throws IOException {
     }
 
     private void registerApplicationPrivileges(Map<String, Set<String>> privileges) throws IOException {
-        Request request = new Request("PUT", "/_security/privilege?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue());
+        Request request = new Request("PUT", "/_security/privilege?refresh=" + IMMEDIATE.getValue());
         request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER);
         final XContentBuilder builder = XContentFactory.jsonBuilder();
         builder.startObject();
@@ -487,6 +486,7 @@ private String getApiKeyFromCredentials(String username, SecureString password)
         );
         final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client).setName("test key")
             .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L)))
+            .setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE, NONE))
             .get();
         assertNotNull(response);
         return Base64.getEncoder().encodeToString((response.getId() + ":" + response.getKey().toString()).getBytes(StandardCharsets.UTF_8));

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java

@@ -42,6 +42,9 @@
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE;
+import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL;
 import static org.elasticsearch.test.SecuritySettingsSource.TEST_PASSWORD_HASHED;
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
@@ -455,6 +458,7 @@ private Client limitedClientApiKey() throws ExecutionException, InterruptedExcep
             ),
             null
         );
+        createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE, NONE));
         final CreateApiKeyResponse createApiKeyResponse = limitedClient().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).get();
 
         final String base64ApiKey = Base64.getEncoder()