From b35cfe427b3ed9918b2d2cf4ae30a76f9b03fac5 Mon Sep 17 00:00:00 2001 From: Albert Zaharovits Date: Fri, 11 Aug 2023 17:38:42 +0300 Subject: [PATCH] Tweak security system indexes refresh behavior for stateless (#98285) This is an omnibus PR that: * toggles the fast_refresh option for the profiles index (to make it alike to the .kibana index) * makes API Key creation refresh policy default to IMMEDIATE from WAIT_UNTIL in serverless, to mitigate the long automatic refresh interval --- .../apikey/AbstractCreateApiKeyRequest.java | 9 ++- .../action/apikey/CreateApiKeyRequest.java | 5 -- .../apikey/CreateApiKeyRequestTests.java | 6 ++ .../CreateCrossClusterApiKeyRequestTests.java | 8 ++- .../idp/action/SamlIdentityProviderTests.java | 12 ++-- .../integration/DlsFlsRequestCacheTests.java | 4 ++ .../security/authc/ApiKeyIntegTests.java | 53 ++++++++++++----- .../security/authc/TokenAuthIntegTests.java | 4 ++ .../authc/apikey/ApiKeySingleNodeTests.java | 59 +++++++++++-------- .../NativePrivilegeStoreSingleNodeTests.java | 4 ++ .../profile/SecurityDomainIntegTests.java | 4 ++ .../user/AnonymousUserIntegTests.java | 15 +++-- .../xpack/security/Security.java | 2 +- .../xpack/security/authc/ApiKeyService.java | 17 +++++- .../InternalEnrollmentTokenGenerator.java | 2 + .../action/apikey/RestCreateApiKeyAction.java | 14 +++-- .../RestCreateCrossClusterApiKeyAction.java | 2 + .../action/apikey/RestGrantApiKeyAction.java | 3 + .../support/SecuritySystemIndices.java | 27 ++++++--- .../authz/store/NativeRolesStoreTests.java | 5 +- ...AccessAuthenticationServiceIntegTests.java | 4 ++ .../support/SecurityIndexManagerTests.java | 3 +- 22 files changed, 183 insertions(+), 79 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/AbstractCreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/AbstractCreateApiKeyRequest.java index 38587d67ae708..d6c97aff52b35 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/AbstractCreateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/AbstractCreateApiKeyRequest.java @@ -21,17 +21,17 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import static org.elasticsearch.action.ValidateActions.addValidationError; public abstract class AbstractCreateApiKeyRequest extends ActionRequest { - public static final WriteRequest.RefreshPolicy DEFAULT_REFRESH_POLICY = WriteRequest.RefreshPolicy.WAIT_UNTIL; protected final String id; protected String name; protected TimeValue expiration; protected Map metadata; protected List roleDescriptors = Collections.emptyList(); - protected WriteRequest.RefreshPolicy refreshPolicy = DEFAULT_REFRESH_POLICY; + protected WriteRequest.RefreshPolicy refreshPolicy; public AbstractCreateApiKeyRequest() { super(); @@ -68,6 +68,10 @@ public WriteRequest.RefreshPolicy getRefreshPolicy() { return refreshPolicy; } + public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { + this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null"); + } + public Map getMetadata() { return metadata; } @@ -94,6 +98,7 @@ public ActionRequestValidationException validate() { validationException ); } + assert refreshPolicy != null : "refresh policy is required"; return validationException; } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java index 889257b1b1c0a..7ea53c9f11fc8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequest.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; /** * Request class used for the creation of an API key. The request requires a name to be provided @@ -103,10 +102,6 @@ public void setRoleDescriptors(@Nullable List roleDescriptors) { this.roleDescriptors = (roleDescriptors == null) ? List.of() : List.copyOf(roleDescriptors); } - public void setRefreshPolicy(WriteRequest.RefreshPolicy refreshPolicy) { - this.refreshPolicy = Objects.requireNonNull(refreshPolicy, "refresh policy may not be null"); - } - public void setMetadata(Map metadata) { this.metadata = metadata; } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestTests.java index 30338774de584..bac11d9f0fc53 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateApiKeyRequestTests.java @@ -21,6 +21,9 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.containsStringIgnoringCase; import static org.hamcrest.Matchers.equalTo; @@ -31,6 +34,7 @@ public class CreateApiKeyRequestTests extends ESTestCase { public void testNameValidation() { final String name = randomAlphaOfLengthBetween(1, 256); CreateApiKeyRequest request = new CreateApiKeyRequest(); + request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); ActionRequestValidationException ve = request.validate(); assertThat(ve.validationErrors().size(), is(1)); @@ -78,6 +82,7 @@ public void testNameValidation() { public void testMetadataKeyValidation() { final String name = randomAlphaOfLengthBetween(1, 256); CreateApiKeyRequest request = new CreateApiKeyRequest(); + request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); request.setName(name); request.setMetadata(Map.of("_foo", "bar")); final ActionRequestValidationException ve = request.validate(); @@ -112,6 +117,7 @@ public void testRoleDescriptorValidation() { ), null ); + request1.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); final ActionRequestValidationException ve1 = request1.validate(); assertNotNull(ve1); assertThat(ve1.validationErrors().get(0), containsString("unknown cluster privilege")); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequestTests.java index 79a5bb9379ef8..a0a9c9b31b430 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/action/apikey/CreateCrossClusterApiKeyRequestTests.java @@ -17,6 +17,10 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; + public class CreateCrossClusterApiKeyRequestTests extends AbstractWireSerializingTestCase { private String access; @@ -35,12 +39,14 @@ protected Writeable.Reader instanceReader() { @Override protected CreateCrossClusterApiKeyRequest createTestInstance() { - return new CreateCrossClusterApiKeyRequest( + CreateCrossClusterApiKeyRequest request = new CreateCrossClusterApiKeyRequest( randomAlphaOfLengthBetween(3, 8), roleDescriptorBuilder, randomExpiration(), randomMetadata() ); + request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); + return request; } @Override diff --git a/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java b/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java index 551b53948c1e5..e72d97d212119 100644 --- a/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java +++ b/x-pack/plugin/identity-provider/src/internalClusterTest/java/org/elasticsearch/xpack/idp/action/SamlIdentityProviderTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.idp.action; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; @@ -57,6 +56,9 @@ import java.util.zip.Deflater; import java.util.zip.DeflaterOutputStream; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; @@ -405,10 +407,7 @@ private void registerServiceProvider(String entityId, String acsUrl) throws Exce ) ); spFields.put("privileges", Map.of("resource", entityId, "roles", Set.of("sso:(\\w+)"))); - Request request = new Request( - "PUT", - "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue() - ); + Request request = new Request("PUT", "/_idp/saml/sp/" + urlEncode(entityId) + "?refresh=" + IMMEDIATE.getValue()); request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER); final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.map(spFields); @@ -437,7 +436,7 @@ private void registerApplicationPrivileges() throws IOException { } private void registerApplicationPrivileges(Map> privileges) throws IOException { - Request request = new Request("PUT", "/_security/privilege?refresh=" + WriteRequest.RefreshPolicy.IMMEDIATE.getValue()); + Request request = new Request("PUT", "/_security/privilege?refresh=" + IMMEDIATE.getValue()); request.setOptions(REQUEST_OPTIONS_AS_CONSOLE_USER); final XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); @@ -487,6 +486,7 @@ private String getApiKeyFromCredentials(String username, SecureString password) ); final CreateApiKeyResponse response = new CreateApiKeyRequestBuilder(client).setName("test key") .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L))) + .setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE, NONE)) .get(); assertNotNull(response); return Base64.getEncoder().encodeToString((response.getId() + ":" + response.getKey().toString()).getBytes(StandardCharsets.UTF_8)); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java index fcea7f26bc265..301b264308af0 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DlsFlsRequestCacheTests.java @@ -42,6 +42,9 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.test.SecuritySettingsSource.TEST_PASSWORD_HASHED; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -455,6 +458,7 @@ private Client limitedClientApiKey() throws ExecutionException, InterruptedExcep ), null ); + createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE, NONE)); final CreateApiKeyResponse createApiKeyResponse = limitedClient().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).get(); final String base64ApiKey = Base64.getEncoder() diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java index ca34de08481c9..ecb26a4ecbee3 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/ApiKeyIntegTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.action.get.GetAction; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Request; import org.elasticsearch.client.RequestOptions; @@ -125,6 +124,9 @@ import java.util.stream.IntStream; import java.util.stream.Stream; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER; import static org.elasticsearch.test.SecuritySettingsSource.HASHER; import static org.elasticsearch.test.SecuritySettingsSource.TEST_ROLE; @@ -264,6 +266,7 @@ public void testCreateApiKey() throws Exception { .setExpiration(TimeValue.timeValueHours(TimeUnit.DAYS.toHours(7L))) .setRoleDescriptors(Collections.singletonList(descriptor)) .setMetadata(ApiKeyTests.randomMetadata()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)) .get(); assertEquals("test key", response.getName()); @@ -278,7 +281,9 @@ public void testCreateApiKey() throws Exception { assertThat(getApiKeyInfo(client(), response.getId(), randomBoolean(), randomBoolean()).getType(), is(ApiKey.Type.REST)); // create simple api key - final CreateApiKeyResponse simple = new CreateApiKeyRequestBuilder(client).setName("simple").get(); + final CreateApiKeyResponse simple = new CreateApiKeyRequestBuilder(client).setName("simple") + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)) + .get(); assertEquals("simple", simple.getName()); assertNotNull(simple.getId()); assertNotNull(simple.getKey()); @@ -320,7 +325,7 @@ public void testMultipleApiKeysCanHaveSameName() { .setExpiration(null) .setRoleDescriptors(Collections.singletonList(descriptor)) .setMetadata(ApiKeyTests.randomMetadata()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.NONE) + .setRefreshPolicy(NONE) .get(); assertNotNull(response.getId()); assertNotNull(response.getKey()); @@ -338,7 +343,7 @@ public void testCreateApiKeyWithoutNameWillFail() { ); final ActionRequestValidationException e = expectThrows( ActionRequestValidationException.class, - () -> new CreateApiKeyRequestBuilder(client).get() + () -> new CreateApiKeyRequestBuilder(client).setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)).get() ); assertThat(e.getMessage(), containsString("api key name is required")); } @@ -643,7 +648,7 @@ private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionP assertFalse(created.isBefore(withinRetention)); UpdateResponse expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(0).getId()) .setDoc("expiration_time", withinRetention.toEpochMilli()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); @@ -653,21 +658,21 @@ private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionP assertTrue(Instant.now().isAfter(outsideRetention)); expirationDateUpdatedResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(1).getId()) .setDoc("expiration_time", outsideRetention.toEpochMilli()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); // Invalidate the 3rd key such that it cannot be deleted by the remover UpdateResponse invalidateUpdateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(2).getId()) .setDoc("invalidation_time", withinRetention.toEpochMilli(), "api_key_invalidated", true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(invalidateUpdateResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); // Invalidate the 4th key such that it will be deleted by the remover invalidateUpdateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(3).getId()) .setDoc("invalidation_time", outsideRetention.toEpochMilli(), "api_key_invalidated", true) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(invalidateUpdateResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); @@ -681,7 +686,7 @@ private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionP "api_key_invalidated", true ) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); @@ -695,7 +700,7 @@ private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionP "api_key_invalidated", true ) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); @@ -703,7 +708,7 @@ private void doTestDeletionBehaviorWhenKeysBecomeInvalidBeforeAndAfterRetentionP // It does not matter whether it has an expiration time or whether the expiration time is still within retention period updateResponse = client.prepareUpdate(SECURITY_MAIN_ALIAS, createdApiKeys.get(6).getId()) .setDoc("api_key_invalidated", true, "expiration_time", randomBoolean() ? withinRetention.toEpochMilli() : null) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(updateResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); @@ -1599,6 +1604,7 @@ public void testDerivedKeys() throws ExecutionException, InterruptedException { Collections.singletonList(new RoleDescriptor("role", new String[] { "manage_api_key", "manage_token" }, null, null)) ) .setMetadata(ApiKeyTests.randomMetadata()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)) .get(); assertEquals("key-1", response.getName()); @@ -1624,13 +1630,19 @@ public void testDerivedKeys() throws ExecutionException, InterruptedException { final IllegalArgumentException e1 = expectThrows( IllegalArgumentException.class, - () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-2").setMetadata(ApiKeyTests.randomMetadata()).get() + () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-2") + .setMetadata(ApiKeyTests.randomMetadata()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)) + .get() ); assertThat(e1.getMessage(), containsString(expectedMessage)); final IllegalArgumentException e2 = expectThrows( IllegalArgumentException.class, - () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-3").setRoleDescriptors(Collections.emptyList()).get() + () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-3") + .setRoleDescriptors(Collections.emptyList()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)) + .get() ); assertThat(e2.getMessage(), containsString(expectedMessage)); @@ -1641,6 +1653,7 @@ public void testDerivedKeys() throws ExecutionException, InterruptedException { .setRoleDescriptors( Collections.singletonList(new RoleDescriptor("role", new String[] { "manage_own_api_key" }, null, null)) ) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)) .get() ); assertThat(e3.getMessage(), containsString(expectedMessage)); @@ -1656,6 +1669,7 @@ public void testDerivedKeys() throws ExecutionException, InterruptedException { () -> new CreateApiKeyRequestBuilder(clientKey1).setName("key-5") .setMetadata(ApiKeyTests.randomMetadata()) .setRoleDescriptors(roleDescriptors) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)) .get() ); assertThat(e4.getMessage(), containsString(expectedMessage)); @@ -1663,6 +1677,7 @@ public void testDerivedKeys() throws ExecutionException, InterruptedException { final CreateApiKeyResponse key100Response = new CreateApiKeyRequestBuilder(clientKey1).setName("key-100") .setMetadata(ApiKeyTests.randomMetadata()) .setRoleDescriptors(Collections.singletonList(new RoleDescriptor("role", null, null, null))) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)) .get(); assertEquals("key-100", key100Response.getName()); assertNotNull(key100Response.getId()); @@ -1696,6 +1711,7 @@ public void testApiKeyRunAsAnotherUserCanCreateApiKey() { final CreateApiKeyResponse response1 = new CreateApiKeyRequestBuilder(client).setName("run-as-key") .setRoleDescriptors(List.of(descriptor)) .setMetadata(ApiKeyTests.randomMetadata()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)) .get(); final String base64ApiKeyKeyValue = Base64.getEncoder() @@ -1705,7 +1721,10 @@ public void testApiKeyRunAsAnotherUserCanCreateApiKey() { client().filterWithHeader( Map.of("Authorization", "ApiKey " + base64ApiKeyKeyValue, "es-security-runas-user", ES_TEST_ROOT_USER) ) - ).setName("create-by run-as user").setRoleDescriptors(List.of(new RoleDescriptor("a", new String[] { "all" }, null, null))).get(); + ).setName("create-by run-as user") + .setRoleDescriptors(List.of(new RoleDescriptor("a", new String[] { "all" }, null, null))) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)) + .get(); final GetApiKeyResponse getApiKeyResponse = client.execute( GetApiKeyAction.INSTANCE, @@ -1732,6 +1751,7 @@ public void testCreationAndAuthenticationReturns429WhenThreadPoolIsSaturated() t final CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyRequestBuilder(client).setName("auth only key") .setRoleDescriptors(Collections.singletonList(descriptor)) .setMetadata(ApiKeyTests.randomMetadata()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)) .get(); assertNotNull(createApiKeyResponse.getId()); @@ -2344,7 +2364,7 @@ public void testInvalidUpdateApiKeysScenarios() throws ExecutionException, Inter assertTrue(Instant.now().isAfter(dayBefore)); final var expirationDateUpdatedResponse = client().prepareUpdate(SECURITY_MAIN_ALIAS, apiKeyId) .setDoc("expiration_time", dayBefore.toEpochMilli()) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .setRefreshPolicy(IMMEDIATE) .get(); assertThat(expirationDateUpdatedResponse.getResult(), is(DocWriteResponse.Result.UPDATED)); } @@ -2846,6 +2866,7 @@ private Tuple createApiKeyAndAuthenticateWithIt() throws IOExcep final CreateApiKeyResponse createApiKeyResponse = new CreateApiKeyRequestBuilder(client).setName("test key") .setMetadata(ApiKeyTests.randomMetadata()) + .setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)) .get(); final String docId = createApiKeyResponse.getId(); authenticateWithApiKey(docId, createApiKeyResponse.getKey()); @@ -3087,7 +3108,7 @@ private Tuple, List>> createApiKe .setExpiration(expiration) .setRoleDescriptors(Collections.singletonList(descriptor)) .setMetadata(metadata) - .setRefreshPolicy(i == noOfApiKeys - 1 ? WriteRequest.RefreshPolicy.IMMEDIATE : WriteRequest.RefreshPolicy.NONE) + .setRefreshPolicy(i == noOfApiKeys - 1 ? randomFrom(IMMEDIATE, WAIT_UNTIL) : NONE) .get(); assertNotNull(response.getId()); assertNotNull(response.getKey()); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java index d8bcd88f85d71..d970ff034438c 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/TokenAuthIntegTests.java @@ -531,6 +531,10 @@ public void testRefreshingMultipleTimesFails() throws Exception { // We now have two documents, the original(now refreshed) token doc and the new one with the new access doc AtomicReference docId = new AtomicReference<>(); assertBusy(() -> { + // refresh to make sure the token docs are visible + Request refreshRequest = new Request(HttpPost.METHOD_NAME, SecuritySystemIndices.SECURITY_TOKENS_ALIAS + "/_refresh"); + refreshRequest.setOptions(SECURITY_REQUEST_OPTIONS); + getRestClient().performRequest(refreshRequest); Request searchRequest = new Request(HttpPost.METHOD_NAME, SecuritySystemIndices.SECURITY_TOKENS_ALIAS + "/_search"); searchRequest.setOptions(SECURITY_REQUEST_OPTIONS); searchRequest.setJsonEntity(""" diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java index 22d2686a744f1..b6834dd69524b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/apikey/ApiKeySingleNodeTests.java @@ -82,6 +82,9 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.test.SecuritySettingsSource.ES_TEST_ROOT_USER; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; @@ -115,14 +118,12 @@ protected boolean addMockHttpTransport() { } public void testQueryWithExpiredKeys() throws InterruptedException { - final String id1 = client().execute( - CreateApiKeyAction.INSTANCE, - new CreateApiKeyRequest("expired-shortly", null, TimeValue.timeValueMillis(1), null) - ).actionGet().getId(); - final String id2 = client().execute( - CreateApiKeyAction.INSTANCE, - new CreateApiKeyRequest("long-lived", null, TimeValue.timeValueDays(1), null) - ).actionGet().getId(); + CreateApiKeyRequest request1 = new CreateApiKeyRequest("expired-shortly", null, TimeValue.timeValueMillis(1), null); + request1.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)); + final String id1 = client().execute(CreateApiKeyAction.INSTANCE, request1).actionGet().getId(); + CreateApiKeyRequest request2 = new CreateApiKeyRequest("long-lived", null, TimeValue.timeValueDays(1), null); + request2.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)); + final String id2 = client().execute(CreateApiKeyAction.INSTANCE, request2).actionGet().getId(); Thread.sleep(10); // just to be 100% sure that the 1st key is expired when we search for it final QueryApiKeyRequest queryApiKeyRequest = new QueryApiKeyRequest( @@ -150,6 +151,7 @@ public void testCreatingApiKeyWithNoAccess() { grantApiKeyRequest.getGrant().setType("password"); grantApiKeyRequest.getGrant().setUsername(username); grantApiKeyRequest.getGrant().setPassword(password); + grantApiKeyRequest.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); grantApiKeyRequest.getApiKeyRequest().setName(randomAlphaOfLength(8)); grantApiKeyRequest.getApiKeyRequest() .setRoleDescriptors( @@ -213,9 +215,11 @@ public void testServiceAccountApiKey() throws IOException { createServiceAccountTokenRequest ).actionGet(); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLength(8), null, null); + createApiKeyRequest.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); final CreateApiKeyResponse createApiKeyResponse = client().filterWithHeader( Map.of("Authorization", "Bearer " + createServiceAccountTokenResponse.getValue()) - ).execute(CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest(randomAlphaOfLength(8), null, null)).actionGet(); + ).execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet(); final Map apiKeyDocument = getApiKeyDocument(createApiKeyResponse.getId()); @@ -237,16 +241,14 @@ public void testServiceAccountApiKey() throws IOException { } public void testGetApiKeyWorksForTheApiKeyItself() { - final String apiKeyName = randomAlphaOfLength(10); - final CreateApiKeyResponse createApiKeyResponse = client().execute( - CreateApiKeyAction.INSTANCE, - new CreateApiKeyRequest( - apiKeyName, - List.of(new RoleDescriptor("x", new String[] { "manage_own_api_key", "manage_token" }, null, null, null, null, null, null)), - null, - null - ) - ).actionGet(); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest( + randomAlphaOfLength(10), + List.of(new RoleDescriptor("x", new String[] { "manage_own_api_key", "manage_token" }, null, null, null, null, null, null)), + null, + null + ); + createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE)); + final CreateApiKeyResponse createApiKeyResponse = client().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet(); final String apiKeyId = createApiKeyResponse.getId(); final String base64ApiKeyKeyValue = Base64.getEncoder() @@ -360,6 +362,7 @@ public void testGrantApiKeyForUserWithRunAs() throws IOException { ).createTokenWithClientCredentialsGrant(); final GrantApiKeyRequest grantApiKeyRequest3 = new GrantApiKeyRequest(); grantApiKeyRequest3.getApiKeyRequest().setName("granted-api-key-must-not-have-chained-runas"); + grantApiKeyRequest3.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)); grantApiKeyRequest3.getGrant().setType("access_token"); grantApiKeyRequest3.getGrant().setAccessToken(new SecureString(oAuth2Token3.accessToken().toCharArray())); grantApiKeyRequest3.getGrant().setRunAsUsername("user2"); @@ -383,6 +386,7 @@ public void testGrantApiKeyForUserWithRunAs() throws IOException { securityClient.putRole(new RoleDescriptor("user1_role", new String[] { "manage_token" }, null, new String[] { "user2" })); final GrantApiKeyRequest grantApiKeyRequest4 = new GrantApiKeyRequest(); grantApiKeyRequest4.getApiKeyRequest().setName("granted-api-key-will-check-token-run-as-privilege"); + grantApiKeyRequest4.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)); grantApiKeyRequest4.getGrant().setType("access_token"); grantApiKeyRequest4.getGrant().setAccessToken(new SecureString(oAuth2Token4.accessToken().toCharArray())); final ElasticsearchStatusException e4 = expectThrows( @@ -400,10 +404,14 @@ public void testGrantApiKeyForUserWithRunAs() throws IOException { } public void testInvalidateApiKeyWillRecordTimestamp() { - final String apiKeyId = client().execute( - CreateApiKeyAction.INSTANCE, - new CreateApiKeyRequest(randomAlphaOfLengthBetween(3, 8), null, TimeValue.timeValueMillis(randomLongBetween(1, 1000)), null) - ).actionGet().getId(); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest( + randomAlphaOfLengthBetween(3, 8), + null, + TimeValue.timeValueMillis(randomLongBetween(1, 1000)), + null + ); + createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE)); + final String apiKeyId = client().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet().getId(); assertThat(getApiKeyDocument(apiKeyId).get("invalidation_time"), nullValue()); final long start = Instant.now().toEpochMilli(); @@ -433,6 +441,7 @@ public void testCreateCrossClusterApiKey() throws IOException { { "search": [ {"names": ["logs"]} ] }"""); + request.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL)); final PlainActionFuture future = new PlainActionFuture<>(); client().execute(CreateCrossClusterApiKeyAction.INSTANCE, request, future); @@ -535,6 +544,7 @@ public void testUpdateCrossClusterApiKey() throws IOException { { "search": [ {"names": ["logs"]} ] }"""); + createApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE)); final CreateApiKeyResponse createApiKeyResponse = client().execute(CreateCrossClusterApiKeyAction.INSTANCE, createApiKeyRequest) .actionGet(); final String apiKeyId = createApiKeyResponse.getId(); @@ -631,6 +641,7 @@ public void testCannotCreateDerivedCrossClusterApiKey() throws IOException { null ) ) + .setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)) .execute() .actionGet(); final String encoded = Base64.getEncoder() @@ -642,6 +653,7 @@ public void testCannotCreateDerivedCrossClusterApiKey() throws IOException { { "search": [ {"names": ["logs"]} ] }"""); + request.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)); final PlainActionFuture future = new PlainActionFuture<>(); client().filterWithHeader(Map.of("Authorization", "ApiKey " + encoded)) @@ -659,6 +671,7 @@ private GrantApiKeyRequest buildGrantApiKeyRequest(String username, SecureString final GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(); // randomly use either password or access token grant grantApiKeyRequest.getApiKeyRequest().setName("granted-api-key-for-" + username + "-runas-" + runAsUsername); + grantApiKeyRequest.setRefreshPolicy(randomFrom(WAIT_UNTIL, IMMEDIATE)); if (randomBoolean()) { grantApiKeyRequest.getApiKeyRequest() .setRoleDescriptors(List.of(new RoleDescriptor(randomAlphaOfLengthBetween(3, 8), new String[] { "monitor" }, null, null))); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreSingleNodeTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreSingleNodeTests.java index c565aa925f5aa..a36461b8d91f3 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreSingleNodeTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreSingleNodeTests.java @@ -46,6 +46,9 @@ import java.util.stream.Collectors; import static java.util.Collections.emptyMap; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD; import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING; @@ -139,6 +142,7 @@ public void testResolvePrivilegesWorkWhenExpensiveQueriesAreDisabled() throws IO if (randomBoolean()) { final var createApiKeyRequest = new CreateApiKeyRequest(); createApiKeyRequest.setName(randomAlphaOfLength(5)); + createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, IMMEDIATE, WAIT_UNTIL)); if (randomBoolean()) { createApiKeyRequest.setRoleDescriptors( List.of( diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/SecurityDomainIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/SecurityDomainIntegTests.java index eb79ca64f6c2b..544d86525a971 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/SecurityDomainIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/SecurityDomainIntegTests.java @@ -43,6 +43,9 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; import static org.hamcrest.Matchers.containsString; @@ -309,6 +312,7 @@ public void testTokenRefreshFailsForUsernameOutsideDomain() throws IOException { public void testDomainCaptureForApiKey() { final CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLengthBetween(3, 8), null, null); + createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)); final CreateApiKeyResponse createApiKeyResponse = client().filterWithHeader( Map.of("Authorization", basicAuthHeaderValue(RAC_USER_NAME, NATIVE_RAC_USER_PASSWORD.clone())) diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java index 71a4d7d9231cb..85b33b2ead61a 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/user/AnonymousUserIntegTests.java @@ -39,6 +39,9 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; @@ -107,10 +110,9 @@ public void testAnonymousViaHttp() throws Exception { } public void testAnonymousRoleShouldBeCaptureWhenCreatingApiKey() throws IOException { - final CreateApiKeyResponse createApiKeyResponse = client().execute( - CreateApiKeyAction.INSTANCE, - new CreateApiKeyRequest(randomAlphaOfLength(8), null, null) - ).actionGet(); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLength(8), null, null); + createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, WAIT_UNTIL, IMMEDIATE)); + final CreateApiKeyResponse createApiKeyResponse = client().execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet(); final Map apiKeyDocument = getApiKeyDocument(createApiKeyResponse.getId()); @@ -130,9 +132,11 @@ public void testAnonymousRoleShouldNotBeCapturedWhenCreatingApiKeyWithServiceAcc createServiceAccountTokenRequest ).actionGet(); + CreateApiKeyRequest createApiKeyRequest = new CreateApiKeyRequest(randomAlphaOfLength(8), null, null); + createApiKeyRequest.setRefreshPolicy(randomFrom(NONE, IMMEDIATE, WAIT_UNTIL)); final CreateApiKeyResponse createApiKeyResponse = client().filterWithHeader( Map.of("Authorization", "Bearer " + createServiceAccountTokenResponse.getValue()) - ).execute(CreateApiKeyAction.INSTANCE, new CreateApiKeyRequest(randomAlphaOfLength(8), null, null)).actionGet(); + ).execute(CreateApiKeyAction.INSTANCE, createApiKeyRequest).actionGet(); final Map apiKeyDocument = getApiKeyDocument(createApiKeyResponse.getId()); @@ -153,6 +157,7 @@ public void testGrantApiKeyForAnonymousUserTokenWithRunAsWillFail() throws IOExc final GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest(); grantApiKeyRequest.getApiKeyRequest().setName("granted-api-key-cannot-have-anonymous-user-token-with-run-as"); + grantApiKeyRequest.setRefreshPolicy(randomFrom(IMMEDIATE, WAIT_UNTIL, NONE)); grantApiKeyRequest.getGrant().setType("access_token"); grantApiKeyRequest.getGrant().setAccessToken(new SecureString(oAuth2Token.accessToken().toCharArray())); grantApiKeyRequest.getGrant().setRunAsUsername("test_user"); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index f10d3dc832e09..cdca07e368677 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -567,7 +567,7 @@ public Security(Settings settings) { this.settings = settings; // TODO this is wrong, we should only use the environment that is provided to createComponents this.enabled = XPackSettings.SECURITY_ENABLED.get(settings); - this.systemIndices = new SecuritySystemIndices(); + this.systemIndices = new SecuritySystemIndices(settings); this.nodeStartedListenable = new ListenableFuture<>(); if (enabled) { runStartupChecks(settings); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 3753e8d81ef69..830df19c7343e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -36,6 +36,7 @@ import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; @@ -563,7 +564,7 @@ private void updateApiKeys( } logger.trace("Executing bulk request to update [{}] API keys", bulkRequestBuilder.numberOfActions()); - bulkRequestBuilder.setRefreshPolicy(RefreshPolicy.WAIT_UNTIL); + bulkRequestBuilder.setRefreshPolicy(defaultCreateDocRefreshPolicy(settings)); securityIndex.prepareIndexIfNeededThenExecute( ex -> listener.onFailure(traceLog("prepare security index before update", ex)), () -> executeAsyncWithOrigin( @@ -1677,7 +1678,7 @@ private void indexInvalidation(Collection apiKeyIds, ActionListener listener.onFailure(traceLog("prepare security index", ex)), () -> executeAsyncWithOrigin( @@ -2324,6 +2325,18 @@ public ApiKeyDoc toApiKeyDoc(BytesReference roleDescriptorsBytes, BytesReference } } + /** + * API Key documents are refreshed after creation, such that the API Key docs are visible in searches after the create-API-key + * endpoint returns. + * In stateful deployments, the automatic refresh interval is short (hard-coded to 1 sec), so the {@code RefreshPolicy#WAIT_UNTIL} + * is an acceptable tradeoff for the superior doc creation throughput compared to {@code RefreshPolicy#IMMEDIATE}. + * But in stateless the automatic refresh interval is too long (at least 10 sec), which translates to long create-API-key endpoint + * latency, so in this case we opt for {@code RefreshPolicy#IMMEDIATE} and acknowledge the lower maximum doc creation throughput. + */ + public static RefreshPolicy defaultCreateDocRefreshPolicy(Settings settings) { + return DiscoveryNode.isStateless(settings) ? RefreshPolicy.IMMEDIATE : RefreshPolicy.WAIT_UNTIL; + } + private static final class ApiKeyDocCache { private final Cache docCache; private final Cache roleDescriptorsBytesCache; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java index 832e730786629..a85c0496bfe08 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoAction; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoRequest; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.UUIDs; @@ -192,6 +193,7 @@ private void assembleToken(EnrollmentTokenType enrollTokenType, HttpInfo httpInf List.of(new RoleDescriptor("create_enrollment_token", new String[] { enrollTokenType.toString() }, null, null)), TimeValue.timeValueMinutes(ENROLL_API_KEY_EXPIRATION_MINUTES) ); + apiKeyRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); client.execute(CreateApiKeyAction.INSTANCE, apiKeyRequest, ActionListener.wrap(createApiKeyResponse -> { final String apiKey = createApiKeyResponse.getId() + ":" + createApiKeyResponse.getKey().toString(); final EnrollmentToken enrollmentToken = new EnrollmentToken(apiKey, fingerprint, Version.CURRENT.toString(), tokenAddresses); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java index 7d4ad8f2ebdeb..2cb5a15f1e0f2 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateApiKeyAction.java @@ -15,8 +15,8 @@ import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyRequestBuilder; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import java.io.IOException; import java.util.List; @@ -51,14 +51,16 @@ public String getName() { @Override protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { - String refresh = request.param("refresh"); CreateApiKeyRequestBuilder builder = new CreateApiKeyRequestBuilder(client).source( request.requiredContent(), request.getXContentType() - ) - .setRefreshPolicy( - (refresh != null) ? WriteRequest.RefreshPolicy.parse(request.param("refresh")) : CreateApiKeyRequest.DEFAULT_REFRESH_POLICY - ); + ); + String refresh = request.param("refresh"); + if (refresh != null) { + builder.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(request.param("refresh"))); + } else { + builder.setRefreshPolicy(ApiKeyService.defaultCreateDocRefreshPolicy(settings)); + } return channel -> builder.execute(new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateCrossClusterApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateCrossClusterApiKeyAction.java index 469571798680b..4c8ceb6bad2ec 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateCrossClusterApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestCreateCrossClusterApiKeyAction.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.CreateCrossClusterApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.CreateCrossClusterApiKeyRequest; import org.elasticsearch.xpack.core.security.action.apikey.CrossClusterApiKeyRoleDescriptorBuilder; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import java.io.IOException; import java.util.List; @@ -75,6 +76,7 @@ public String getName() { @Override protected RestChannelConsumer innerPrepareRequest(final RestRequest request, final NodeClient client) throws IOException { final CreateCrossClusterApiKeyRequest createCrossClusterApiKeyRequest = PARSER.parse(request.contentParser(), null); + createCrossClusterApiKeyRequest.setRefreshPolicy(ApiKeyService.defaultCreateDocRefreshPolicy(settings)); return channel -> client.execute( CreateCrossClusterApiKeyAction.INSTANCE, createCrossClusterApiKeyRequest, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java index 1758b9db47201..46d2fa4605f9d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestGrantApiKeyAction.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.core.security.action.apikey.CreateApiKeyResponse; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyAction; import org.elasticsearch.xpack.core.security.action.apikey.GrantApiKeyRequest; +import org.elasticsearch.xpack.security.authc.ApiKeyService; import java.io.IOException; import java.util.Arrays; @@ -94,6 +95,8 @@ protected RestChannelConsumer innerPrepareRequest(final RestRequest request, fin final GrantApiKeyRequest grantRequest = PARSER.parse(parser, null); if (refresh != null) { grantRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.parse(refresh)); + } else { + grantRequest.setRefreshPolicy(ApiKeyService.defaultCreateDocRefreshPolicy(settings)); } return channel -> client.execute( GrantApiKeyAction.INSTANCE, diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java index cb6d2a5318f5a..0f9f37b392529 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java @@ -12,8 +12,10 @@ import org.elasticsearch.Version; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.indices.ExecutorNames; import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.transport.TcpTransport; @@ -58,10 +60,10 @@ public class SecuritySystemIndices { private SecurityIndexManager tokenIndexManager; private SecurityIndexManager profileIndexManager; - public SecuritySystemIndices() { + public SecuritySystemIndices(Settings settings) { this.mainDescriptor = getSecurityMainIndexDescriptor(); this.tokenDescriptor = getSecurityTokenIndexDescriptor(); - this.profileDescriptor = getSecurityProfileIndexDescriptor(); + this.profileDescriptor = getSecurityProfileIndexDescriptor(settings); this.initialized = new AtomicBoolean(false); this.mainIndexManager = null; this.tokenIndexManager = null; @@ -778,13 +780,13 @@ private static XContentBuilder getTokenIndexMappings() { } } - private SystemIndexDescriptor getSecurityProfileIndexDescriptor() { + private SystemIndexDescriptor getSecurityProfileIndexDescriptor(Settings settings) { return SystemIndexDescriptor.builder() .setIndexPattern(".security-profile-[0-9]+*") .setPrimaryIndex(INTERNAL_SECURITY_PROFILE_INDEX_8) .setDescription("Contains user profile documents") .setMappings(getProfileIndexMappings()) - .setSettings(getProfileIndexSettings()) + .setSettings(getProfileIndexSettings(settings)) .setAliasName(SECURITY_PROFILE_ALIAS) .setIndexFormat(INTERNAL_PROFILE_INDEX_FORMAT) .setVersionMetaKey(SECURITY_VERSION_STRING) @@ -798,7 +800,7 @@ private SystemIndexDescriptor getSecurityProfileIndexDescriptor() { .setPrimaryIndex(INTERNAL_SECURITY_PROFILE_INDEX_8) .setDescription("Contains user profile documents") .setMappings(getProfileIndexMappings()) - .setSettings(getProfileIndexSettings()) + .setSettings(getProfileIndexSettings(settings)) .setAliasName(SECURITY_PROFILE_ALIAS) .setIndexFormat(INTERNAL_PROFILE_INDEX_FORMAT) .setVersionMetaKey(SECURITY_VERSION_STRING) @@ -810,8 +812,8 @@ private SystemIndexDescriptor getSecurityProfileIndexDescriptor() { .build(); } - private static Settings getProfileIndexSettings() { - return Settings.builder() + private static Settings getProfileIndexSettings(Settings settings) { + final Settings.Builder settingsBuilder = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-1") @@ -821,8 +823,15 @@ private static Settings getProfileIndexSettings() { .put("analysis.filter.email.preserve_original", true) .putList("analysis.filter.email.patterns", List.of("([^@]+)", "(\\p{L}+)", "(\\d+)", "@(.+)")) .put("analysis.analyzer.email.tokenizer", "uax_url_email") - .putList("analysis.analyzer.email.filter", List.of("email", "lowercase", "unique")) - .build(); + .putList("analysis.analyzer.email.filter", List.of("email", "lowercase", "unique")); + if (DiscoveryNode.isStateless(settings)) { + // The profiles functionality is intrinsically related to Kibana. Only Kibana uses this index (via dedicated APIs). + // Since the regular ".kibana" index is marked "fast_refresh", we opt to mark the user profiles index as "fast_refresh" too. + // This way the profiles index has the same availability and latency characteristics as the regular ".kibana" index, so APIs + // touching either of the two indices are more predictable. + settingsBuilder.put(IndexSettings.INDEX_FAST_REFRESH_SETTING.getKey(), true); + } + return settingsBuilder.build(); } private XContentBuilder getProfileIndexMappings() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java index 1ec4d1c9ef343..2369bf2d45be8 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/NativeRolesStoreTests.java @@ -306,7 +306,7 @@ public void testPutOfRoleWithFlsDlsUnlicensed() throws IOException { final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecuritySystemIndices systemIndices = new SecuritySystemIndices(); + final SecuritySystemIndices systemIndices = new SecuritySystemIndices(clusterService.getSettings()); systemIndices.init(client, clusterService); final SecurityIndexManager securityIndex = systemIndices.getMainIndexManager(); @@ -394,7 +394,7 @@ public void testPutRoleWithRemoteIndicesUnsupportedMinNodeVersion() { final XPackLicenseState licenseState = mock(XPackLicenseState.class); final AtomicBoolean methodCalled = new AtomicBoolean(false); - final SecuritySystemIndices systemIndices = new SecuritySystemIndices(); + final SecuritySystemIndices systemIndices = new SecuritySystemIndices(clusterService.getSettings()); systemIndices.init(client, clusterService); final SecurityIndexManager securityIndex = systemIndices.getMainIndexManager(); @@ -441,6 +441,7 @@ void innerPutRole(final PutRoleRequest request, final RoleDescriptor role, final private ClusterService mockClusterServiceWithMinNodeVersion(TransportVersion transportVersion) { final ClusterService clusterService = mock(ClusterService.class, Mockito.RETURNS_DEEP_STUBS); when(clusterService.state().getMinTransportVersion()).thenReturn(transportVersion); + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); return clusterService; } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java index 09d26c6d20ada..549ff09948cf0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/crossclusteraccess/CrossClusterAccessAuthenticationServiceIntegTests.java @@ -35,6 +35,9 @@ import java.util.concurrent.ExecutionException; import java.util.function.Consumer; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.NONE; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.WAIT_UNTIL; import static org.elasticsearch.xpack.core.security.authc.CrossClusterAccessSubjectInfo.CROSS_CLUSTER_ACCESS_SUBJECT_INFO_HEADER_KEY; import static org.elasticsearch.xpack.security.authc.CrossClusterAccessHeaders.CROSS_CLUSTER_ACCESS_CREDENTIALS_HEADER_KEY; import static org.hamcrest.Matchers.containsString; @@ -142,6 +145,7 @@ public void testInvalidHeaders() throws IOException { private String getEncodedCrossClusterAccessApiKey() throws IOException { final CreateCrossClusterApiKeyRequest request = CreateCrossClusterApiKeyRequest.withNameAndAccess("cross_cluster_access_key", """ {"search": [{"names": ["*"]}]}"""); + request.setRefreshPolicy(randomFrom(NONE, IMMEDIATE, WAIT_UNTIL)); final CreateApiKeyResponse response = client().execute(CreateCrossClusterApiKeyAction.INSTANCE, request).actionGet(); return ApiKeyService.withApiKeyPrefix( Base64.getEncoder().encodeToString((response.getId() + ":" + response.getKey()).getBytes(StandardCharsets.UTF_8)) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java index da6b64064de4f..1e0969c96c0de 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/SecurityIndexManagerTests.java @@ -103,7 +103,8 @@ protected void }; final ClusterService clusterService = mock(ClusterService.class); - final SystemIndexDescriptor descriptor = new SecuritySystemIndices().getSystemIndexDescriptors() + when(clusterService.getSettings()).thenReturn(Settings.EMPTY); + final SystemIndexDescriptor descriptor = new SecuritySystemIndices(clusterService.getSettings()).getSystemIndexDescriptors() .stream() .filter(d -> d.getAliasName().equals(SecuritySystemIndices.SECURITY_MAIN_ALIAS)) .findFirst()