From d3bc0356610dc41cfa8aedf4f9657ccfcf3917e2 Mon Sep 17 00:00:00 2001 From: Dmitri Bourlatchkov Date: Wed, 25 Dec 2024 21:14:55 -0500 Subject: [PATCH] Convert main integration tests into reusable backbox tests * Introduce new module: `integration-tests` to contain the test code for backbox integration tests * Refactors existing main integration tests to avoid relying on Dropwizard internals. * Add new test extension for managing the test environment with custom server managers discovered via `ServiceLoader` * Use Dropwizard test extentions for the reference implementation of `PolarisServerManager`. Custom server builds can provide their own environment-specific implementations. * Add helper classes for REST API (Management and Catalog) * Run reusable tests by extending them in the DW module. Using test suites is also possible, but it only uses one gradle test worker, so test parallelism is reduced. At the same time running tests in parallel withing the same JVM using JUnit5 threads currently leads to errors. --- dropwizard/service/build.gradle.kts | 3 + ...PolarisRestCatalogViewIntegrationTest.java | 188 --- .../DropwizardApplicationIntegrationTest.java | 23 + ...izardManagementServiceIntegrationTest.java | 24 + .../DropwizardRestCatalogIntegrationTest.java | 23 + ...zardRestCatalogViewAwsIntegrationTest.java | 24 + ...rdRestCatalogViewAzureIntegrationTest.java | 24 + ...ardRestCatalogViewFileIntegrationTest.java | 24 + ...zardRestCatalogViewGcpIntegrationTest.java | 24 + .../it/DropwizardServerManager.java | 117 ++ .../it/DropwizardSparkIntegrationTest.java | 23 + ...olaris.service.it.ext.PolarisServerManager | 20 + .../polaris-server-integrationtest.yml | 5 + gradle/projects.main.properties | 1 + integration-tests/build.gradle.kts | 67 + .../polaris/service/it/env/CatalogApi.java | 92 ++ .../service/it/env/ClientCredentials.java | 21 + .../polaris/service/it/env/IcebergHelper.java | 69 + .../polaris/service/it/env/ManagementApi.java | 256 ++++ .../service/it/env/PolarisApiEndpoints.java | 47 + .../polaris/service/it/env/PolarisClient.java | 98 ++ .../polaris/service/it/env/RestApi.java | 56 + .../apache/polaris/service/it/env/Server.java | 29 + .../ext/PolarisIntegrationTestExtension.java | 103 ++ .../service/it/ext/PolarisServerManager.java | 34 + .../PolarisApplicationIntegrationTest.java | 311 ++--- ...larisManagementServiceIntegrationTest.java | 1111 ++++++----------- .../PolarisRestCatalogIntegrationTest.java | 565 +++------ ...arisRestCatalogViewAwsIntegrationTest.java | 4 +- ...isRestCatalogViewAzureIntegrationTest.java | 4 +- ...risRestCatalogViewFileIntegrationTest.java | 4 +- ...arisRestCatalogViewGcpIntegrationTest.java | 4 +- ...PolarisRestCatalogViewIntegrationBase.java | 148 +++ .../it/test}/PolarisSparkIntegrationTest.java | 173 +-- 34 files changed, 2058 insertions(+), 1661 deletions(-) delete mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java create mode 100644 dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java create mode 100644 dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager create mode 100644 integration-tests/build.gradle.kts create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisApplicationIntegrationTest.java (70%) rename dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java => integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java (65%) rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisRestCatalogIntegrationTest.java (62%) rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisRestCatalogViewAwsIntegrationTest.java (94%) rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisRestCatalogViewAzureIntegrationTest.java (94%) rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisRestCatalogViewFileIntegrationTest.java (93%) rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisRestCatalogViewGcpIntegrationTest.java (94%) create mode 100644 integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java rename {dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog => integration-tests/src/main/java/org/apache/polaris/service/it/test}/PolarisSparkIntegrationTest.java (67%) diff --git a/dropwizard/service/build.gradle.kts b/dropwizard/service/build.gradle.kts index 294279946..21498a592 100644 --- a/dropwizard/service/build.gradle.kts +++ b/dropwizard/service/build.gradle.kts @@ -96,6 +96,7 @@ dependencies { compileOnly(libs.jakarta.annotation.api) compileOnly(libs.spotbugs.annotations) + testImplementation(project(":polaris-tests")) testImplementation(project(":polaris-api-management-model")) testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") @@ -140,6 +141,8 @@ tasks.named("test").configure { if (System.getenv("AWS_REGION") == null) { environment("AWS_REGION", "us-west-2") } + environment("POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_ID", "test-admin") + environment("POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_SECRET", "test-secret") jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") useJUnitPlatform() maxParallelForks = 4 diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java deleted file mode 100644 index 215288dbc..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.catalog; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Map; -import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.view.ViewCatalogTests; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironment; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog - * client. - */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - private RESTCatalog restCatalog; - - @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); - } - - @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm, - TestEnvironment testEnv) { - - Assumptions.assumeFalse(shouldSkip()); - - String userToken = adminToken.token(); - testInfo - .getTestMethod() - .ifPresent( - method -> { - String catalogName = method.getName() + testEnv.testId(); - try (Response response = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - if (response.getStatus() == Response.Status.OK.getStatusCode()) { - // Already exists! Must be in a parameterized test. - // Quick hack to get a unique catalogName. - // TODO: Have a while-loop instead with consecutive incrementing suffixes. - catalogName = catalogName + System.currentTimeMillis(); - } - } - - StorageConfigInfo storageConfig = getStorageConfigInfo(); - String defaultBaseLocation = - storageConfig.getAllowedLocations().getFirst() - + "/" - + System.getenv("USER") - + "/path/to/data"; - - org.apache.polaris.core.admin.model.CatalogProperties props = - org.apache.polaris.core.admin.model.CatalogProperties.builder(defaultBaseLocation) - .addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, - "file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), - "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - "true") - .build(); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(props) - .setStorageConfigInfo(storageConfig) - .build(); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminToken, - snowmanCredentials, - realm, - catalog, - Map.of()); - }); - } - - /** - * @return The catalog's storage config. - */ - protected abstract StorageConfigInfo getStorageConfigInfo(); - - /** - * @return Whether the tests should be skipped, for example due to environment variables not being - * specified. - */ - protected abstract boolean shouldSkip(); - - @Override - protected RESTCatalog catalog() { - return restCatalog; - } - - @Override - protected org.apache.iceberg.catalog.Catalog tableCatalog() { - return restCatalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } - - @Override - protected boolean supportsServerSideRetry() { - return true; - } - - @Override - protected boolean overridesRequestedLocation() { - return true; - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java new file mode 100644 index 000000000..6eeb25ae1 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; + +public class DropwizardApplicationIntegrationTest extends PolarisApplicationIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java new file mode 100644 index 000000000..894d9a74f --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; + +public class DropwizardManagementServiceIntegrationTest + extends PolarisManagementServiceIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java new file mode 100644 index 000000000..5443ef393 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogIntegrationTest; + +public class DropwizardRestCatalogIntegrationTest extends PolarisRestCatalogIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java new file mode 100644 index 000000000..b4cbf1d51 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAwsIntegrationTest; + +public class DropwizardRestCatalogViewAwsIntegrationTest + extends PolarisRestCatalogViewAwsIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java new file mode 100644 index 000000000..248877513 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAzureIntegrationTest; + +public class DropwizardRestCatalogViewAzureIntegrationTest + extends PolarisRestCatalogViewAzureIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java new file mode 100644 index 000000000..885ef4e1f --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; + +public class DropwizardRestCatalogViewFileIntegrationTest + extends PolarisRestCatalogViewFileIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java new file mode 100644 index 000000000..45cde359c --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewGcpIntegrationTest; + +public class DropwizardRestCatalogViewGcpIntegrationTest + extends PolarisRestCatalogViewGcpIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java new file mode 100644 index 000000000..0fcf3f1b2 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.polaris.service.dropwizard.PolarisApplication; +import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.Server; +import org.apache.polaris.service.it.ext.PolarisServerManager; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DropwizardServerManager implements PolarisServerManager { + // referenced in polaris-server-integrationtest.yml + public static final String TEST_REALM = "POLARIS"; + public static final String SERVER_CONFIG_PATH = "polaris-server-integrationtest.yml"; + + @Override + public Server serverForContext(ExtensionContext context) { + try { + return new Holder(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class Holder implements Server { + private final DropwizardAppExtension ext; + private final Path logDir; + + public Holder() throws IOException { + logDir = Files.createTempDirectory("polaris-dw-"); + Path logFile = logDir.resolve("application.log").toAbsolutePath(); + + Map config = new HashMap<>(); + // Bind to random port to support parallelism + config.put("server.applicationConnectors[0].port", "0"); + config.put("server.adminConnectors[0].port", "0"); + config.put("logging.appenders[1].type", "file"); + config.put("logging.appenders[1].currentLogFilename", logFile.toString()); + + ConfigOverride[] overrides = + config.entrySet().stream() + .map((e) -> ConfigOverride.config(e.getKey(), e.getValue())) + .toList() + .toArray(new ConfigOverride[0]); + ext = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath(SERVER_CONFIG_PATH), + overrides); + + try { + ext.before(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertThat(logFile) + .exists() + .content() + .hasSizeGreaterThan(0) + .doesNotContain("ERROR", "FATAL") + .contains("PolarisApplication: Server started successfully"); + } + + @Override + public String realmId() { + return TEST_REALM; + } + + @Override + public URI baseUri() { + return URI.create(String.format("http://localhost:%d", ext.getLocalPort())); + } + + @Override + public ClientCredentials adminCredentials() { + // These credentials are injected via env. variables from build scripts. + // Cf. POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_ID + return new ClientCredentials("test-admin", "test-secret", "root"); + } + + @Override + public void close() throws IOException { + ext.after(); + FileUtils.deleteDirectory(logDir.toFile()); + } + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java new file mode 100644 index 000000000..26fe3fb70 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest; + +public class DropwizardSparkIntegrationTest extends PolarisSparkIntegrationTest {} diff --git a/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 000000000..37adc23a1 --- /dev/null +++ b/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.dropwizard.it.DropwizardServerManager \ No newline at end of file diff --git a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml index 9d8f770e5..1f4f2117b 100644 --- a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml +++ b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml @@ -162,3 +162,8 @@ tokenBucketFactory: type: default requestsPerSecond: 9999 windowSeconds: 10 + +# Fake GCP credentials for tests that do not need access to storage +gcp_credentials: + access_token: "abc" + expires_in: 12345 \ No newline at end of file diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index bf7feec16..71490c3cb 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -26,5 +26,6 @@ polaris-service-common=service/common polaris-dropwizard-service=dropwizard/service polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model +polaris-tests=integration-tests aggregated-license-report=aggregated-license-report polaris-version=tools/version diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts new file mode 100644 index 000000000..ccafcbeac --- /dev/null +++ b/integration-tests/build.gradle.kts @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { id("polaris-server") } + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-model")) + + implementation(libs.jakarta.ws.rs.api) + implementation(libs.guava) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") + implementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") + + implementation(libs.hadoop.common) { + exclude("org.slf4j", "slf4j-reload4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + exclude("org.apache.zookeeper", "zookeeper") + } + + implementation(libs.auth0.jwt) + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + implementation("org.apache.iceberg:iceberg-spark-3.5_2.12") + implementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") + implementation("org.apache.spark:spark-sql_2.12:3.5.1") { + // exclude log4j dependencies + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-api") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + } + + implementation(platform(libs.junit.bom)) + implementation("org.junit.jupiter:junit-jupiter") + compileOnly("org.junit.jupiter:junit-jupiter-engine") + implementation(libs.assertj.core) + implementation(libs.mockito.core) +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java new file mode 100644 index 000000000..94feee5f0 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; + +public class CatalogApi extends RestApi { + CatalogApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, endpoints, authToken, uri); + } + + public String obtainToken(ClientCredentials credentials) { + try (Response response = + request("v1/oauth/tokens") + .post( + Entity.form( + new MultivaluedHashMap<>( + Map.of( + "grant_type", + "client_credentials", + "scope", + "PRINCIPAL_ROLE:ALL", + "client_id", + credentials.clientId(), + "client_secret", + credentials.clientSecret()))))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + return response.readEntity(OAuthTokenResponse.class).token(); + } + } + + public void createNamespace(String catalogName, String namespaceName) { + try (Response response = + request("v1/{cat}/namespaces", Map.of("cat", catalogName)) + .post( + Entity.json( + CreateNamespaceRequest.builder() + .withNamespace(Namespace.of(namespaceName)) + .build()))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + } + } + + public List listNamespaces(String catalog) { + try (Response response = request("v1/{cat}/namespaces", Map.of("cat", catalog)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + ListNamespacesResponse res = response.readEntity(ListNamespacesResponse.class); + return res.namespaces(); + } + } + + public void deleteNamespaces(String catalog, Namespace namespace) { + try (Response response = + request( + "v1/{cat}/namespaces/{ns}", + Map.of("cat", catalog, "ns", RESTUtil.encodeNamespace(namespace))) + .delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java new file mode 100644 index 000000000..841cbc4a5 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +public record ClientCredentials(String clientId, String clientSecret, String principalName) {} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java new file mode 100644 index 000000000..020a12b27 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.apache.polaris.service.it.env.PolarisApiEndpoints.REALM_HEADER; +import static org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest.PRINCIPAL_ROLE_ALL; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.iceberg.catalog.SessionCatalog; +import org.apache.iceberg.rest.HTTPClient; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public final class IcebergHelper { + private IcebergHelper() {} + + public static RESTCatalog restCatalog( + PolarisApiEndpoints endpoints, + PrincipalWithCredentials credentials, + String catalog, + Map extraProperties) { + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + RESTCatalog restCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) + .build()); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put( + org.apache.iceberg.CatalogProperties.URI, endpoints.catalogApiEndpoint().toString()) + .put( + OAuth2Properties.CREDENTIAL, + credentials.getCredentials().getClientId() + + ":" + + credentials.getCredentials().getClientSecret()) + .put(OAuth2Properties.SCOPE, PRINCIPAL_ROLE_ALL) + .put( + org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO") + .put("warehouse", catalog) + .put("header." + REALM_HEADER, endpoints.realm()) + .putAll(extraProperties); + + restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + return restCatalog; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java new file mode 100644 index 000000000..46df2b168 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogGrant; +import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogRoles; +import org.apache.polaris.core.admin.model.Catalogs; +import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.GrantResources; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalRoles; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.Principals; +import org.apache.polaris.core.admin.model.UpdateCatalogRequest; + +public class ManagementApi extends RestApi { + ManagementApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, endpoints, authToken, uri); + } + + public PrincipalWithCredentials createPrincipalWithRole(String principalName, String roleName) { + PrincipalWithCredentials credentials = createPrincipal(principalName); + createPrincipalRole(roleName); + assignPrincipalRole(principalName, roleName); + return credentials; + } + + public PrincipalWithCredentials createPrincipal(String name) { + try (Response createPResponse = + request("v1/principals").post(Entity.json(new Principal(name)))) { + assertThat(createPResponse).returns(CREATED.getStatusCode(), Response::getStatus); + return createPResponse.readEntity(PrincipalWithCredentials.class); + } + } + + public void createPrincipalRole(String name) { + createPrincipalRole(new PrincipalRole(name)); + } + + public void createPrincipalRole(PrincipalRole role) { + try (Response createPrResponse = request("v1/principal-roles").post(Entity.json(role))) { + assertThat(createPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void assignPrincipalRole(String principalName, String roleName) { + try (Response assignPrResponse = + request("v1/principals/{prince}/principal-roles", Map.of("prince", principalName)) + .put(Entity.json(new GrantPrincipalRoleRequest(new PrincipalRole(roleName))))) { + assertThat(assignPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void createCatalogRole(String catalogName, String catalogRoleName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) + .post(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .put(Entity.json(grant))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void grantCatalogRoleToPrincipalRole( + String principalRoleName, String catalogName, CatalogRole catalogRole) { + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalogName)) + .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public GrantResources listGrants(String catalogName, String catalogRoleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .get()) { + assertThat(response).returns(OK.getStatusCode(), Response::getStatus); + return response.readEntity(GrantResources.class); + } + } + + public void createCatalog(String principalRoleName, Catalog catalog) { + createCatalog(catalog); + + // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS + String catalogRoleName = "custom-admin"; + createCatalogRole(catalog.getName(), catalogRoleName); + + CatalogGrant grantResource = + new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); + Map catalogVars = Map.of("cat", catalog.getName(), "role", catalogRoleName); + try (Response response = + request("v1/catalogs/{cat}/catalog-roles/{role}/grants", catalogVars) + .put(Entity.json(grantResource))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + + CatalogGrant grantAccessResource = + new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); + try (Response response = + request("v1/catalogs/{cat}/catalog-roles/{role}/grants", catalogVars) + .put(Entity.json(grantAccessResource))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + + // Assign this new CatalogRole to the service_admin PrincipalRole + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalog.getName())) + .put(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void createCatalog(Catalog catalog) { + try (Response response = request("v1/catalogs").post(Entity.json(catalog))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public Catalog getCatalog(String name) { + try (Response response = request("v1/catalogs/{name}", Map.of("name", name)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Catalog.class); + } + } + + public void updateCatalog(Catalog catalog, Map catalogProps) { + try (Response response = + request("v1/catalogs/{name}", Map.of("name", catalog.getName())) + .put( + Entity.json( + new UpdateCatalogRequest( + catalog.getEntityVersion(), + catalogProps, + catalog.getStorageConfigInfo())))) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + } + } + + public void deleteCatalog(String catalogName) { + try (Response response = request("v1/catalogs/{cat}", Map.of("cat", catalogName)).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public CatalogRole getCatalogRole(String catalogName, String roleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of("cat", catalogName, "role", roleName)) + .get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(CatalogRole.class); + } + } + + public List listCatalogRoles(String catalogName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(CatalogRoles.class).getRoles(); + } + } + + public List listPrincipals() { + try (Response response = request("v1/principals").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Principals.class).getPrincipals(); + } + } + + public List listPrincipalRoles() { + try (Response response = request("v1/principal-roles").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(PrincipalRoles.class).getRoles(); + } + } + + public List listCatalogs() { + try (Response response = request("v1/catalogs").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Catalogs.class).getCatalogs(); + } + } + + public void deleteCatalogRole(String catalogName, CatalogRole role) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of("cat", catalogName, "role", role.getName())) + .delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void deletePrincipal(Principal principal) { + try (Response response = + request("v1/principals/{name}", Map.of("name", principal.getName())).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void deletePrincipalRole(PrincipalRole role) { + try (Response response = + request("v1/principal-roles/{name}", Map.of("name", role.getName())).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java new file mode 100644 index 000000000..e1233aee5 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.io.Serializable; +import java.net.URI; + +public final class PolarisApiEndpoints implements Serializable { + + public static String REALM_HEADER = "realm"; + + private final URI baseUri; + private final String realm; + + public PolarisApiEndpoints(URI baseUri, String realm) { + this.baseUri = baseUri; + this.realm = realm; + } + + public URI catalogApiEndpoint() { + return baseUri.resolve("api/catalog"); + } + + public URI managementApiEndpoint() { + return baseUri.resolve("api/management"); + } + + public String realm() { + return realm; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java new file mode 100644 index 000000000..1373c0d21 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static java.util.concurrent.TimeUnit.MINUTES; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.jakarta.rs.json.JacksonJsonProvider; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public final class PolarisClient implements AutoCloseable { + private final PolarisApiEndpoints endpoints; + private final Client client; + + private PolarisClient(PolarisApiEndpoints endpoints) { + this.endpoints = endpoints; + + ObjectMapper mapper = new ObjectMapper(); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(mapper); + + this.client = + ClientBuilder.newBuilder() + .readTimeout(5, MINUTES) + .connectTimeout(1, MINUTES) + .register(new JacksonJsonProvider(mapper)) + .build(); + } + + public static PolarisClient polarisClient(PolarisApiEndpoints endpoints) { + return new PolarisClient(endpoints); + } + + public ManagementApi managementApi(String authToken) { + return new ManagementApi(client, endpoints, authToken, endpoints.managementApiEndpoint()); + } + + public ManagementApi managementApi(ClientCredentials credentials) { + return managementApi(obtainToken(credentials)); + } + + public ManagementApi managementApi(PrincipalWithCredentials principal) { + return managementApi(obtainToken(principal)); + } + + public CatalogApi catalogApi(PrincipalWithCredentials principal) { + return new CatalogApi( + client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); + } + + public CatalogApi catalogApi(ClientCredentials credentials) { + return new CatalogApi( + client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + } + + public String obtainToken(PrincipalWithCredentials principal) { + return obtainToken( + new ClientCredentials( + principal.getCredentials().getClientId(), + principal.getCredentials().getClientSecret(), + "dummy-principal")); + } + + public String obtainToken(ClientCredentials credentials) { + CatalogApi anon = new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint()); + return anon.obtainToken(credentials); + } + + @Override + public void close() throws Exception { + client.close(); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java new file mode 100644 index 000000000..89abcb99e --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; +import java.net.URI; +import java.util.Map; + +public class RestApi { + private final Client client; + private final PolarisApiEndpoints endpoints; + private final String authToken; + private final URI uri; + + RestApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + this.client = client; + this.endpoints = endpoints; + this.authToken = authToken; + this.uri = uri; + } + + public Invocation.Builder request(String path) { + return request(path, Map.of()); + } + + public Invocation.Builder request(String path, Map templateValues) { + WebTarget target = client.target(uri).path(path); + for (Map.Entry entry : templateValues.entrySet()) { + target = target.resolveTemplate(entry.getKey(), entry.getValue()); + } + Invocation.Builder request = target.request("application/json"); + request = request.header(PolarisApiEndpoints.REALM_HEADER, endpoints.realm()); + if (authToken != null) { + request = request.header("Authorization", "Bearer " + authToken); + } + return request; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java new file mode 100644 index 000000000..1a16abbc1 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.net.URI; + +public interface Server extends AutoCloseable { + String realmId(); + + URI baseUri(); + + ClientCredentials adminCredentials(); +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java new file mode 100644 index 000000000..1432fd44f --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import java.util.ServiceLoader; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ExtensionContext.Store.CloseableResource; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.platform.engine.UniqueId; + +public class PolarisIntegrationTestExtension implements ParameterResolver { + private static final Namespace NAMESPACE = + Namespace.create(PolarisIntegrationTestExtension.class); + + private static final PolarisServerManager manager = + ServiceLoader.load(PolarisServerManager.class) + .findFirst() + .orElseThrow(() -> new IllegalStateException("PolarisServerManager not found")); + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + Class type = parameterContext.getParameter().getType(); + return type.isAssignableFrom(PolarisApiEndpoints.class) + || type.isAssignableFrom(ClientCredentials.class); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + Env env = env(extensionContext); + Class type = parameterContext.getParameter().getType(); + if (type.isAssignableFrom(PolarisApiEndpoints.class)) { + return env.endpoints(); + } else if (type.isAssignableFrom(ClientCredentials.class)) { + return env.server.adminCredentials(); + } + throw new IllegalStateException("Unable to resolve parameter: " + parameterContext); + } + + private Env env(ExtensionContext context) { + ExtensionContext classCtx = classContext(context); + ExtensionContext.Store store = classCtx.getStore(NAMESPACE); + return store.getOrComputeIfAbsent( + Env.class, (key) -> new Env(manager.serverForContext(classCtx)), Env.class); + } + + private ExtensionContext classContext(ExtensionContext context) { + while (context.getParent().isPresent()) { + UniqueId id = UniqueId.parse(context.getUniqueId()); + if ("class".equals(id.getLastSegment().getType())) { + break; + } + + context = context.getParent().get(); + } + + return context; + } + + private static class Env implements CloseableResource { + private final Server server; + private final PolarisApiEndpoints endpoints; + + private Env(Server server) { + this.server = server; + this.endpoints = new PolarisApiEndpoints(server.baseUri(), server.realmId()); + } + + PolarisApiEndpoints endpoints() { + return endpoints; + } + + @Override + public void close() throws Throwable { + server.close(); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java new file mode 100644 index 000000000..3bdbb3688 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; + +public interface PolarisServerManager { + + /** + * Returns server connection parameters for the tests under the specified context. + * + *

Implementations may reuse the same server for multiple contexts or create a fresh one for + * each context. In any case, {@link Server#close()} will be invoked when the context provided as + * the argument to this call is closed. + */ + Server serverForContext(ExtensionContext context); +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java similarity index 70% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 651581382..c51d41674 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -16,31 +16,25 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; -import static org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse.RequestThrottlingErrorType.REQUEST_TOO_LARGE; +import static org.apache.polaris.service.it.env.PolarisApiEndpoints.REALM_HEADER; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.ProcessingException; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; @@ -81,14 +75,11 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.dropwizard.throttling.RequestThrottlingErrorResponse; -import org.assertj.core.api.Assertions; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.env.RestApi; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -96,120 +87,66 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public class PolarisApplicationIntegrationTest { - @TempDir private static Path tempDir; - private static final Supplier CURRENT_LOG = - () -> tempDir.resolve("application.log").toString(); - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); +@ExtendWith(PolarisIntegrationTestExtension.class) +public class PolarisApplicationIntegrationTest { public static final String PRINCIPAL_ROLE_NAME = "admin"; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0"), // Bind to random port to support parallelism - ConfigOverride.config("logging.appenders[1].type", "file"), - ConfigOverride.config("logging.appenders[1].currentLogFilename", CURRENT_LOG)); + public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; private static String userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; private static Path testDir; private static String realm; + private static RestApi managementApi; + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ClientCredentials clientCredentials; + @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String polarisRealm) + public static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) throws IOException { - realm = polarisRealm; - - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .contains("PolarisApplication: Server started successfully"); + endpoints = apiEndpoints; + client = polarisClient(endpoints); + realm = endpoints.realm(); + clientCredentials = credentials; testDir = Path.of("build/test_data/iceberg/" + realm); FileUtils.deleteQuietly(testDir.toFile()); Files.createDirectories(testDir); - PolarisApplicationIntegrationTest.userToken = userToken.token(); - PolarisApplicationIntegrationTest.snowmanCredentials = snowmanCredentials; + userToken = client.obtainToken(credentials); + managementApi = client.managementApi(credentials); PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); try (Response createPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principalRole))) { + managementApi.request("v1/principal-roles").post(Entity.json(principalRole))) { assertThat(createPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response assignPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principals/%s/principal-roles", - EXT.getLocalPort(), snowmanCredentials.identifier().principalName())) - .request("application/json") - .header("Authorization", "Bearer " + PolarisApplicationIntegrationTest.userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/principals/{name}/principal-roles", Map.of("name", credentials.principalName())) .put(Entity.json(principalRole))) { assertThat(assignPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - - assertZeroErrorsInApplicationLog(); } @AfterAll - public static void deletePrincipalRole() { - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s", - EXT.getLocalPort(), PRINCIPAL_ROLE_NAME)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + public static void deletePrincipalRole() throws Exception { + managementApi + .request("v1/principal-roles/{role}", Map.of("role", PRINCIPAL_ROLE_NAME)) .delete() .close(); - } - private static void assertZeroErrorsInApplicationLog() { - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .hasSizeGreaterThan(0) - .doesNotContain("ERROR", "FATAL"); + client.close(); } /** * Create a new catalog for each test case. Assign the snowman catalog-admin principal role the * admin role of the new catalog. - * - * @param testInfo */ @BeforeEach public void before(TestInfo testInfo) { @@ -264,40 +201,24 @@ private static void createCatalog( .setProperties(props) .setStorageConfigInfo(storageConfig) .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalog))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - EXT.getLocalPort(), - catalogName, - PolarisEntityConstants.getNameOfCatalogAdminRole())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of( + "cat", catalogName, "role", PolarisEntityConstants.getNameOfCatalogAdminRole())) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - EXT.getLocalPort(), principalRoleName, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/principal-roles/{prin-role}/catalog-roles/{cat}", + Map.of("cat", catalogName, "prin-role", principalRoleName)) .put(Entity.json(catalogRole))) { assertThat(assignResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -311,14 +232,14 @@ private static RESTSessionCatalog newSessionCatalog(String catalog) { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + endpoints.catalogApiEndpoint().toString(), OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", catalog, - "header." + REALM_PROPERTY_KEY, + "header." + REALM_HEADER, realm)); return sessionCatalog; } @@ -333,7 +254,7 @@ public void testIcebergListNamespaces() throws IOException { } @Test - public void testConfigureCatalogCaseSensitive() throws IOException { + public void testConfigureCatalogCaseSensitive() { assertThatThrownBy(() -> newSessionCatalog("TESTCONFIGURECATALOGCASESENSITIVE")) .isInstanceOf(RESTException.class) .hasMessage( @@ -397,7 +318,7 @@ public void testIcebergCreateNamespace() throws IOException { @Test public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -416,7 +337,7 @@ public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throw @Test public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -433,7 +354,7 @@ public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws @Test public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); @@ -460,56 +381,51 @@ public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws I @Test public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "Internal"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "Internal"; createCatalog(catalogName, Catalog.TypeEnum.INTERNAL, PRINCIPAL_ROLE_NAME); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); - try { - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties( - Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - } catch (BadRequestException e) { - LOGGER.info("Received expected exception {}", e.getMessage()); - } + assertThatThrownBy( + () -> + sessionCatalog + .buildTable( + sessionContext, + TableIdentifier.of(ns, "the_table"), + new Schema( + List.of( + Types.NestedField.of( + 1, false, "theField", Types.StringType.get())))) + .withSortOrder(SortOrder.unsorted()) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) + .create()) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Forbidden: Invalid locations"); + + assertThatThrownBy( + () -> + sessionCatalog + .buildTable( + sessionContext, + TableIdentifier.of(ns, "the_table"), + new Schema( + List.of( + Types.NestedField.of( + 1, false, "theField", Types.StringType.get())))) + .withSortOrder(SortOrder.unsorted()) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withProperties(Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) + .create()) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Forbidden: Invalid locations"); } } @Test public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, @@ -519,7 +435,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -556,7 +472,7 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws @Test public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, @@ -566,7 +482,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -609,7 +525,7 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO @Test public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, @@ -619,7 +535,7 @@ public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOEx .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -663,14 +579,14 @@ public void testWarehouseNotSpecified() throws IOException { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + endpoints.catalogApiEndpoint().toString(), OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", emptyEnvironmentVariable, - "header." + REALM_PROPERTY_KEY, + "header." + REALM_HEADER, realm))) .isInstanceOf(BadRequestException.class) .hasMessage("Malformed request: Please specify a warehouse"); @@ -679,12 +595,7 @@ public void testWarehouseNotSpecified() throws IOException { @Test public void testRequestHeaderTooLarge() { - Invocation.Builder request = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json"); + Invocation.Builder request = managementApi.request("v1/principal-roles"); // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely // exceeds the limit @@ -693,11 +604,7 @@ public void testRequestHeaderTooLarge() { } try { - try (Response response = - request - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(new PrincipalRole("r")))) { + try (Response response = request.post(Entity.json(new PrincipalRole("r")))) { assertThat(response) .returns( Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), @@ -715,36 +622,22 @@ public void testRequestBodyTooLarge() { // The size is set to be higher than the limit in polaris-server-integrationtest.yml Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(largeRequest)) { + try (Response response = managementApi.request("v1/principal-roles").post(largeRequest)) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) - .matches( - r -> - r.readEntity(RequestThrottlingErrorResponse.class) - .errorType() - .equals(REQUEST_TOO_LARGE)); + .extracting(r -> r.readEntity(Map.class)) + .extracting("error_type") + .isEqualTo("REQUEST_TOO_LARGE"); } } @Test public void testRefreshToken() throws IOException { - String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", EXT.getLocalPort()); + String path = endpoints.catalogApiEndpoint().resolve("v1/oauth/tokens").toString(); try (RESTClient client = - HTTPClient.builder(ImmutableMap.of()) - .withHeader(REALM_PROPERTY_KEY, realm) - .uri(path) - .build()) { + HTTPClient.builder(Map.of()).withHeader(REALM_HEADER, realm).uri(path).build()) { String credentialString = - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(); + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(); String expiredToken = JWT.create().withExpiresAt(Instant.EPOCH).sign(Algorithm.HMAC256("irrelevant-secret")); var authConfig = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java similarity index 65% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java index 2aa5bd6c4..4762332bf 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java @@ -16,25 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.it.test; -import static io.dropwizard.jackson.Jackson.newObjectMapper; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; +import static org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest.PRINCIPAL_ROLE_ALL; import static org.assertj.core.api.Assertions.assertThat; import com.auth0.jwt.JWT; import com.auth0.jwt.JWTCreator; import com.auth0.jwt.algorithms.Algorithm; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; import java.io.IOException; import java.time.Duration; @@ -45,12 +40,7 @@ import java.util.Map; import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.rest.RESTUtil; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.polaris.core.admin.model.AddGrantRequest; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; @@ -67,7 +57,6 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; @@ -76,7 +65,6 @@ import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalRoles; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.Principals; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; @@ -84,28 +72,22 @@ import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.slf4j.LoggerFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) -public class PolarisServiceImplIntegrationTest { +@ExtendWith(PolarisIntegrationTestExtension.class) +public class PolarisManagementServiceIntegrationTest { private static final int MAX_IDENTIFIER_LENGTH = 256; private static final String ISSUER_KEY = "polaris"; private static final String CLAIM_KEY_ACTIVE = "active"; @@ -113,147 +95,56 @@ public class PolarisServiceImplIntegrationTest { private static final String CLAIM_KEY_PRINCIPAL_ID = "principalId"; private static final String CLAIM_KEY_SCOPE = "scope"; - // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh - // slate on every test case; otherwise, leftover state from one test from failures will interfere - // with other test cases. - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("gcp_credentials.access_token", "abc"), - ConfigOverride.config("gcp_credentials.expires_in", "12345")); - private static String userToken; - private static String realm; - private static String clientId; + private static PolarisClient client; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; + private static ClientCredentials rootCredentials; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, - PolarisPrincipalSecrets adminSecrets, - @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - clientId = adminSecrets.getPrincipalClientId(); - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public static void setup(PolarisApiEndpoints endpoints, ClientCredentials credentials) { + client = polarisClient(endpoints); + managementApi = client.managementApi(credentials); + catalogApi = client.catalogApi(credentials); + rootCredentials = credentials; + } + + @AfterAll + public static void close() throws Exception { + client.close(); } @AfterEach public void tearDown() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - response - .readEntity(Catalogs.class) - .getCatalogs() - .forEach( - catalog -> { - // clean up the catalog before we try to drop it - - // delete all the namespaces - try (Response res = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list namespaces in catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - } else { - res.readEntity(ListNamespacesResponse.class) - .namespaces() - .forEach( - namespace -> { - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces/" - + RESTUtil.encodeNamespace(namespace)) - .delete() - .close(); - }); - } - } - - // delete all the catalog roles except catalog_admin - try (Response res = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list catalog roles for catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - return; - } - res.readEntity(CatalogRoles.class).getRoles().stream() - .filter(cr -> !cr.getName().equals("catalog_admin")) - .forEach( - cr -> - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles/" - + cr.getName()) - .delete() - .close()); - } - - Response deleteResponse = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalog.getName()) - .delete(); - if (deleteResponse.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to delete catalog {}: {}", - catalog.getName(), - deleteResponse.readEntity(String.class)); - } - deleteResponse.close(); - }); - } - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { - response.readEntity(Principals.class).getPrincipals().stream() - .filter( - principal -> - !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) - .forEach( - principal -> { - newRequest( - "http://localhost:%d/api/management/v1/principals/" + principal.getName()) - .delete() - .close(); - }); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { - response.readEntity(PrincipalRoles.class).getRoles().stream() - .filter( - principalRole -> - !principalRole - .getName() - .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) - .forEach( - principalRole -> { - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRole.getName()) - .delete() - .close(); - }); - } + managementApi + .listCatalogs() + .forEach( + catalog -> { + // clean up the catalog before we try to drop it + // delete all the namespaces + catalogApi + .listNamespaces(catalog.getName()) + .forEach(namespace -> catalogApi.deleteNamespaces(catalog.getName(), namespace)); + + // delete all the catalog roles except catalog_admin + managementApi.listCatalogRoles(catalog.getName()).stream() + .filter(cr -> !cr.getName().equals("catalog_admin")) + .forEach(role -> managementApi.deleteCatalogRole(catalog.getName(), role)); + + managementApi.deleteCatalog(catalog.getName()); + }); + + managementApi.listPrincipals().stream() + .filter( + principal -> !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) + .forEach(principal -> managementApi.deletePrincipal(principal)); + + managementApi.listPrincipalRoles().stream() + .filter( + principalRole -> + !principalRole + .getName() + .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) + .forEach(principalRole -> managementApi.deletePrincipalRole(principalRole)); } @Test @@ -284,7 +175,7 @@ public void testCatalogSerializing() throws IOException { @Test public void testListCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -299,23 +190,8 @@ public void testListCatalogs() { @Test public void testListCatalogsUnauthorized() { - Principal principal = new Principal("a_new_user"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { + PrincipalWithCredentials principal = managementApi.createPrincipal("a_new_user"); + try (Response response = client.managementApi(principal).request("v1/catalogs").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -323,7 +199,8 @@ public void testListCatalogsUnauthorized() { @Test public void testCreateCatalog() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + managementApi + .request("v1/catalogs") .post( Entity.json( "{\"catalog\":{\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"},\"storageConfigInfo\":{\"storageType\":\"S3\",\"roleArn\":\"arn:aws:iam::123456789012:role/my-role\",\"externalId\":\"externalId\",\"userArn\":\"userArn\",\"allowedLocations\":[\"s3://my-old-bucket/path/to/data\"]}}}"))) { @@ -331,8 +208,7 @@ public void testCreateCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -350,8 +226,6 @@ public void testCreateCatalogWithInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - ObjectMapper mapper = newObjectMapper(); - Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -359,12 +233,8 @@ public void testCreateCatalogWithInvalidName() { .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); } String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); @@ -386,16 +256,12 @@ public void testCreateCatalogWithInvalidName() { .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); assertThat(errorResponse.message()).contains("Invalid value:"); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); } } } @@ -419,12 +285,10 @@ public void testCreateCatalogWithAzureStorageConfig() { .setStorageConfigInfo(azureConfigInfo) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -452,12 +316,10 @@ public void testCreateCatalogWithGcpStorageConfig() { .setStorageConfigInfo(gcpConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -486,9 +348,7 @@ public void testCreateCatalogWithNullBaseLocation() { catalogNode.set("properties", mapper.createObjectNode()); ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -513,9 +373,7 @@ public void testCreateCatalogWithoutProperties() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -528,12 +386,11 @@ public void testCreateCatalogWithoutProperties() { } @Test - public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingException { + public void testCreateCatalogWithoutStorageConfig() { String catalogString = "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(catalogString))) { + managementApi.request("v1/catalogs").post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -546,11 +403,10 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti } @Test - public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { + public void testCreateCatalogWithUnparsableJson() { String catalogString = "{\"catalog\": {{\"bad data}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(catalogString))) { + managementApi.request("v1/catalogs").post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -563,7 +419,7 @@ public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException } @Test - public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonProcessingException { + public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() { AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -581,16 +437,13 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -607,10 +460,9 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro fetchedCatalog.getEntityVersion(), Map.of("foo", "bar"), null /* storageConfigIno */); // Successfully update - Catalog updatedCatalog = null; + Catalog updatedCatalog; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/" + catalogName).put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); updatedCatalog = response.readEntity(Catalog.class); @@ -642,10 +494,9 @@ public void testCreateExternalCatalog() { .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).get()) { + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog fetchedCatalog = response.readEntity(Catalog.class); assertThat(fetchedCatalog) @@ -661,8 +512,7 @@ public void testCreateExternalCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).delete()) { + try (Response response = managementApi.request("v1/catalogs/" + catalogName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -689,16 +539,14 @@ public void testCreateCatalogWithoutDefaultLocation() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } } @Test - public void serialization() throws JsonProcessingException { + public void serialization() { CatalogProperties properties = new CatalogProperties("s3://my-bucket/path/to/data"); ObjectMapper mapper = new ObjectMapper(); CatalogProperties translated = mapper.convertValue(properties, CatalogProperties.class); @@ -719,12 +567,11 @@ public void testCreateAndUpdateAzureCatalog() { .build(); // 200 Successful create - createCatalog(catalog); + managementApi.createCatalog(catalog); // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/myazurecatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -743,8 +590,7 @@ public void testCreateAndUpdateAzureCatalog() { Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), modifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(badUpdateRequest))) { + managementApi.request("v1/catalogs/myazurecatalog").put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -763,8 +609,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/myazurecatalog").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -774,8 +619,7 @@ public void testCreateAndUpdateAzureCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/myazurecatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -793,19 +637,17 @@ public void testCreateListUpdateAndDeleteCatalog() { .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -816,7 +658,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // Should list the catalog. - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -836,8 +678,7 @@ public void testCreateListUpdateAndDeleteCatalog() { Map.of("default-base-location", "s3://newbucket/"), invalidModifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(badUpdateRequest))) { + managementApi.request("v1/catalogs/mycatalog").put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -862,8 +703,7 @@ public void testCreateListUpdateAndDeleteCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/mycatalog").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -875,8 +715,7 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -885,19 +724,17 @@ public void testCreateListUpdateAndDeleteCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -910,23 +747,10 @@ public void testCreateListUpdateAndDeleteCatalog() { } } - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); - } - - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); - } - @Test public void testGetCatalogNotFound() { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } } @@ -944,9 +768,7 @@ public void testGetCatalogInvalidName() { for (String invalidCatalogName : invalidCatalogNames) { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + invalidCatalogName) - .get()) { + try (Response response = managementApi.request("v1/catalogs/" + invalidCatalogName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -967,7 +789,7 @@ public void testCatalogRoleInvalidName() { new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidCatalogRoleNames = @@ -980,9 +802,8 @@ public void testCatalogRoleInvalidName() { for (String invalidCatalogRoleName : invalidCatalogRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/" - + invalidCatalogRoleName) + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles/" + invalidCatalogRoleName) .get()) { assertThat(response) @@ -996,23 +817,8 @@ public void testCatalogRoleInvalidName() { @Test public void testListPrincipalsUnauthorized() { - Principal principal = new Principal("new_admin"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + PrincipalWithCredentials principal = managementApi.createPrincipal("new_admin"); + try (Response response = client.managementApi(principal).request("v1/principals").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -1025,40 +831,29 @@ public void testCreatePrincipalAndRotateCredentials() { .setProperties(Map.of("custom-tag", "foo")) .build(); - PrincipalWithCredentialsCredentials creds = null; - Principal returnedPrincipal = null; + PrincipalWithCredentials creds; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, true)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); + creds = response.readEntity(PrincipalWithCredentials.class); } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); - - String oldClientId = creds.getClientId(); - String oldSecret = creds.getClientSecret(); + assertThat(creds.getCredentials().getClientId()).isEqualTo(creds.getPrincipal().getClientId()); // Now rotate the credentials. First, if we try to just use the adminToken to rotate the // newly created principal's credentials, we should fail; rotateCredentials is only // a "self" privilege that even admins can't inherit. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal/rotate") - .post(Entity.json(""))) { + managementApi.request("v1/principals/myprincipal/rotate").post(Entity.json(""))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } - // Get a fresh token associate with the principal itself. - String newPrincipalToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), EXT.getLocalPort(), oldClientId, oldSecret, realm); + String oldUserToken = client.obtainToken(creds); // Any call should initially fail with error indicating that rotation is needed. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal", newPrincipalToken) - .get()) { + client.managementApi(oldUserToken).request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); assertThat(error) @@ -1069,21 +864,23 @@ public void testCreatePrincipalAndRotateCredentials() { } // Now try to rotate using the principal's token. + PrincipalWithCredentials newCreds; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal/rotate", - newPrincipalToken) + client + .managementApi(oldUserToken) + .request("v1/principals/myprincipal/rotate") .post(Entity.json(""))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); + newCreds = response.readEntity(PrincipalWithCredentials.class); } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); + assertThat(newCreds.getCredentials().getClientId()) + .isEqualTo(newCreds.getPrincipal().getClientId()); // ClientId shouldn't change - assertThat(creds.getClientId()).isEqualTo(oldClientId); - assertThat(creds.getClientSecret()).isNotEqualTo(oldSecret); + assertThat(newCreds.getCredentials().getClientId()) + .isEqualTo(creds.getCredentials().getClientId()); + assertThat(newCreds.getCredentials().getClientSecret()) + .isNotEqualTo(creds.getCredentials().getClientSecret()); // TODO: Test the validity of the old secret for getting tokens, here and then after a second // rotation that makes the old secret fall off retention. @@ -1097,22 +894,23 @@ public void testCreateListUpdateAndDeletePrincipal() { .setProperties(Map.of("custom-tag", "foo")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Principal fetchedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + Principal fetchedPrincipal; + try (Response response = managementApi.request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1122,7 +920,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // Should list the principal. - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = managementApi.request("v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1137,8 +935,7 @@ public void testCreateListUpdateAndDeletePrincipal() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/principals/myprincipal").put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1146,8 +943,7 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = managementApi.request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1155,19 +951,17 @@ public void testCreateListUpdateAndDeletePrincipal() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").delete()) { + try (Response response = managementApi.request("v1/principals/myprincipal").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + try (Response response = managementApi.request("v1/principals/myprincipal").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = managementApi.request("v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1186,7 +980,8 @@ public void testCreatePrincipalWithInvalidName() { .setProperties(Map.of("custom-tag", "good_principal")) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } @@ -1209,7 +1004,8 @@ public void testCreatePrincipalWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1233,8 +1029,7 @@ public void testGetPrincipalWithInvalidName() { for (String invalidPrincipalName : invalidPrincipalNames) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/" + invalidPrincipalName) - .get()) { + managementApi.request("v1/principals/" + invalidPrincipalName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1248,20 +1043,20 @@ public void testGetPrincipalWithInvalidName() { public void testCreateListUpdateAndDeletePrincipalRole() { PrincipalRole principalRole = new PrincipalRole("myprincipalrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + managementApi + .request("v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - PrincipalRole fetchedPrincipalRole = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + PrincipalRole fetchedPrincipalRole; + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1272,8 +1067,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // Should list the principalRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = managementApi.request("v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1289,7 +1083,8 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") + managementApi + .request("v1/principal-roles/myprincipalrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1298,8 +1093,7 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // 200 GET after update should show new properties - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1307,23 +1101,19 @@ public void testCreateListUpdateAndDeletePrincipalRole() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = managementApi.request("v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1339,7 +1129,7 @@ public void testCreatePrincipalRoleInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); PrincipalRole principalRole = new PrincipalRole(goodName, Map.of("custom-tag", "good_principal_role"), 0L, 0L, 1); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidPrincipalRoleNames = @@ -1357,7 +1147,8 @@ public void testCreatePrincipalRoleInvalidName() { invalidPrincipalRoleName, Map.of("custom-tag", "bad_principal_role"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + managementApi + .request("v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1381,10 +1172,7 @@ public void testGetPrincipalRoleInvalidName() { for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + invalidPrincipalRoleName) - .get()) { + managementApi.request("v1/principal-roles/" + invalidPrincipalRoleName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1405,7 +1193,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); Catalog catalog2 = PolarisCatalog.builder() @@ -1416,12 +1204,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://required/base/other_location")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); CatalogRole catalogRole = new CatalogRole("mycatalogrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1429,18 +1218,17 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - CatalogRole fetchedCatalogRole = null; + CatalogRole fetchedCatalogRole; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1451,9 +1239,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // Should list the catalogRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1464,9 +1250,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // Empty list if listing in catalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2/catalog-roles") - .get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog2/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1487,8 +1271,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful update try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1498,9 +1282,7 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 GET after update should show new properties try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1509,26 +1291,20 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 204 Successful delete try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .delete()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") - .get()) { + managementApi.request("v1/catalogs/mycatalog1/catalog-roles/mycatalogrole").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") - .get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog1/catalog-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1539,15 +1315,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1558,7 +1332,8 @@ public void testAssignListAndRevokePrincipalRoles() { // Create two Principals Principal principal1 = new Principal("myprincipal1"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal1, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1566,7 +1341,8 @@ public void testAssignListAndRevokePrincipalRoles() { Principal principal2 = new Principal("myprincipal2"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal2, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1574,11 +1350,12 @@ public void testAssignListAndRevokePrincipalRoles() { // One PrincipalRole PrincipalRole principalRole = new PrincipalRole("myprincipalrole"); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); // Assign the role to myprincipal1 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + managementApi + .request("v1/principals/myprincipal1/principal-roles") .put(Entity.json(principalRole))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1586,8 +1363,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + managementApi.request("v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1601,9 +1377,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Should list myprincipal1 if listing assignees of myprincipalrole try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + managementApi.request("v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1616,8 +1390,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list if listing in principal2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2/principal-roles") - .get()) { + managementApi.request("v1/principals/myprincipal2/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1627,8 +1400,8 @@ public void testAssignListAndRevokePrincipalRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") + managementApi + .request("v1/principals/myprincipal1/principal-roles/myprincipalrole") .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1636,8 +1409,7 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") - .get()) { + managementApi.request("v1/principals/myprincipal1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1645,9 +1417,7 @@ public void testAssignListAndRevokePrincipalRoles() { .returns(List.of(), PrincipalRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") - .get()) { + managementApi.request("v1/principal-roles/myprincipalrole/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1656,23 +1426,19 @@ public void testAssignListAndRevokePrincipalRoles() { } // 204 Successful delete myprincipal1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1").delete()) { + try (Response response = managementApi.request("v1/principals/myprincipal1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipal2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2").delete()) { + try (Response response = managementApi.request("v1/principals/myprincipal2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipalrole - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { + try (Response response = managementApi.request("v1/principal-roles/myprincipalrole").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1682,10 +1448,10 @@ public void testAssignListAndRevokePrincipalRoles() { public void testAssignListAndRevokeCatalogRoles() { // Create two PrincipalRoles PrincipalRole principalRole1 = new PrincipalRole("mypr1"); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); PrincipalRole principalRole2 = new PrincipalRole("mypr2"); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // One CatalogRole Catalog catalog = @@ -1697,11 +1463,12 @@ public void testAssignListAndRevokeCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); CatalogRole catalogRole = new CatalogRole("mycr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles") + managementApi + .request("v1/catalogs/mycatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1717,11 +1484,12 @@ public void testAssignListAndRevokeCatalogRoles() { new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(otherCatalog); + managementApi.createCatalog(otherCatalog); CatalogRole otherCatalogRole = new CatalogRole("myothercr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles") + managementApi + .request("v1/catalogs/othercatalog/catalog-roles") .post(Entity.json(new CreateCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1729,15 +1497,15 @@ public void testAssignListAndRevokeCatalogRoles() { // Assign both the roles to mypr1 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + managementApi + .request("v1/principal-roles/mypr1/catalog-roles/mycatalog") .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") + managementApi + .request("v1/principal-roles/mypr1/catalog-roles/othercatalog") .put(Entity.json(new GrantCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1745,9 +1513,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list only mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + managementApi.request("v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1760,9 +1526,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list mypr1 if listing assignees of mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") - .get()) { + managementApi.request("v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1775,9 +1539,7 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list if listing in principalRole2 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog") - .get()) { + managementApi.request("v1/principal-roles/mypr2/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1787,18 +1549,14 @@ public void testAssignListAndRevokeCatalogRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") - .delete()) { + managementApi.request("v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Empty list try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") - .get()) { + managementApi.request("v1/principal-roles/mypr1/catalog-roles/mycatalog").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1806,9 +1564,7 @@ public void testAssignListAndRevokeCatalogRoles() { .returns(List.of(), CatalogRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") - .get()) { + managementApi.request("v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) @@ -1817,46 +1573,39 @@ public void testAssignListAndRevokeCatalogRoles() { } // 204 Successful delete mypr1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr1").delete()) { + try (Response response = managementApi.request("v1/principal-roles/mypr1").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mypr2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr2").delete()) { + try (Response response = managementApi.request("v1/principal-roles/mypr2").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycr try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr") - .delete()) { + managementApi.request("v1/catalogs/mycatalog/catalog-roles/mycr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myothercr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr") - .delete()) { + managementApi.request("v1/catalogs/othercatalog/catalog-roles/myothercr").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete othercatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/othercatalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1867,8 +1616,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal // role String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRoleName); String catalogName = "myuniquetestcatalog"; Catalog catalog = @@ -1880,56 +1628,54 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); + PrincipalWithCredentials catalogAdminPrincipal = managementApi.createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRoleName); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String catalogAdminToken = client.obtainToken(catalogAdminPrincipal); // Create a second principal role. Use the catalog admin principal to list principal roles and // grant a catalog role to the new principal role String principalRoleName2 = "mypr2"; PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // create a catalog role and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName, catalogRoleName, catalogAdminToken); + client.managementApi(catalogAdminToken).createCatalogRole(catalogName, catalogRoleName); CatalogPrivilege privilege = CatalogPrivilege.CATALOG_MANAGE_CONTENT; - grantPrivilegeToCatalogRole( - catalogName, - catalogRoleName, - new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG), - catalogAdminToken, - Response.Status.CREATED); + client + .managementApi(catalogAdminToken) + .addGrant( + catalogName, + catalogRoleName, + new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG)); // The catalog admin can grant the new catalog role to the mypr2 principal role - grantCatalogRoleToPrincipalRole( - principalRoleName2, catalogName, new CatalogRole(catalogRoleName), catalogAdminToken); + client + .managementApi(catalogAdminToken) + .grantCatalogRoleToPrincipalRole( + principalRoleName2, catalogName, new CatalogRole(catalogRoleName)); // But the catalog admin cannot revoke the role because it requires // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + client + .managementApi(catalogAdminToken) + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" - + catalogRoleName, - catalogAdminToken) + + catalogRoleName) .delete()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } @@ -1937,14 +1683,14 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // The service admin can revoke the role because it has the // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE privilege try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + managementApi + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" - + catalogRoleName, - userToken) + + catalogRoleName) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1956,7 +1702,7 @@ public void testServiceAdminCanTransferCatalogAdmin() { // role String principalRoleName = "mypr33"; PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); String catalogName = "myothertestcatalog"; Catalog catalog = @@ -1968,31 +1714,26 @@ public void testServiceAdminCanTransferCatalogAdmin() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); + PrincipalWithCredentials catalogAdminPrincipal = managementApi.createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String catalogAdminToken = client.obtainToken(catalogAdminPrincipal); // service_admin revokes the catalog_admin privilege from its principal role try { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" + managementApi + .request( + "v1/principal-roles/service_admin/catalog-roles/" + catalogName - + "/catalog_admin", - userToken) + + "/catalog_admin") .delete()) { assertThat(response) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -2000,21 +1741,23 @@ public void testServiceAdminCanTransferCatalogAdmin() { // the service_admin can not revoke the catalog_admin privilege from the new principal role try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + client + .managementApi(catalogAdminToken) + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName - + "/catalog_admin", - catalogAdminToken) + + "/catalog_admin") .delete()) { assertThat(response) .returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } finally { // grant the admin role back to service_admin so that cleanup can happen - grantCatalogRoleToPrincipalRole( - "service_admin", catalogName, catalogAdminRole, catalogAdminToken); + client + .managementApi(catalogAdminToken) + .grantCatalogRoleToPrincipalRole("service_admin", catalogName, catalogAdminRole); } } @@ -2024,7 +1767,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // role String principalRoleName = "mypr33"; PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); // create a catalog String catalogName = "mytestcatalog"; @@ -2037,7 +1780,7 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a second catalog String catalogName2 = "anothercatalog"; @@ -2050,42 +1793,34 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); // create a catalog role *in the second catalog* and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, userToken); + managementApi.createCatalogRole(catalogName2, catalogRoleName); // Get the catalog admin role from the *first* catalog and grant that role to the principal role - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + PrincipalWithCredentials catalogAdminPrincipal = managementApi.createPrincipal("principal1"); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String catalogAdminToken = client.obtainToken(catalogAdminPrincipal); // Create a second principal role. String principalRoleName2 = "mypr2"; PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // The catalog admin cannot grant the new catalog role to the mypr2 principal role because the // catalog role is in the wrong catalog try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName2, - catalogAdminToken) + client + .managementApi(catalogAdminToken) + .request("v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName2) .put(Entity.json(new GrantCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } @@ -2096,7 +1831,7 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // Create a PrincipalRole and a new catalog. String principalRoleName = "mypr33"; PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); // create a catalog String catalogName = "mytablemanagecatalog"; @@ -2109,10 +1844,10 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", userToken); + managementApi.createCatalogRole(catalogName, "target_catalog_role"); // create a second catalog String catalogName2 = "anothertablemanagecatalog"; @@ -2125,59 +1860,53 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", userToken); + managementApi.createCatalogRole(catalogName2, "invalid_target_catalog_role"); // create the namespace "c" in *both* namespaces String namespaceName = "c"; - createNamespace(catalogName, namespaceName); - createNamespace(catalogName2, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName2, namespaceName); // create a catalog role *in the first catalog* and grant it manage_content privilege at the // namespace level // grant that role to the PrincipalRole String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, userToken); - grantPrivilegeToCatalogRole( + managementApi.createCatalogRole(catalogName, catalogRoleName); + managementApi.addGrant( catalogName, catalogRoleName, new NamespaceGrant( List.of(namespaceName), NamespacePrivilege.CATALOG_MANAGE_ACCESS, - GrantResource.TypeEnum.NAMESPACE), - userToken, - Response.Status.CREATED); + GrantResource.TypeEnum.NAMESPACE)); - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), userToken); + managementApi.grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, new CatalogRole(catalogRoleName)); // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + PrincipalWithCredentials catalogAdminPrincipal = + managementApi.createPrincipal("ns_manage_access_user"); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String manageAccessUserToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String manageAccessUserToken = client.obtainToken(catalogAdminPrincipal); // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target // catalog role is in // the same catalog - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.CREATED); + client + .managementApi(manageAccessUserToken) + .addGrant( + catalogName, + "target_catalog_role", + new NamespaceGrant( + List.of(namespaceName), + NamespacePrivilege.TABLE_CREATE, + GrantResource.TypeEnum.NAMESPACE)); // Even though the ns_manage_access_role can grant privileges to the catalog role, it cannot // grant the target @@ -2185,45 +1914,59 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // on the catalog role // as a securable try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - manageAccessUserToken) + client + .managementApi(manageAccessUserToken) + .request("v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName) .put( Entity.json(new GrantCatalogRoleRequest(new CatalogRole("target_catalog_role"))))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } // The user cannot grant catalog-level privileges to the catalog role - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + client + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", "target_catalog_role")) + .put( + Entity.json( + new CatalogGrant( + CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } // even though the namespace "c" exists in both catalogs, the ns_manage_access_role can only // grant privileges for // the namespace in its own catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + client + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName2, "role", "invalid_target_catalog_role")) + .put( + Entity.json( + new NamespaceGrant( + List.of(namespaceName), + NamespacePrivilege.TABLE_CREATE, + GrantResource.TypeEnum.NAMESPACE)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } // nor can it grant privileges to the catalog role in the second catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + client + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName2, "role", "invalid_target_catalog_role")) + .put( + Entity.json( + new CatalogGrant( + CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } } @Test @@ -2234,13 +1977,13 @@ public void testTokenExpiry() { .withExpiresAt(Instant.now().plus(1, ChronoUnit.SECONDS)) .sign(Algorithm.HMAC256("polaris")); Awaitility.await("expected list of records should be produced") - .atMost(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(20)) .pollDelay(Duration.ofSeconds(1)) .pollInterval(Duration.ofSeconds(1)) .untilAsserted( () -> { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2252,8 +1995,7 @@ public void testTokenInactive() { // InvalidClaimException - if a claim contained a different value than the expected one. String newToken = defaultJwt().withClaim(CLAIM_KEY_ACTIVE, false).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2263,8 +2005,7 @@ public void testTokenInactive() { public void testTokenInvalidSignature() { // SignatureVerificationException - if the signature is invalid. String newToken = defaultJwt().sign(Algorithm.HMAC256("invalid_secret")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2274,8 +2015,7 @@ public void testTokenInvalidSignature() { public void testTokenInvalidPrincipalId() { String newToken = defaultJwt().withClaim(CLAIM_KEY_PRINCIPAL_ID, 0).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2294,21 +2034,15 @@ public void testNamespaceExistsStatus() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a namespace String namespaceName = "c"; - createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); // check if a namespace existed try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) - .head()) { + catalogApi.request("v1/" + catalogName + "/namespaces/" + namespaceName).head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -2326,21 +2060,15 @@ public void testDropNamespaceStatus() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a namespace String namespaceName = "c"; - createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); // drop a namespace try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) - .delete()) { + catalogApi.request("v1/" + catalogName + "/namespaces/" + namespaceName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -2354,119 +2082,8 @@ public static JWTCreator.Builder defaultJwt() { .withExpiresAt(now.plus(10, ChronoUnit.SECONDS)) .withJWTId(UUID.randomUUID().toString()) .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) + .withClaim(CLAIM_KEY_CLIENT_ID, rootCredentials.clientId()) .withClaim(CLAIM_KEY_PRINCIPAL_ID, 1) - .withClaim(CLAIM_KEY_SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL); - } - - private static void createNamespace(String catalogName, String namespaceName) { - try (Response response = - newRequest("http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", userToken) - .post( - Entity.json( - CreateNamespaceRequest.builder() - .withNamespace(Namespace.of(namespaceName)) - .build()))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalog(Catalog catalog) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void grantPrivilegeToCatalogRole( - String catalogName, - String catalogRoleName, - GrantResource grant, - String catalogAdminToken, - Response.Status expectedStatus) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + catalogRoleName - + "/grants", - catalogAdminToken) - .put(Entity.json(new AddGrantRequest(grant)))) { - assertThat(response).returns(expectedStatus.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalogRole( - String catalogName, String catalogRoleName, String catalogAdminToken) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", - catalogAdminToken) - .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void grantPrincipalRoleToPrincipal( - String principalName, PrincipalRole principalRole) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/" - + principalName - + "/principal-roles") - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static PrincipalWithCredentials createPrincipal(String principalName) { - PrincipalWithCredentials catalogAdminPrincipal; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(new Principal(principalName), false)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - catalogAdminPrincipal = response.readEntity(PrincipalWithCredentials.class); - } - return catalogAdminPrincipal; - } - - private static void grantCatalogRoleToPrincipalRole( - String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - token) - .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static CatalogRole readCatalogRole(String catalogName, String roleName) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + roleName) - .get()) { - - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - return response.readEntity(CatalogRole.class); - } - } - - private static void createPrincipalRole(PrincipalRole principalRole1) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") - .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + .withClaim(CLAIM_KEY_SCOPE, PRINCIPAL_ROLE_ALL); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java similarity index 62% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 948f6d581..7413cc8bd 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -16,22 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,35 +61,31 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.GrantResources; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.TableGrant; import org.apache.polaris.core.admin.model.TablePrivilege; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.ViewGrant; import org.apache.polaris.core.admin.model.ViewPrivilege; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -103,12 +96,7 @@ * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisRestCatalogIntegrationTest extends CatalogTests { private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) @@ -116,22 +104,17 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String S3_BUCKET_BASE = Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) .orElse("file:///tmp/buckets/my-bucket"); - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; + private static String principalRoleName; + private static PrincipalWithCredentials principalCredentials; + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; private RESTCatalog restCatalog; private String currentCatalogName; - private String userToken; - private String realm; private final String catalogBaseLocation = S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; @@ -157,93 +140,79 @@ String[] properties() default { } @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + endpoints = apiEndpoints; + client = polarisClient(endpoints); + managementApi = client.managementApi(credentials); + String principalName = "snowman-rest-" + UUID.randomUUID(); + principalRoleName = "rest-admin-" + UUID.randomUUID(); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + catalogApi = client.catalogApi(principalCredentials); + } + + @AfterAll + static void close() throws Exception { + client.close(); } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - this.realm = realm; - userToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - snowmanCredentials.clientId(), - snowmanCredentials.clientSecret(), - realm); - testInfo - .getTestMethod() - .ifPresent( - method -> { - currentCatalogName = method.getName() + UUID.randomUUID(); - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn(TEST_ROLE_ARN) - .setExternalId("externalId") - .setUserArn("a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - Optional catalogConfig = - testInfo - .getTestMethod() - .flatMap(m -> Optional.ofNullable(m.getAnnotation(CatalogConfig.class))); - - org.apache.polaris.core.admin.model.CatalogProperties.Builder catalogPropsBuilder = - org.apache.polaris.core.admin.model.CatalogProperties.builder( - catalogBaseLocation); - String[] properties = - catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); - for (int i = 0; i < properties.length; i += 2) { - catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); - } - if (!S3_BUCKET_BASE.startsWith("file:/")) { - catalogPropsBuilder.addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); - } - Catalog catalog = - PolarisCatalog.builder() - .setType( - catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) - .setName(currentCatalogName) - .setProperties(catalogPropsBuilder.build()) - .setStorageConfigInfo( - S3_BUCKET_BASE.startsWith("file:/") - ? new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) - : awsConfigModel) - .build(); - - Optional restCatalogConfig = - testInfo - .getTestMethod() - .flatMap( - m -> - Optional.ofNullable( - m.getAnnotation( - PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); - ImmutableMap.Builder extraPropertiesBuilder = - ImmutableMap.builder(); - restCatalogConfig.ifPresent( - config -> { - for (int i = 0; i < config.value().length; i += 2) { - extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); - } - }); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - EXT, - adminToken, - snowmanCredentials, - realm, - catalog, - extraPropertiesBuilder.build()); - }); + public void before(TestInfo testInfo) { + Method method = testInfo.getTestMethod().orElseThrow(); + currentCatalogName = method.getName() + UUID.randomUUID(); + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn(TEST_ROLE_ARN) + .setExternalId("externalId") + .setUserArn("a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) + .build(); + Optional catalogConfig = + Optional.ofNullable(method.getAnnotation(CatalogConfig.class)); + + CatalogProperties.Builder catalogPropsBuilder = CatalogProperties.builder(catalogBaseLocation); + String[] properties = + catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); + for (int i = 0; i < properties.length; i += 2) { + catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); + } + if (!S3_BUCKET_BASE.startsWith("file:/")) { + catalogPropsBuilder.addProperty( + CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); + } + Catalog catalog = + PolarisCatalog.builder() + .setType(catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) + .setName(currentCatalogName) + .setProperties(catalogPropsBuilder.build()) + .setStorageConfigInfo( + S3_BUCKET_BASE.startsWith("file:/") + ? new FileStorageConfigInfo( + StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) + : awsConfigModel) + .build(); + + managementApi.createCatalog(principalRoleName, catalog); + + Optional restCatalogConfig = + testInfo + .getTestMethod() + .flatMap( + m -> + Optional.ofNullable( + m.getAnnotation( + PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); + ImmutableMap.Builder extraPropertiesBuilder = ImmutableMap.builder(); + restCatalogConfig.ifPresent( + config -> { + for (int i = 0; i < config.value().length; i += 2) { + extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); + } + }); + + restCatalog = + IcebergHelper.restCatalog( + endpoints, principalCredentials, currentCatalogName, extraPropertiesBuilder.build()); } @Override @@ -271,37 +240,6 @@ protected boolean overridesRequestedLocation() { return true; } - private void createCatalogRole(String catalogRoleName) { - CatalogRole catalogRole = new CatalogRole(catalogRoleName); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalogRole))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private void addGrant(String catalogRoleName, GrantResource grant) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, catalogRoleName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .put(Entity.json(grant))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - @Test public void testListGrantsOnCatalogObjectsToCatalogRoles() { restCatalog.createNamespace(Namespace.of("ns1")); @@ -396,8 +334,8 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { ViewPrivilege.VIEW_WRITE_PROPERTIES, GrantResource.TypeEnum.VIEW); - createCatalogRole("catalogrole1"); - createCatalogRole("catalogrole2"); + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + managementApi.createCatalogRole(currentCatalogName, "catalogrole2"); List role1Grants = List.of( @@ -409,7 +347,7 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { tableGrant2, viewGrant1, viewGrant2); - role1Grants.forEach(grant -> addGrant("catalogrole1", grant)); + role1Grants.forEach(grant -> managementApi.addGrant(currentCatalogName, "catalogrole1", grant)); List role2Grants = List.of( catalogGrant1, @@ -420,45 +358,19 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { tableGrant3, viewGrant1, viewGrant3); - role2Grants.forEach(grant -> addGrant("catalogrole2", grant)); + role2Grants.forEach(grant -> managementApi.addGrant(currentCatalogName, "catalogrole2", grant)); // List grants for catalogrole1 - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole1")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); // List grants for catalogrole2 - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole2")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole2")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); } @Test @@ -478,8 +390,8 @@ public void testListGrantsAfterRename() { TablePrivilege.TABLE_FULL_METADATA, GrantResource.TypeEnum.TABLE); - createCatalogRole("catalogrole1"); - addGrant("catalogrole1", tableGrant1); + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + managementApi.addGrant(currentCatalogName, "catalogrole1", tableGrant1); // Grants will follow the table through the rename restCatalog.renameTable( @@ -493,60 +405,19 @@ public void testListGrantsAfterRename() { TablePrivilege.TABLE_FULL_METADATA, GrantResource.TypeEnum.TABLE); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactly(expectedGrant); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole1")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactly(expectedGrant); } @Test - public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocation() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -569,41 +440,12 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { } @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( - PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -635,41 +477,12 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( } @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( - PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -790,27 +603,20 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled @Test public void testSendNotificationInternalCatalog() { - NotificationRequest notification = new NotificationRequest(); - notification.setNotificationType(NotificationType.CREATE); - notification.setPayload( - new TableUpdateNotification( - "tbl1", - System.currentTimeMillis(), - UUID.randomUUID().toString(), - "s3://my-bucket/path/to/metadata.json", - null)); + Map payload = + ImmutableMap.builder() + .put("table-name", "tbl1") + .put("timestamps", "" + System.currentTimeMillis()) + .put("table-uuid", UUID.randomUUID().toString()) + .put("metadata-location", "s3://my-bucket/path/to/metadata.json") + .build(); restCatalog.createNamespace(Namespace.of("ns1")); - String notificationUrl = - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - EXT.getLocalPort(), currentCatalogName); + Invocation.Builder notificationEndpoint = + catalogApi.request( + "v1/{cat}/namespaces/ns1/tables/tbl1/notifications", Map.of("cat", currentCatalogName)); try (Response response = - EXT.client() - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(notification))) { + notificationEndpoint.post( + Entity.json(Map.of("notification-type", "CREATE", "payload", payload)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(ErrorResponse.class)) @@ -818,14 +624,9 @@ public void testSendNotificationInternalCatalog() { } // NotificationType.VALIDATE should also surface the same error. - notification.setNotificationType(NotificationType.VALIDATE); try (Response response = - EXT.client() - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(notification))) { + notificationEndpoint.post( + Entity.json(Map.of("notification-type", "VALIDATE", "payload", payload)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(ErrorResponse.class)) @@ -1033,14 +834,10 @@ public void testTableExistsStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "table", tableName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1059,14 +856,10 @@ public void testDropTableStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "table", tableName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1093,14 +886,10 @@ public void testViewExistsStatus() { .create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1127,14 +916,10 @@ public void testDropViewStatus() { .create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1168,42 +953,28 @@ public void testRenameViewStatus() { // Perform view rename try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/views/rename", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request("v1/{cat}/views/rename", Map.of("cat", currentCatalogName)) .post(Entity.json(payload))) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Original view should no longer exists try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .head()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } - // New view should exists + // New view should exist try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, newViewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", newViewName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java index b7fdbee57..d6b7e39cb 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.Optional; @@ -27,7 +27,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on AWS. */ public class PolarisRestCatalogViewAwsIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) // Backward compatibility .orElse(System.getenv("INTEGRATION_TEST_S3_ROLE_ARN")); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java index be755526f..4a4eef984 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.stream.Stream; @@ -26,7 +26,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on Azure. */ public class PolarisRestCatalogViewAzureIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID"); public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_AZURE_PATH"); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java similarity index 93% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java index c7853df97..6b6bc5548 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; @@ -24,7 +24,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */ public class PolarisRestCatalogViewFileIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket"; @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java index b74677275..f1c4a762a 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.stream.Stream; @@ -26,7 +26,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on GCP. */ public class PolarisRestCatalogViewGcpIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String SERVICE_ACCOUNT = System.getenv("INTEGRATION_TEST_GCS_SERVICE_ACCOUNT"); public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_GCS_PATH"); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java new file mode 100644 index 000000000..c7b1fcbe2 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; + +import java.lang.reflect.Method; +import java.util.Map; +import java.util.UUID; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.view.ViewCatalogTests; +import org.apache.polaris.core.PolarisConfiguration; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog + * client. + */ +@ExtendWith(PolarisIntegrationTestExtension.class) +public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogTests { + + private static String principalRoleName; + private static PrincipalWithCredentials principalCredentials; + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ManagementApi managementApi; + + private RESTCatalog restCatalog; + + @BeforeAll + static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + endpoints = apiEndpoints; + client = polarisClient(endpoints); + managementApi = client.managementApi(credentials); + String principalName = "snowman-rest-" + UUID.randomUUID(); + principalRoleName = "rest-admin-" + UUID.randomUUID(); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + } + + @AfterAll + static void close() throws Exception { + client.close(); + } + + @BeforeEach + public void before(TestInfo testInfo) { + + Assumptions.assumeFalse(shouldSkip()); + + Method method = testInfo.getTestMethod().orElseThrow(); + String catalogName = method.getName() + UUID.randomUUID(); + + StorageConfigInfo storageConfig = getStorageConfigInfo(); + String defaultBaseLocation = + storageConfig.getAllowedLocations().getFirst() + + "/" + + System.getenv("USER") + + "/path/to/data"; + + CatalogProperties props = + CatalogProperties.builder(defaultBaseLocation) + .addProperty( + CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:") + .addProperty(PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .build(); + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(props) + .setStorageConfigInfo(storageConfig) + .build(); + managementApi.createCatalog(principalRoleName, catalog); + + restCatalog = IcebergHelper.restCatalog(endpoints, principalCredentials, catalogName, Map.of()); + } + + /** + * @return The catalog's storage config. + */ + protected abstract StorageConfigInfo getStorageConfigInfo(); + + /** + * @return Whether the tests should be skipped, for example due to environment variables not being + * specified. + */ + protected abstract boolean shouldSkip(); + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected org.apache.iceberg.catalog.Catalog tableCatalog() { + return restCatalog; + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } + + @Override + protected boolean overridesRequestedLocation() { + return true; + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java similarity index 67% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java index 173e2bda6..a5bc28c6b 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java @@ -16,17 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import com.google.common.collect.ImmutableMap; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; @@ -41,14 +38,12 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; @@ -61,39 +56,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisSparkIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism public static final String CATALOG_NAME = "mycatalog"; public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; private static final S3MockContainer s3Container = new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - private static PolarisConnectionExtension.PolarisToken polarisToken; private static SparkSession spark; - private String realm; + private PolarisApiEndpoints endpoints; + private PolarisClient client; + private ManagementApi managementApi; + private CatalogApi catalogApi; + private String sparkToken; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken polarisToken, @PolarisRealm String realm) - throws IOException { + public static void setup() throws IOException { s3Container.start(); - PolarisSparkIntegrationTest.polarisToken = polarisToken; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); } @AfterAll @@ -102,8 +81,13 @@ public static void cleanup() { } @BeforeEach - public void before(@PolarisRealm String realm) { - this.realm = realm; + public void before(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + endpoints = apiEndpoints; + client = polarisClient(endpoints); + sparkToken = client.obtainToken(credentials); + managementApi = client.managementApi(credentials); + catalogApi = client.catalogApi(credentials); + AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -139,16 +123,7 @@ public void before(@PolarisRealm String realm) { .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + managementApi.createCatalog(catalog); CatalogProperties externalProps = new CatalogProperties("s3://my-bucket/path/to/data"); externalProps.putAll( @@ -177,16 +152,9 @@ public void before(@PolarisRealm String realm) { .setStorageConfigInfo(awsConfigModel) .setRemoteUrl("http://dummy_url") .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(externalCatalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + + managementApi.createCatalog(externalCatalog); + SparkSession.Builder sessionBuilder = SparkSession.builder() .master("local[1]") @@ -215,11 +183,11 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") .config( String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + EXT.getLocalPort() + "/api/catalog") + endpoints.catalogApiEndpoint().toString()) .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), polarisToken.token()) + .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), endpoints.realm()) + .config(String.format("spark.sql.catalog.%s.token", catalogName), sparkToken) .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") .config( String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") @@ -227,7 +195,7 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca } @AfterEach - public void after() { + public void after() throws Exception { cleanupCatalog(CATALOG_NAME); cleanupCatalog(EXTERNAL_CATALOG_NAME); try { @@ -237,6 +205,8 @@ public void after() { } catch (Exception e) { LoggerFactory.getLogger(getClass()).error("Unable to close spark session", e); } + + client.close(); } private void cleanupCatalog(String catalogName) { @@ -253,18 +223,8 @@ private void cleanupCatalog(String catalogName) { } onSpark("DROP NAMESPACE " + namespace.getString(0)); } - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + + managementApi.deleteCatalog(catalogName); } @Test @@ -303,16 +263,9 @@ public void testCreateAndUpdateExternalTable() { LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); try (Response registerResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/" - + EXTERNAL_CATALOG_NAME - + "/namespaces/externalns1/register", - EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/externalns1/register", Map.of("cat", EXTERNAL_CATALOG_NAME)) .post( Entity.json( ImmutableRegisterTableRequest.builder() @@ -333,28 +286,28 @@ public void testCreateAndUpdateExternalTable() { onSpark("INSERT INTO " + CATALOG_NAME + ".ns1.tb1 VALUES (20, 'new_text')"); tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); - TableUpdateNotification updateNotification = - new TableUpdateNotification( - "mytb1", - Instant.now().toEpochMilli(), - tableResponse.tableMetadata().uuid(), - tableResponse.metadataLocation(), - tableResponse.tableMetadata()); - NotificationRequest notificationRequest = new NotificationRequest(); - notificationRequest.setPayload(updateNotification); - notificationRequest.setNotificationType(NotificationType.UPDATE); + Map updateNotification = + ImmutableMap.builder() + .put("table-name", "mytb1") + .put("timestamp", "" + Instant.now().toEpochMilli()) + .put("table-uuid", tableResponse.tableMetadata().uuid()) + .put("metadata-location", tableResponse.metadataLocation()) + .put("metadata", tableResponse.tableMetadata()) + .build(); + Map notificationRequest = + ImmutableMap.builder() + .put("payload", updateNotification) + .put("notification-type", "UPDATE") + .build(); try (Response notifyResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - EXT.getLocalPort(), EXTERNAL_CATALOG_NAME)) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/externalns1/tables/mytb1/notifications", + Map.of("cat", EXTERNAL_CATALOG_NAME)) .post(Entity.json(notificationRequest))) { assertThat(notifyResponse) - .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); + .extracting(Response::getStatus) + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); } // refresh the table so it queries for the latest metadata.json onSpark("REFRESH TABLE mytb1"); @@ -378,14 +331,10 @@ public void testCreateView() { private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), catalog, namespace, table)) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", catalog, "ns", namespace, "table", table)) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); return response.readEntity(LoadTableResponse.class);