diff --git a/dropwizard/service/build.gradle.kts b/dropwizard/service/build.gradle.kts index 28d885dd6..f64d2ad88 100644 --- a/dropwizard/service/build.gradle.kts +++ b/dropwizard/service/build.gradle.kts @@ -97,6 +97,7 @@ dependencies { compileOnly(libs.jakarta.annotation.api) compileOnly(libs.spotbugs.annotations) + testImplementation(project(":polaris-tests")) testImplementation(project(":polaris-api-management-model")) testImplementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") @@ -141,6 +142,8 @@ tasks.named("test").configure { if (System.getenv("AWS_REGION") == null) { environment("AWS_REGION", "us-west-2") } + // Note: the test secrets are referenced in DropwizardServerManager + environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,root,test-admin,test-secret") jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") useJUnitPlatform() maxParallelForks = 4 diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java deleted file mode 100644 index 215288dbc..000000000 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewIntegrationTest.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.dropwizard.catalog; - -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; - -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.ws.rs.core.Response; -import java.io.IOException; -import java.util.Map; -import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.view.ViewCatalogTests; -import org.apache.polaris.core.PolarisConfiguration; -import org.apache.polaris.core.admin.model.Catalog; -import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.core.entity.CatalogEntity; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironment; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.ExtendWith; - -/** - * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog - * client. - */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) -public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - private RESTCatalog restCatalog; - - @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); - } - - @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm, - TestEnvironment testEnv) { - - Assumptions.assumeFalse(shouldSkip()); - - String userToken = adminToken.token(); - testInfo - .getTestMethod() - .ifPresent( - method -> { - String catalogName = method.getName() + testEnv.testId(); - try (Response response = - testEnv - .apiClient() - .target( - String.format( - "%s/api/management/v1/catalogs/%s", testEnv.baseUri(), catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - if (response.getStatus() == Response.Status.OK.getStatusCode()) { - // Already exists! Must be in a parameterized test. - // Quick hack to get a unique catalogName. - // TODO: Have a while-loop instead with consecutive incrementing suffixes. - catalogName = catalogName + System.currentTimeMillis(); - } - } - - StorageConfigInfo storageConfig = getStorageConfigInfo(); - String defaultBaseLocation = - storageConfig.getAllowedLocations().getFirst() - + "/" - + System.getenv("USER") - + "/path/to/data"; - - org.apache.polaris.core.admin.model.CatalogProperties props = - org.apache.polaris.core.admin.model.CatalogProperties.builder(defaultBaseLocation) - .addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, - "file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), - "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), - "true") - .build(); - Catalog catalog = - PolarisCatalog.builder() - .setType(Catalog.TypeEnum.INTERNAL) - .setName(catalogName) - .setProperties(props) - .setStorageConfigInfo(storageConfig) - .build(); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - testEnv.apiClient(), - testEnv.baseUri().toString(), - adminToken, - snowmanCredentials, - realm, - catalog, - Map.of()); - }); - } - - /** - * @return The catalog's storage config. - */ - protected abstract StorageConfigInfo getStorageConfigInfo(); - - /** - * @return Whether the tests should be skipped, for example due to environment variables not being - * specified. - */ - protected abstract boolean shouldSkip(); - - @Override - protected RESTCatalog catalog() { - return restCatalog; - } - - @Override - protected org.apache.iceberg.catalog.Catalog tableCatalog() { - return restCatalog; - } - - @Override - protected boolean requiresNamespaceCreate() { - return true; - } - - @Override - protected boolean supportsServerSideRetry() { - return true; - } - - @Override - protected boolean overridesRequestedLocation() { - return true; - } -} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java new file mode 100644 index 000000000..6eeb25ae1 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardApplicationIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; + +public class DropwizardApplicationIntegrationTest extends PolarisApplicationIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java new file mode 100644 index 000000000..894d9a74f --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardManagementServiceIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; + +public class DropwizardManagementServiceIntegrationTest + extends PolarisManagementServiceIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java new file mode 100644 index 000000000..5443ef393 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogIntegrationTest; + +public class DropwizardRestCatalogIntegrationTest extends PolarisRestCatalogIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java new file mode 100644 index 000000000..b4cbf1d51 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAwsIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAwsIntegrationTest; + +public class DropwizardRestCatalogViewAwsIntegrationTest + extends PolarisRestCatalogViewAwsIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java new file mode 100644 index 000000000..248877513 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewAzureIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewAzureIntegrationTest; + +public class DropwizardRestCatalogViewAzureIntegrationTest + extends PolarisRestCatalogViewAzureIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java new file mode 100644 index 000000000..885ef4e1f --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewFileIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; + +public class DropwizardRestCatalogViewFileIntegrationTest + extends PolarisRestCatalogViewFileIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java new file mode 100644 index 000000000..45cde359c --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardRestCatalogViewGcpIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisRestCatalogViewGcpIntegrationTest; + +public class DropwizardRestCatalogViewGcpIntegrationTest + extends PolarisRestCatalogViewGcpIntegrationTest {} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java new file mode 100644 index 000000000..3e1156844 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardServerManager.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.dropwizard.testing.ConfigOverride; +import io.dropwizard.testing.ResourceHelpers; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.apache.polaris.service.dropwizard.PolarisApplication; +import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.Server; +import org.apache.polaris.service.it.ext.PolarisServerManager; +import org.junit.jupiter.api.extension.ExtensionContext; + +public class DropwizardServerManager implements PolarisServerManager { + // referenced in polaris-server-integrationtest.yml + public static final String TEST_REALM = "POLARIS"; + public static final String SERVER_CONFIG_PATH = "polaris-server-integrationtest.yml"; + + @Override + public Server serverForContext(ExtensionContext context) { + try { + return new Holder(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static class Holder implements Server { + private final DropwizardAppExtension ext; + private final Path logDir; + + public Holder() throws IOException { + logDir = Files.createTempDirectory("polaris-dw-"); + Path logFile = logDir.resolve("application.log").toAbsolutePath(); + + Map config = new HashMap<>(); + // Bind to random port to support parallelism + config.put("server.applicationConnectors[0].port", "0"); + config.put("server.adminConnectors[0].port", "0"); + config.put("logging.appenders[1].type", "file"); + config.put("logging.appenders[1].currentLogFilename", logFile.toString()); + + ConfigOverride[] overrides = + config.entrySet().stream() + .map((e) -> ConfigOverride.config(e.getKey(), e.getValue())) + .toList() + .toArray(ConfigOverride[]::new); + ext = + new DropwizardAppExtension<>( + PolarisApplication.class, + ResourceHelpers.resourceFilePath(SERVER_CONFIG_PATH), + overrides); + + try { + ext.before(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + assertThat(logFile) + .exists() + .content() + .hasSizeGreaterThan(0) + .doesNotContain("ERROR", "FATAL") + .contains("PolarisApplication: Server started successfully"); + } + + @Override + public String realmId() { + return TEST_REALM; + } + + @Override + public URI baseUri() { + return URI.create(String.format("http://localhost:%d", ext.getLocalPort())); + } + + @Override + public ClientPrincipal adminCredentials() { + // These credentials are injected via env. variables from build scripts. + // Cf. POLARIS_BOOTSTRAP_POLARIS_ROOT_CLIENT_ID + return new ClientPrincipal("root", new ClientCredentials("test-admin", "test-secret")); + } + + @Override + public void close() throws IOException { + ext.after(); + FileUtils.deleteDirectory(logDir.toFile()); + } + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java new file mode 100644 index 000000000..26fe3fb70 --- /dev/null +++ b/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/it/DropwizardSparkIntegrationTest.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.dropwizard.it; + +import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest; + +public class DropwizardSparkIntegrationTest extends PolarisSparkIntegrationTest {} diff --git a/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 000000000..37adc23a1 --- /dev/null +++ b/dropwizard/service/src/test/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.dropwizard.it.DropwizardServerManager \ No newline at end of file diff --git a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml index 9d8f770e5..1f4f2117b 100644 --- a/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml +++ b/dropwizard/service/src/test/resources/polaris-server-integrationtest.yml @@ -162,3 +162,8 @@ tokenBucketFactory: type: default requestsPerSecond: 9999 windowSeconds: 10 + +# Fake GCP credentials for tests that do not need access to storage +gcp_credentials: + access_token: "abc" + expires_in: 12345 \ No newline at end of file diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index bf7feec16..71490c3cb 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -26,5 +26,6 @@ polaris-service-common=service/common polaris-dropwizard-service=dropwizard/service polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model +polaris-tests=integration-tests aggregated-license-report=aggregated-license-report polaris-version=tools/version diff --git a/integration-tests/build.gradle.kts b/integration-tests/build.gradle.kts new file mode 100644 index 000000000..5c39b8f33 --- /dev/null +++ b/integration-tests/build.gradle.kts @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { id("polaris-server") } + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-model")) + + implementation(libs.jakarta.ws.rs.api) + implementation(libs.guava) + + implementation(platform(libs.jackson.bom)) + implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider") + + implementation(platform(libs.iceberg.bom)) + implementation("org.apache.iceberg:iceberg-api") + implementation("org.apache.iceberg:iceberg-core") + + implementation("org.apache.iceberg:iceberg-api:${libs.versions.iceberg.get()}:tests") + implementation("org.apache.iceberg:iceberg-core:${libs.versions.iceberg.get()}:tests") + + implementation(libs.hadoop.common) { + exclude("org.slf4j", "slf4j-reload4j") + exclude("org.slf4j", "slf4j-log4j12") + exclude("ch.qos.reload4j", "reload4j") + exclude("log4j", "log4j") + exclude("org.apache.zookeeper", "zookeeper") + } + + implementation(libs.auth0.jwt) + + implementation(platform(libs.testcontainers.bom)) + implementation("org.testcontainers:testcontainers") + implementation(libs.s3mock.testcontainers) + + implementation("org.apache.iceberg:iceberg-spark-3.5_2.12") + implementation("org.apache.iceberg:iceberg-spark-extensions-3.5_2.12") + implementation("org.apache.spark:spark-sql_2.12:3.5.1") { + // exclude log4j dependencies + exclude("org.apache.logging.log4j", "log4j-slf4j2-impl") + exclude("org.apache.logging.log4j", "log4j-api") + exclude("org.apache.logging.log4j", "log4j-1.2-api") + exclude("org.slf4j", "jul-to-slf4j") + } + + implementation(platform(libs.junit.bom)) + implementation("org.junit.jupiter:junit-jupiter") + compileOnly("org.junit.jupiter:junit-jupiter-engine") + implementation(libs.assertj.core) + implementation(libs.mockito.core) +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java new file mode 100644 index 000000000..9f646d874 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/CatalogApi.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.MultivaluedHashMap; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.requests.CreateNamespaceRequest; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.OAuthTokenResponse; + +/** + * A simple, non-exhaustive set of helper methods for accessing the Iceberg REST API. + * + * @see PolarisClient#catalogApi(ClientCredentials) + */ +public class CatalogApi extends RestApi { + CatalogApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, endpoints, authToken, uri); + } + + public String obtainToken(ClientCredentials credentials) { + try (Response response = + request("v1/oauth/tokens") + .post( + Entity.form( + new MultivaluedHashMap<>( + Map.of( + "grant_type", + "client_credentials", + "scope", + "PRINCIPAL_ROLE:ALL", + "client_id", + credentials.clientId(), + "client_secret", + credentials.clientSecret()))))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + return response.readEntity(OAuthTokenResponse.class).token(); + } + } + + public void createNamespace(String catalogName, String namespaceName) { + try (Response response = + request("v1/{cat}/namespaces", Map.of("cat", catalogName)) + .post( + Entity.json( + CreateNamespaceRequest.builder() + .withNamespace(Namespace.of(namespaceName)) + .build()))) { + assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); + } + } + + public List listNamespaces(String catalog, Namespace parent) { + Map queryParams = new HashMap<>(); + if (!parent.isEmpty()) { + queryParams.put("parent", RESTUtil.encodeNamespace(parent)); + } + try (Response response = + request("v1/{cat}/namespaces", Map.of("cat", catalog), queryParams).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + ListNamespacesResponse res = response.readEntity(ListNamespacesResponse.class); + return res.namespaces(); + } + } + + public List listAllNamespacesChildFirst(String catalog) { + List result = new ArrayList<>(); + for (int idx = -1; idx < result.size(); idx++) { + Namespace parent = Namespace.empty(); + if (idx >= 0) { + parent = result.get(idx); + } + + result.addAll(listNamespaces(catalog, parent)); + } + + return result.reversed(); + } + + public void deleteNamespace(String catalog, Namespace namespace) { + String ns = RESTUtil.encodeNamespace(namespace); + try (Response response = + request("v1/{cat}/namespaces/" + ns, Map.of("cat", catalog)).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void purge(String catalog) { + listAllNamespacesChildFirst(catalog).forEach(ns -> purge(catalog, ns)); + } + + public void purge(String catalog, Namespace ns) { + listTables(catalog, ns).forEach(t -> dropTable(catalog, t)); + listViews(catalog, ns).forEach(t -> dropView(catalog, t)); + deleteNamespace(catalog, ns); + } + + public List listTables(String catalog, Namespace namespace) { + String ns = RESTUtil.encodeNamespace(namespace); + try (Response res = + request("v1/{cat}/namespaces/" + ns + "/tables", Map.of("cat", catalog)).get()) { + assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + return res.readEntity(ListTablesResponse.class).identifiers(); + } + } + + public void dropTable(String catalog, TableIdentifier id) { + String ns = RESTUtil.encodeNamespace(id.namespace()); + try (Response res = + request( + "v1/{cat}/namespaces/" + ns + "/tables/{table}", + Map.of("cat", catalog, "table", id.name())) + .delete()) { + assertThat(res.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public List listViews(String catalog, Namespace namespace) { + String ns = RESTUtil.encodeNamespace(namespace); + try (Response res = + request("v1/{cat}/namespaces/" + ns + "/views", Map.of("cat", catalog)).get()) { + assertThat(res.getStatus()).isEqualTo(Response.Status.OK.getStatusCode()); + return res.readEntity(ListTablesResponse.class).identifiers(); + } + } + + public void dropView(String catalog, TableIdentifier id) { + String ns = RESTUtil.encodeNamespace(id.namespace()); + try (Response res = + request( + "v1/{cat}/namespaces/" + ns + "/views/{view}", + Map.of("cat", catalog, "view", id.name())) + .delete()) { + assertThat(res.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java new file mode 100644 index 000000000..afa97bd57 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientCredentials.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; + +/** + * This class holds credentials for accessing the test Polaris Server. An instance of this class + * representing an admin user is injected into test parameters by {@link + * PolarisIntegrationTestExtension}. + */ +public record ClientCredentials(String clientId, String clientSecret) {} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java new file mode 100644 index 000000000..7090b4147 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ClientPrincipal.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; + +/** + * This class holds principal name and credentials for accessing the test Polaris Server. An + * instance of this class representing an admin user is injected into test parameters by {@link + * PolarisIntegrationTestExtension}. + * + * @see Server#adminCredentials() + */ +public record ClientPrincipal(String principalName, ClientCredentials credentials) {} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java new file mode 100644 index 000000000..020a12b27 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.apache.polaris.service.it.env.PolarisApiEndpoints.REALM_HEADER; +import static org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest.PRINCIPAL_ROLE_ALL; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.iceberg.catalog.SessionCatalog; +import org.apache.iceberg.rest.HTTPClient; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.rest.auth.OAuth2Properties; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; + +public final class IcebergHelper { + private IcebergHelper() {} + + public static RESTCatalog restCatalog( + PolarisApiEndpoints endpoints, + PrincipalWithCredentials credentials, + String catalog, + Map extraProperties) { + SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); + RESTCatalog restCatalog = + new RESTCatalog( + context, + (config) -> + HTTPClient.builder(config) + .uri(config.get(org.apache.iceberg.CatalogProperties.URI)) + .build()); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put( + org.apache.iceberg.CatalogProperties.URI, endpoints.catalogApiEndpoint().toString()) + .put( + OAuth2Properties.CREDENTIAL, + credentials.getCredentials().getClientId() + + ":" + + credentials.getCredentials().getClientSecret()) + .put(OAuth2Properties.SCOPE, PRINCIPAL_ROLE_ALL) + .put( + org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO") + .put("warehouse", catalog) + .put("header." + REALM_HEADER, endpoints.realm()) + .putAll(extraProperties); + + restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + return restCatalog; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java new file mode 100644 index 000000000..83ec02038 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/ManagementApi.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static jakarta.ws.rs.core.Response.Status.CREATED; +import static jakarta.ws.rs.core.Response.Status.NO_CONTENT; +import static jakarta.ws.rs.core.Response.Status.OK; +import static org.assertj.core.api.Assertions.assertThat; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.core.Response; +import java.net.URI; +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogGrant; +import org.apache.polaris.core.admin.model.CatalogPrivilege; +import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogRoles; +import org.apache.polaris.core.admin.model.Catalogs; +import org.apache.polaris.core.admin.model.CreatePrincipalRequest; +import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; +import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; +import org.apache.polaris.core.admin.model.GrantResource; +import org.apache.polaris.core.admin.model.GrantResources; +import org.apache.polaris.core.admin.model.Principal; +import org.apache.polaris.core.admin.model.PrincipalRole; +import org.apache.polaris.core.admin.model.PrincipalRoles; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.Principals; +import org.apache.polaris.core.admin.model.UpdateCatalogRequest; + +/** + * A simple, non-exhaustive set of helper methods for accessing the Polaris Management API. + * + * @see PolarisClient#managementApi(ClientCredentials) + */ +public class ManagementApi extends RestApi { + ManagementApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + super(client, endpoints, authToken, uri); + } + + public PrincipalWithCredentials createPrincipalWithRole(String principalName, String roleName) { + PrincipalWithCredentials credentials = createPrincipal(principalName); + createPrincipalRole(roleName); + assignPrincipalRole(principalName, roleName); + return credentials; + } + + public PrincipalWithCredentials createPrincipal(String name) { + try (Response createPResponse = + request("v1/principals").post(Entity.json(new Principal(name)))) { + assertThat(createPResponse).returns(CREATED.getStatusCode(), Response::getStatus); + return createPResponse.readEntity(PrincipalWithCredentials.class); + } + } + + public PrincipalWithCredentials createPrincipal(CreatePrincipalRequest request) { + try (Response createPResponse = request("v1/principals").post(Entity.json(request))) { + assertThat(createPResponse).returns(CREATED.getStatusCode(), Response::getStatus); + return createPResponse.readEntity(PrincipalWithCredentials.class); + } + } + + public void createPrincipalRole(String name) { + createPrincipalRole(new PrincipalRole(name)); + } + + public void createPrincipalRole(PrincipalRole role) { + try (Response createPrResponse = request("v1/principal-roles").post(Entity.json(role))) { + assertThat(createPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void assignPrincipalRole(String principalName, String roleName) { + try (Response assignPrResponse = + request("v1/principals/{prince}/principal-roles", Map.of("prince", principalName)) + .put(Entity.json(new GrantPrincipalRoleRequest(new PrincipalRole(roleName))))) { + assertThat(assignPrResponse).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void createCatalogRole(String catalogName, String catalogRoleName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) + .post(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .put(Entity.json(grant))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public void grantCatalogRoleToPrincipalRole( + String principalRoleName, String catalogName, CatalogRole catalogRole) { + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalogName)) + .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { + assertThat(response).returns(CREATED.getStatusCode(), Response::getStatus); + } + } + + public GrantResources listGrants(String catalogName, String catalogRoleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", catalogRoleName)) + .get()) { + assertThat(response).returns(OK.getStatusCode(), Response::getStatus); + return response.readEntity(GrantResources.class); + } + } + + public void createCatalog(String principalRoleName, Catalog catalog) { + createCatalog(catalog); + + // Create a new CatalogRole that has CATALOG_MANAGE_CONTENT and CATALOG_MANAGE_ACCESS + String catalogRoleName = "custom-admin"; + createCatalogRole(catalog.getName(), catalogRoleName); + + CatalogGrant grantResource = + new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_CONTENT, GrantResource.TypeEnum.CATALOG); + Map catalogVars = Map.of("cat", catalog.getName(), "role", catalogRoleName); + try (Response response = + request("v1/catalogs/{cat}/catalog-roles/{role}/grants", catalogVars) + .put(Entity.json(grantResource))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + + CatalogGrant grantAccessResource = + new CatalogGrant(CatalogPrivilege.CATALOG_MANAGE_ACCESS, GrantResource.TypeEnum.CATALOG); + try (Response response = + request("v1/catalogs/{cat}/catalog-roles/{role}/grants", catalogVars) + .put(Entity.json(grantAccessResource))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + + // Assign this new CatalogRole to the service_admin PrincipalRole + try (Response response = + request( + "v1/principal-roles/{role}/catalog-roles/{cat}", + Map.of("role", principalRoleName, "cat", catalog.getName())) + .put(Entity.json(new CatalogRole(catalogRoleName)))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public void createCatalog(Catalog catalog) { + try (Response response = request("v1/catalogs").post(Entity.json(catalog))) { + assertThat(response.getStatus()).isEqualTo(CREATED.getStatusCode()); + } + } + + public Catalog getCatalog(String name) { + try (Response response = request("v1/catalogs/{name}", Map.of("name", name)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Catalog.class); + } + } + + public void updateCatalog(Catalog catalog, Map catalogProps) { + try (Response response = + request("v1/catalogs/{name}", Map.of("name", catalog.getName())) + .put( + Entity.json( + new UpdateCatalogRequest( + catalog.getEntityVersion(), + catalogProps, + catalog.getStorageConfigInfo())))) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + } + } + + public void deleteCatalog(String catalogName) { + try (Response response = request("v1/catalogs/{cat}", Map.of("cat", catalogName)).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public CatalogRole getCatalogRole(String catalogName, String roleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of("cat", catalogName, "role", roleName)) + .get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(CatalogRole.class); + } + } + + public List listCatalogRoles(String catalogName) { + try (Response response = + request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)).get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(CatalogRoles.class).getRoles(); + } + } + + public List listPrincipals() { + try (Response response = request("v1/principals").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Principals.class).getPrincipals(); + } + } + + public List listPrincipalRoles() { + try (Response response = request("v1/principal-roles").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(PrincipalRoles.class).getRoles(); + } + } + + public List listCatalogs() { + try (Response response = request("v1/catalogs").get()) { + assertThat(response.getStatus()).isEqualTo(OK.getStatusCode()); + return response.readEntity(Catalogs.class).getCatalogs(); + } + } + + public void deleteCatalogRole(String catalogName, CatalogRole role) { + deleteCatalogRole(catalogName, role.getName()); + } + + public void deleteCatalogRole(String catalogName, String roleName) { + try (Response response = + request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of("cat", catalogName, "role", roleName)) + .delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void deletePrincipal(Principal principal) { + deletePrincipal(principal.getName()); + } + + public void deletePrincipal(String principalName) { + try (Response response = + request("v1/principals/{name}", Map.of("name", principalName)).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void deletePrincipalRole(PrincipalRole role) { + try (Response response = + request("v1/principal-roles/{name}", Map.of("name", role.getName())).delete()) { + assertThat(response.getStatus()).isEqualTo(NO_CONTENT.getStatusCode()); + } + } + + public void dropCatalog(String catalogName) { + listCatalogRoles(catalogName).stream() + .filter(cr -> !cr.getName().equals("catalog_admin")) + .forEach(role -> deleteCatalogRole(catalogName, role)); + + deleteCatalog(catalogName); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java new file mode 100644 index 000000000..ff4b17485 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisApiEndpoints.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.io.Serializable; +import java.net.URI; + +/** + * This class contains the most fundamental information for accessing Polaris APIs, such as the base + * URI and realm ID and provides methods for obtaining Icenberg REST API and Polaris Management + * endpoints. + */ +public final class PolarisApiEndpoints implements Serializable { + + public static String REALM_HEADER = "realm"; + + private final URI baseUri; + private final String realm; + + public PolarisApiEndpoints(URI baseUri, String realm) { + this.baseUri = baseUri; + this.realm = realm; + } + + public URI catalogApiEndpoint() { + return baseUri.resolve("api/catalog"); + } + + public URI managementApiEndpoint() { + return baseUri.resolve("api/management"); + } + + public String realm() { + return realm; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java new file mode 100644 index 000000000..8c25335f4 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/PolarisClient.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import static org.apache.polaris.service.it.ext.PolarisServerManagerLoader.polarisServerManager; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import jakarta.ws.rs.client.Client; +import java.util.Random; +import org.apache.iceberg.rest.RESTSerializers; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.service.it.ext.PolarisServerManager; + +/** + * This is a holder for the heavy-weight HTTP client for accessing Polaris APIs. This class provides + * method for constructing light-weight API wrappers for Iceberg REST and Polaris Management API + * that reuse the same shared HTTP client. + */ +public final class PolarisClient implements AutoCloseable { + private final PolarisApiEndpoints endpoints; + private final Client client; + // Use an alphanumeric ID for widest compatibility in HTTP and SQL. + // Use MAX_RADIX for shorter output. + private final String clientId = + Long.toString(Math.abs(new Random().nextLong()), Character.MAX_RADIX); + + private PolarisClient(PolarisApiEndpoints endpoints) { + this.endpoints = endpoints; + + this.client = polarisServerManager().createClient(); + } + + public static PolarisClient polarisClient(PolarisApiEndpoints endpoints) { + return new PolarisClient(endpoints); + } + + /** + * Utility method that creates an {@link ObjectMapper} sufficient for (de-)serializing client-side + * payloads for Iceberg REST and Polaris Management APIs. + * + *

It is recommended for {@link PolarisServerManager} implementations to use this {@link + * ObjectMapper} if the make custom {@link PolarisServerManager#createClient() clients}. + */ + public static ObjectMapper buildObjectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + mapper.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(mapper); + return mapper; + } + + /** + * This method should be used by test code to make top-level entity names. The purpose of this + * method is two-fold: + *

  • Identify top-level entities for latger clean-up by {@link #cleanUp(ClientCredentials)}. + *
  • Allow {@link PolarisServerManager}s to customize top-level entities per environment. + */ + public String newEntityName(String hint) { + return polarisServerManager().transformEntityName(hint + "_" + clientId); + } + + public ManagementApi managementApi(String authToken) { + return new ManagementApi(client, endpoints, authToken, endpoints.managementApiEndpoint()); + } + + public ManagementApi managementApi(ClientCredentials credentials) { + return managementApi(obtainToken(credentials)); + } + + public ManagementApi managementApi(PrincipalWithCredentials principal) { + return managementApi(obtainToken(principal)); + } + + public CatalogApi catalogApi(PrincipalWithCredentials principal) { + return new CatalogApi( + client, endpoints, obtainToken(principal), endpoints.catalogApiEndpoint()); + } + + public CatalogApi catalogApi(ClientCredentials credentials) { + return new CatalogApi( + client, endpoints, obtainToken(credentials), endpoints.catalogApiEndpoint()); + } + + /** + * Requests an access token from the Polaris server for the client ID/secret pair that is part of + * the given principal data object. + */ + public String obtainToken(PrincipalWithCredentials principal) { + return obtainToken( + new ClientCredentials( + principal.getCredentials().getClientId(), + principal.getCredentials().getClientSecret())); + } + + /** Requests an access token from the Polaris server for the given {@link ClientCredentials}. */ + public String obtainToken(ClientCredentials credentials) { + CatalogApi anon = new CatalogApi(client, endpoints, null, endpoints.catalogApiEndpoint()); + return anon.obtainToken(credentials); + } + + private boolean ownedName(String name) { + return name != null && name.contains(clientId); + } + + public void cleanUp(ClientCredentials credentials) { + ManagementApi managementApi = managementApi(credentials); + CatalogApi catalogApi = catalogApi(credentials); + + managementApi.listCatalogs().stream() + .filter(c -> ownedName(c.getName())) + .forEach( + c -> { + catalogApi.purge(c.getName()); + managementApi.dropCatalog(c.getName()); + }); + + managementApi.listPrincipalRoles().stream() + .filter(r -> ownedName(r.getName())) + .forEach(managementApi::deletePrincipalRole); + managementApi.listPrincipals().stream() + .filter(p -> ownedName(p.getName())) + .forEach(p -> managementApi.deletePrincipal(p.getName())); + } + + @Override + public void close() throws Exception { + client.close(); + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java new file mode 100644 index 000000000..e862fc2c8 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/RestApi.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; +import java.net.URI; +import java.util.Map; + +/** Base class for API helper classes. */ +public class RestApi { + private final Client client; + private final PolarisApiEndpoints endpoints; + private final String authToken; + private final URI uri; + + RestApi(Client client, PolarisApiEndpoints endpoints, String authToken, URI uri) { + this.client = client; + this.endpoints = endpoints; + this.authToken = authToken; + this.uri = uri; + } + + public Invocation.Builder request(String path) { + return request(path, Map.of()); + } + + public Invocation.Builder request(String path, Map templateValues) { + return request(path, templateValues, Map.of()); + } + + public Invocation.Builder request( + String path, Map templateValues, Map queryParams) { + WebTarget target = client.target(uri).path(path); + for (Map.Entry entry : templateValues.entrySet()) { + target = target.resolveTemplate(entry.getKey(), entry.getValue()); + } + for (Map.Entry entry : queryParams.entrySet()) { + target = target.queryParam(entry.getKey(), entry.getValue()); + } + Invocation.Builder request = target.request("application/json"); + request = request.header(PolarisApiEndpoints.REALM_HEADER, endpoints.realm()); + if (authToken != null) { + request = request.header("Authorization", "Bearer " + authToken); + } + return request; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java new file mode 100644 index 000000000..22fecf163 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/Server.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.env; + +import java.net.URI; + +/** + * This is a holder for access information to a particular Polaris Server. Test cases may use only + * the provided admin credentials or create new principals. + */ +public interface Server extends AutoCloseable { + String realmId(); + + /** + * The base URI to all Polaris APIs (e.g. the common base of the Iceberg REST API endpoints and + * Polaris Management API endpoints). + * + * @see PolarisApiEndpoints + */ + URI baseUri(); + + ClientPrincipal adminCredentials(); +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java new file mode 100644 index 000000000..e2b6c43b7 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisIntegrationTestExtension.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import static org.apache.polaris.service.it.ext.PolarisServerManagerLoader.polarisServerManager; + +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ExtensionContext.Namespace; +import org.junit.jupiter.api.extension.ExtensionContext.Store.CloseableResource; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolutionException; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.platform.engine.UniqueId; + +/** + * A JUnit test extension that connects {@link PolarisServerManager} with test code by resolving + * test parameters and managing the lifecycle of {@link Server} objects. + */ +public class PolarisIntegrationTestExtension implements ParameterResolver { + private static final Namespace NAMESPACE = + Namespace.create(PolarisIntegrationTestExtension.class); + + @Override + public boolean supportsParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + Class type = parameterContext.getParameter().getType(); + return type.isAssignableFrom(PolarisApiEndpoints.class) + || type.isAssignableFrom(ClientPrincipal.class) + || type.isAssignableFrom(ClientCredentials.class); + } + + @Override + public Object resolveParameter( + ParameterContext parameterContext, ExtensionContext extensionContext) + throws ParameterResolutionException { + Env env = env(extensionContext); + Class type = parameterContext.getParameter().getType(); + if (type.isAssignableFrom(PolarisApiEndpoints.class)) { + return env.endpoints(); + } else if (type.isAssignableFrom(ClientPrincipal.class)) { + return env.server.adminCredentials(); + } else if (type.isAssignableFrom(ClientCredentials.class)) { + return env.server.adminCredentials().credentials(); + } + throw new IllegalStateException("Unable to resolve parameter: " + parameterContext); + } + + private Env env(ExtensionContext context) { + ExtensionContext classCtx = classContext(context); + ExtensionContext.Store store = classCtx.getStore(NAMESPACE); + return store.getOrComputeIfAbsent( + Env.class, (key) -> new Env(polarisServerManager().serverForContext(classCtx)), Env.class); + } + + private ExtensionContext classContext(ExtensionContext context) { + while (context.getParent().isPresent()) { + UniqueId id = UniqueId.parse(context.getUniqueId()); + if ("class".equals(id.getLastSegment().getType())) { + break; + } + + context = context.getParent().get(); + } + + return context; + } + + private static class Env implements CloseableResource { + private final Server server; + private final PolarisApiEndpoints endpoints; + + private Env(Server server) { + this.server = server; + this.endpoints = new PolarisApiEndpoints(server.baseUri(), server.realmId()); + } + + PolarisApiEndpoints endpoints() { + return endpoints; + } + + @Override + public void close() throws Throwable { + server.close(); + } + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java new file mode 100644 index 000000000..c23c1f6c9 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManager.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import static java.util.concurrent.TimeUnit.MINUTES; +import static org.apache.polaris.service.it.env.PolarisClient.buildObjectMapper; + +import com.fasterxml.jackson.jakarta.rs.json.JacksonJsonProvider; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientBuilder; +import java.util.ServiceLoader; +import org.apache.polaris.service.it.env.Server; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * This is a plugin interfaces to allow different test execution environments to control how tests + * access Polaris Servers when they run under {@link PolarisIntegrationTestExtension}. + * + *

    Implementations are loaded via {@link ServiceLoader} assuming there is only one implementation + * per test runtime class path. + */ +public interface PolarisServerManager { + + /** + * Returns server connection parameters for the tests under the specified context. + * + *

    Implementations may reuse the same server for multiple contexts (with the same of different + * {@link Server#realmId() realm IDs}) or create a fresh server for each context. In any case, + * {@link Server#close()} will be invoked when the context provided as the argument to this call + * is closed. + * + *

    Note: {@link Server} objects are generally attached to the test {@code class} context, but + * this is not guaranteed. + */ + Server serverForContext(ExtensionContext context); + + /** Create a new HTTP client for accessing the server targeted by tests. */ + default Client createClient() { + return ClientBuilder.newBuilder() + .readTimeout(5, MINUTES) + .connectTimeout(1, MINUTES) + .register(new JacksonJsonProvider(buildObjectMapper())) + .build(); + } + + /** + * Transforms the name of an entity that tests need to create. Implementations may prepend of + * append text to the original name, but they should not alter the original name text or add any + * characters that have special meaning in Spark SQL, HTTP or Iceberg REST specifications. + * + *

    This method will be called for all top-level entities (catalogs, principal, principal + * roles), but may not be called for secondary entities (such as catalog roles, namespaces, + * tables, etc.). + */ + default String transformEntityName(String name) { + return name; + } +} diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManagerLoader.java b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManagerLoader.java new file mode 100644 index 000000000..76879f880 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/ext/PolarisServerManagerLoader.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.ext; + +import java.util.ServiceLoader; + +public final class PolarisServerManagerLoader { + + private static final PolarisServerManager manager = + ServiceLoader.load(PolarisServerManager.class) + .findFirst() + .orElseThrow(() -> new IllegalStateException("PolarisServerManager not found")); + + private PolarisServerManagerLoader() {} + + public static PolarisServerManager polarisServerManager() { + return manager; + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java similarity index 61% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index e7f7cd4a2..f7c10fc24 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -16,30 +16,27 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.it.env.PolarisApiEndpoints.REALM_HEADER; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; -import jakarta.ws.rs.ProcessingException; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; @@ -80,145 +77,88 @@ import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.assertj.core.api.Assertions; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ClientPrincipal; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.env.RestApi; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; - -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) + +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisApplicationIntegrationTest { - @TempDir private static Path tempDir; - private static final Supplier CURRENT_LOG = - () -> tempDir.resolve("application.log").toString(); - - private static final Logger LOGGER = - LoggerFactory.getLogger(PolarisApplicationIntegrationTest.class); - - public static final String PRINCIPAL_ROLE_NAME = "admin"; - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0"), // Bind to random port to support parallelism - ConfigOverride.config("logging.appenders[1].type", "file"), - ConfigOverride.config("logging.appenders[1].currentLogFilename", CURRENT_LOG)); - - private static String userToken; - private static SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials; + + public static final String PRINCIPAL_ROLE_ALL = "PRINCIPAL_ROLE:ALL"; + private static Path testDir; private static String realm; + private static RestApi managementApi; + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ClientCredentials clientCredentials; + private static ClientPrincipal admin; + + private String principalRoleName; + private String internalCatalogName; + @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken userToken, - SnowmanCredentialsExtension.SnowmanCredentials snowmanCredentials, - @PolarisRealm String polarisRealm) + public static void setup(PolarisApiEndpoints apiEndpoints, ClientPrincipal adminCredentials) throws IOException { - realm = polarisRealm; - - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .contains("PolarisApplication: Server started successfully"); + endpoints = apiEndpoints; + client = polarisClient(endpoints); + realm = endpoints.realm(); + admin = adminCredentials; + clientCredentials = adminCredentials.credentials(); testDir = Path.of("build/test_data/iceberg/" + realm); FileUtils.deleteQuietly(testDir.toFile()); Files.createDirectories(testDir); - PolarisApplicationIntegrationTest.userToken = userToken.token(); - PolarisApplicationIntegrationTest.snowmanCredentials = snowmanCredentials; - PrincipalRole principalRole = new PrincipalRole(PRINCIPAL_ROLE_NAME); + managementApi = client.managementApi(clientCredentials); + } + + @AfterAll + public static void close() throws Exception { + client.close(); + } + + /** + * Create a new catalog for each test case. Assign the snowman catalog-admin principal role the + * admin role of the new catalog. + */ + @BeforeEach + public void before(TestInfo testInfo) { + principalRoleName = client.newEntityName("admin"); + PrincipalRole principalRole = new PrincipalRole(principalRoleName); try (Response createPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(principalRole))) { + managementApi.request("v1/principal-roles").post(Entity.json(principalRole))) { assertThat(createPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response assignPrResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principals/%s/principal-roles", - EXT.getLocalPort(), snowmanCredentials.identifier().principalName())) - .request("application/json") - .header("Authorization", "Bearer " + PolarisApplicationIntegrationTest.userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request("v1/principals/{name}/principal-roles", Map.of("name", admin.principalName())) .put(Entity.json(principalRole))) { assertThat(assignPrResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - assertZeroErrorsInApplicationLog(); + internalCatalogName = client.newEntityName(testInfo.getTestMethod().orElseThrow().getName()); + createCatalog(internalCatalogName, Catalog.TypeEnum.INTERNAL, principalRoleName); } - @AfterAll - public static void deletePrincipalRole() { - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s", - EXT.getLocalPort(), PRINCIPAL_ROLE_NAME)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .delete() - .close(); - } - - private static void assertZeroErrorsInApplicationLog() { - assertThat(new File(CURRENT_LOG.get())) - .exists() - .content() - .hasSizeGreaterThan(0) - .doesNotContain("ERROR", "FATAL"); - } - - /** - * Create a new catalog for each test case. Assign the snowman catalog-admin principal role the - * admin role of the new catalog. - * - * @param testInfo - */ - @BeforeEach - public void before(TestInfo testInfo) { - testInfo - .getTestMethod() - .ifPresent( - method -> { - String catalogName = method.getName(); - Catalog.TypeEnum catalogType = Catalog.TypeEnum.INTERNAL; - createCatalog(catalogName, catalogType, PRINCIPAL_ROLE_NAME); - }); + @AfterEach + public void cleanUp() throws Exception { + client.cleanUp(clientCredentials); } private static void createCatalog( @@ -262,40 +202,24 @@ private static void createCatalog( .setProperties(props) .setStorageConfigInfo(storageConfig) .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalog))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s", - EXT.getLocalPort(), - catalogName, - PolarisEntityConstants.getNameOfCatalogAdminRole())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/catalogs/{cat}/catalog-roles/{role}", + Map.of( + "cat", catalogName, "role", PolarisEntityConstants.getNameOfCatalogAdminRole())) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); CatalogRole catalogRole = response.readEntity(CatalogRole.class); try (Response assignResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles/%s/catalog-roles/%s", - EXT.getLocalPort(), principalRoleName, catalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + managementApi + .request( + "v1/principal-roles/{prin-role}/catalog-roles/{cat}", + Map.of("cat", catalogName, "prin-role", principalRoleName)) .put(Entity.json(catalogRole))) { assertThat(assignResponse) .returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -309,21 +233,21 @@ private static RESTSessionCatalog newSessionCatalog(String catalog) { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + endpoints.catalogApiEndpoint().toString(), OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", catalog, - "header." + REALM_PROPERTY_KEY, + "header." + REALM_HEADER, realm)); return sessionCatalog; } @Test public void testIcebergListNamespaces() throws IOException { - try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergListNamespaces")) { + try (RESTSessionCatalog sessionCatalog = newSessionCatalog(internalCatalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); List namespaces = sessionCatalog.listNamespaces(sessionContext); assertThat(namespaces).isNotNull().isEmpty(); @@ -331,7 +255,7 @@ public void testIcebergListNamespaces() throws IOException { } @Test - public void testConfigureCatalogCaseSensitive() throws IOException { + public void testConfigureCatalogCaseSensitive() { assertThatThrownBy(() -> newSessionCatalog("TESTCONFIGURECATALOGCASESENSITIVE")) .isInstanceOf(RESTException.class) .hasMessage( @@ -340,8 +264,7 @@ public void testConfigureCatalogCaseSensitive() throws IOException { @Test public void testIcebergListNamespacesNotFound() throws IOException { - try (RESTSessionCatalog sessionCatalog = - newSessionCatalog("testIcebergListNamespacesNotFound")) { + try (RESTSessionCatalog sessionCatalog = newSessionCatalog(internalCatalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); assertThatThrownBy( () -> sessionCatalog.listNamespaces(sessionContext, Namespace.of("whoops"))) @@ -352,8 +275,7 @@ public void testIcebergListNamespacesNotFound() throws IOException { @Test public void testIcebergListNamespacesNestedNotFound() throws IOException { - try (RESTSessionCatalog sessionCatalog = - newSessionCatalog("testIcebergListNamespacesNestedNotFound")) { + try (RESTSessionCatalog sessionCatalog = newSessionCatalog(internalCatalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace topLevelNamespace = Namespace.of("top_level"); sessionCatalog.createNamespace(sessionContext, topLevelNamespace); @@ -369,8 +291,7 @@ public void testIcebergListNamespacesNestedNotFound() throws IOException { @Test public void testIcebergListTablesNamespaceNotFound() throws IOException { - try (RESTSessionCatalog sessionCatalog = - newSessionCatalog("testIcebergListTablesNamespaceNotFound")) { + try (RESTSessionCatalog sessionCatalog = newSessionCatalog(internalCatalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); assertThatThrownBy(() -> sessionCatalog.listTables(sessionContext, Namespace.of("whoops"))) .isInstanceOf(NoSuchNamespaceException.class) @@ -380,7 +301,7 @@ public void testIcebergListTablesNamespaceNotFound() throws IOException { @Test public void testIcebergCreateNamespace() throws IOException { - try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergCreateNamespace")) { + try (RESTSessionCatalog sessionCatalog = newSessionCatalog(internalCatalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace topLevelNamespace = Namespace.of("top_level"); sessionCatalog.createNamespace(sessionContext, topLevelNamespace); @@ -394,9 +315,10 @@ public void testIcebergCreateNamespace() throws IOException { } @Test - public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); + public void testIcebergCreateNamespaceInExternalCatalog() throws IOException { + String catalogName = + client.newEntityName("testIcebergCreateNamespaceInExternalCatalogExternal"); + createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, principalRoleName); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); @@ -413,9 +335,9 @@ public void testIcebergCreateNamespaceInExternalCatalog(TestInfo testInfo) throw } @Test - public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); + public void testIcebergDropNamespaceInExternalCatalog() throws IOException { + String catalogName = client.newEntityName("testIcebergDropNamespaceInExternalCatalogExternal"); + createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, principalRoleName); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); @@ -431,8 +353,8 @@ public void testIcebergDropNamespaceInExternalCatalog(TestInfo testInfo) throws @Test public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; - createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, PRINCIPAL_ROLE_NAME); + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; + createCatalog(catalogName, Catalog.TypeEnum.EXTERNAL, principalRoleName); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); @@ -458,66 +380,61 @@ public void testIcebergCreateTablesInExternalCatalog(TestInfo testInfo) throws I @Test public void testIcebergCreateTablesWithWritePathBlocked(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "Internal"; - createCatalog(catalogName, Catalog.TypeEnum.INTERNAL, PRINCIPAL_ROLE_NAME); + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "Internal"; + createCatalog(catalogName, Catalog.TypeEnum.INTERNAL, principalRoleName); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName)) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); - try { - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - - Assertions.assertThatThrownBy( - () -> - sessionCatalog - .buildTable( - sessionContext, - TableIdentifier.of(ns, "the_table"), - new Schema( - List.of( - Types.NestedField.of( - 1, false, "theField", Types.StringType.get())))) - .withSortOrder(SortOrder.unsorted()) - .withPartitionSpec(PartitionSpec.unpartitioned()) - .withProperties( - Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) - .create()) - .isInstanceOf(ForbiddenException.class) - .hasMessageContaining("Forbidden: Invalid locations"); - } catch (BadRequestException e) { - LOGGER.info("Received expected exception {}", e.getMessage()); - } + assertThatThrownBy( + () -> + sessionCatalog + .buildTable( + sessionContext, + TableIdentifier.of(ns, "the_table"), + new Schema( + List.of( + Types.NestedField.of( + 1, false, "theField", Types.StringType.get())))) + .withSortOrder(SortOrder.unsorted()) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withProperties(Map.of("write.data.path", "s3://my-bucket/path/to/data")) + .create()) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Forbidden: Invalid locations"); + + assertThatThrownBy( + () -> + sessionCatalog + .buildTable( + sessionContext, + TableIdentifier.of(ns, "the_table"), + new Schema( + List.of( + Types.NestedField.of( + 1, false, "theField", Types.StringType.get())))) + .withSortOrder(SortOrder.unsorted()) + .withPartitionSpec(PartitionSpec.unpartitioned()) + .withProperties(Map.of("write.metadata.path", "s3://my-bucket/path/to/data")) + .create()) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("Forbidden: Invalid locations"); } } @Test public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, - PRINCIPAL_ROLE_NAME, + principalRoleName, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -554,17 +471,17 @@ public void testIcebergRegisterTableInExternalCatalog(TestInfo testInfo) throws @Test public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + String catalogName = testInfo.getTestMethod().orElseThrow().getName() + "External"; createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, - PRINCIPAL_ROLE_NAME, + principalRoleName, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -606,18 +523,18 @@ public void testIcebergUpdateTableInExternalCatalog(TestInfo testInfo) throws IO } @Test - public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOException { - String catalogName = testInfo.getTestMethod().get().getName() + "External"; + public void testIcebergDropTableInExternalCatalog() throws IOException { + String catalogName = client.newEntityName("testIcebergDropTableInExternalCatalogExternal"); createCatalog( catalogName, Catalog.TypeEnum.EXTERNAL, - PRINCIPAL_ROLE_NAME, + principalRoleName, FileStorageConfigInfo.builder(StorageConfigInfo.StorageTypeEnum.FILE) .setAllowedLocations(List.of("file://" + testDir.toFile().getAbsolutePath())) .build(), "file://" + testDir.toFile().getAbsolutePath()); try (RESTSessionCatalog sessionCatalog = newSessionCatalog(catalogName); - HadoopFileIO fileIo = new HadoopFileIO(new Configuration()); ) { + HadoopFileIO fileIo = new HadoopFileIO(new Configuration())) { SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty(); Namespace ns = Namespace.of("db1"); sessionCatalog.createNamespace(sessionContext, ns); @@ -626,7 +543,7 @@ public void testIcebergDropTableInExternalCatalog(TestInfo testInfo) throws IOEx "file://" + testDir.toFile().getAbsolutePath() + "/" - + testInfo.getTestMethod().get().getName(); + + "testIcebergDropTableInExternalCatalog"; String metadataLocation = location + "/metadata/000001-494949494949494949.metadata.json"; TableMetadata tableMetadata = @@ -661,14 +578,14 @@ public void testWarehouseNotSpecified() throws IOException { "polaris_catalog_test", Map.of( "uri", - "http://localhost:" + EXT.getLocalPort() + "/api/catalog", + endpoints.catalogApiEndpoint().toString(), OAuth2Properties.CREDENTIAL, - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(), + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(), OAuth2Properties.SCOPE, - BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL, + PRINCIPAL_ROLE_ALL, "warehouse", emptyEnvironmentVariable, - "header." + REALM_PROPERTY_KEY, + "header." + REALM_HEADER, realm))) .isInstanceOf(BadRequestException.class) .hasMessage("Malformed request: Please specify a warehouse"); @@ -676,79 +593,67 @@ public void testWarehouseNotSpecified() throws IOException { } @Test - public void testRequestHeaderTooLarge() { - Invocation.Builder request = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json"); - - // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 definitely - // exceeds the limit - for (int i = 0; i < 1500; i++) { - request = request.header("header" + i, "" + i); - } - - try { - try (Response response = - request - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(new PrincipalRole("r")))) { - assertThat(response) - .returns( - Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode(), - Response::getStatus); - } - } catch (ProcessingException e) { - // In some runtime environments the request above will return a 431 but in others it'll result - // in a ProcessingException from the socket being closed. The test asserts that one of those - // things happens. + public void testRequestHeaderTooLarge() throws Exception { + // Use a dedicated client with retries due to non-deterministic behaviour of reading the + // response code and possible abrupt connection resets + try (PolarisClient localClient = polarisClient(endpoints)) { + await() + .atMost(Duration.of(1, ChronoUnit.MINUTES)) + .untilAsserted( + () -> { + Invocation.Builder request = + localClient.managementApi(clientCredentials).request("v1/principal-roles"); + // The default limit is 8KiB and each of these headers is at least 8 bytes, so 1500 + // definitely exceeds the limit + for (int i = 0; i < 1500; i++) { + request = request.header("header" + i, "" + i); + } + + try (Response response = request.post(Entity.json(new PrincipalRole("r")))) { + assertThat(response.getStatus()) + .isEqualTo(Response.Status.REQUEST_HEADER_FIELDS_TOO_LARGE.getStatusCode()); + } + }); } } @Test - public void testRequestBodyTooLarge() { - // The behaviour in case of large requests depends on the specific server configuration. - // This test assumes that the server under test is configured to deny requests larger than - // 1000000 bytes. The test payload below assumes UTF8 encoding of ASCII charts plus a bit of - // JSON overhead. - Entity largeRequest = Entity.json(new PrincipalRole("r".repeat(1000001))); - - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/principal-roles", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(largeRequest)) { - // Note we only validate the status code here because per RFC 9110, the server MAY not provide - // a response body. The HTTP status line is still expected to be provided. - assertThat(response.getStatus()) - .isEqualTo(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode()); - } catch (ProcessingException e) { - // Per RFC 9110 servers MAY close the connection in case of 413 responses, which - // might cause the client to fail to read the status code (cf. RFC 9112, section 9.6). - // TODO: servers are expected to close connections gracefully. It might be worth investigating - // whether "connection closed" exceptions are a client-side bug. - assertThat(e).hasMessageContaining("Connection was closed"); + public void testRequestBodyTooLarge() throws Exception { + // Use a dedicated client with retries due to non-deterministic behaviour of reading the + // response code and possible abrupt connection resets + try (PolarisClient localClient = polarisClient(endpoints)) { + await() + .atMost(Duration.of(1, ChronoUnit.MINUTES)) + .untilAsserted( + () -> { + // The behaviour in case of large requests depends on the specific server + // configuration. This test assumes that the server under test is configured to deny + // requests larger than 1000000 bytes. The test payload below assumes UTF8 encoding + // of ASCII charts plus a bit of JSON overhead. + Entity largeRequest = + Entity.json(new PrincipalRole("r".repeat(1000001))); + try (Response response = + localClient + .managementApi(clientCredentials) + .request("v1/principal-roles") + .post(largeRequest)) { + // Note we only validate the status code here because per RFC 9110, the server MAY + // not provide a response body. The HTTP status line is still expected to be + // provided most of the time. + assertThat(response.getStatus()) + .isEqualTo(Response.Status.REQUEST_ENTITY_TOO_LARGE.getStatusCode()); + } + }); } } @Test public void testRefreshToken() throws IOException { - String path = - String.format("http://localhost:%d/api/catalog/v1/oauth/tokens", EXT.getLocalPort()); + String path = endpoints.catalogApiEndpoint() + "/v1/oauth/tokens"; try (RESTClient client = - HTTPClient.builder(ImmutableMap.of()) - .withHeader(REALM_PROPERTY_KEY, realm) - .uri(path) - .build()) { + HTTPClient.builder(Map.of()).withHeader(REALM_HEADER, realm).uri(path).build()) { String credentialString = - snowmanCredentials.clientId() + ":" + snowmanCredentials.clientSecret(); + clientCredentials.clientId() + ":" + clientCredentials.clientSecret(); String expiredToken = JWT.create().withExpiresAt(Instant.EPOCH).sign(Algorithm.HMAC256("irrelevant-secret")); var authConfig = diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java similarity index 61% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java index f2120c363..efe91ae6d 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/admin/PolarisServiceImplIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisManagementServiceIntegrationTest.java @@ -16,25 +16,20 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.admin; +package org.apache.polaris.service.it.test; -import static io.dropwizard.jackson.Jackson.newObjectMapper; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static javax.ws.rs.core.Response.Status.FORBIDDEN; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; +import static org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest.PRINCIPAL_ROLE_ALL; import static org.assertj.core.api.Assertions.assertThat; import com.auth0.jwt.JWT; import com.auth0.jwt.JWTCreator; import com.auth0.jwt.algorithms.Algorithm; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; import java.io.IOException; import java.time.Duration; @@ -45,12 +40,7 @@ import java.util.Map; import java.util.UUID; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.rest.RESTUtil; -import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.iceberg.rest.responses.ListNamespacesResponse; -import org.apache.polaris.core.admin.model.AddGrantRequest; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; import org.apache.polaris.core.admin.model.AzureStorageConfigInfo; import org.apache.polaris.core.admin.model.Catalog; @@ -67,7 +57,6 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest; -import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; @@ -76,7 +65,6 @@ import org.apache.polaris.core.admin.model.PrincipalRole; import org.apache.polaris.core.admin.model.PrincipalRoles; import org.apache.polaris.core.admin.model.PrincipalWithCredentials; -import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials; import org.apache.polaris.core.admin.model.Principals; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.UpdateCatalogRequest; @@ -84,28 +72,22 @@ import org.apache.polaris.core.admin.model.UpdatePrincipalRequest; import org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.core.entity.PolarisPrincipalSecrets; -import org.apache.polaris.service.auth.BasePolarisAuthenticator; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.slf4j.LoggerFactory; import org.testcontainers.shaded.org.awaitility.Awaitility; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) -public class PolarisServiceImplIntegrationTest { +@ExtendWith(PolarisIntegrationTestExtension.class) +public class PolarisManagementServiceIntegrationTest { private static final int MAX_IDENTIFIER_LENGTH = 256; private static final String ISSUER_KEY = "polaris"; private static final String CLAIM_KEY_ACTIVE = "active"; @@ -113,147 +95,27 @@ public class PolarisServiceImplIntegrationTest { private static final String CLAIM_KEY_PRINCIPAL_ID = "principalId"; private static final String CLAIM_KEY_SCOPE = "scope"; - // TODO: Add a test-only hook that fully clobbers all persistence state so we can have a fresh - // slate on every test case; otherwise, leftover state from one test from failures will interfere - // with other test cases. - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config("server.adminConnectors[0].port", "0"), - ConfigOverride.config("gcp_credentials.access_token", "abc"), - ConfigOverride.config("gcp_credentials.expires_in", "12345")); - private static String userToken; - private static String realm; - private static String clientId; + private static PolarisClient client; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; + private static ClientCredentials rootCredentials; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken adminToken, - PolarisPrincipalSecrets adminSecrets, - @PolarisRealm String polarisRealm) - throws IOException { - userToken = adminToken.token(); - realm = polarisRealm; - clientId = adminSecrets.getPrincipalClientId(); - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + public static void setup(PolarisApiEndpoints endpoints, ClientCredentials credentials) { + client = polarisClient(endpoints); + managementApi = client.managementApi(credentials); + catalogApi = client.catalogApi(credentials); + rootCredentials = credentials; + } + + @AfterAll + public static void close() throws Exception { + client.close(); } @AfterEach public void tearDown() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { - response - .readEntity(Catalogs.class) - .getCatalogs() - .forEach( - catalog -> { - // clean up the catalog before we try to drop it - - // delete all the namespaces - try (Response res = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list namespaces in catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - } else { - res.readEntity(ListNamespacesResponse.class) - .namespaces() - .forEach( - namespace -> { - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalog.getName() - + "/namespaces/" - + RESTUtil.encodeNamespace(namespace)) - .delete() - .close(); - }); - } - } - - // delete all the catalog roles except catalog_admin - try (Response res = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles") - .get()) { - if (res.getStatus() != Response.Status.OK.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to list catalog roles for catalog {}: {}", - catalog.getName(), - res.readEntity(String.class)); - return; - } - res.readEntity(CatalogRoles.class).getRoles().stream() - .filter(cr -> !cr.getName().equals("catalog_admin")) - .forEach( - cr -> - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalog.getName() - + "/catalog-roles/" - + cr.getName()) - .delete() - .close()); - } - - Response deleteResponse = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalog.getName()) - .delete(); - if (deleteResponse.getStatus() != Response.Status.NO_CONTENT.getStatusCode()) { - LoggerFactory.getLogger(getClass()) - .warn( - "Unable to delete catalog {}: {}", - catalog.getName(), - deleteResponse.readEntity(String.class)); - } - deleteResponse.close(); - }); - } - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { - response.readEntity(Principals.class).getPrincipals().stream() - .filter( - principal -> - !principal.getName().equals(PolarisEntityConstants.getRootPrincipalName())) - .forEach( - principal -> { - newRequest( - "http://localhost:%d/api/management/v1/principals/" + principal.getName()) - .delete() - .close(); - }); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { - response.readEntity(PrincipalRoles.class).getRoles().stream() - .filter( - principalRole -> - !principalRole - .getName() - .equals(PolarisEntityConstants.getNameOfPrincipalServiceAdminRole())) - .forEach( - principalRole -> { - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRole.getName()) - .delete() - .close(); - }); - } + client.cleanUp(rootCredentials); } @Test @@ -284,7 +146,7 @@ public void testCatalogSerializing() throws IOException { @Test public void testListCatalogs() { - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -299,23 +161,9 @@ public void testListCatalogs() { @Test public void testListCatalogsUnauthorized() { - Principal principal = new Principal("a_new_user"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", newToken).get()) { + PrincipalWithCredentials principal = + managementApi.createPrincipal(client.newEntityName("a_new_user")); + try (Response response = client.managementApi(principal).request("v1/catalogs").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -323,7 +171,8 @@ public void testListCatalogsUnauthorized() { @Test public void testCreateCatalog() { try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") + managementApi + .request("v1/catalogs") .post( Entity.json( "{\"catalog\":{\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"},\"storageConfigInfo\":{\"storageType\":\"S3\",\"roleArn\":\"arn:aws:iam::123456789012:role/my-role\",\"externalId\":\"externalId\",\"userArn\":\"userArn\",\"allowedLocations\":[\"s3://my-old-bucket/path/to/data\"]}}}"))) { @@ -331,8 +180,7 @@ public void testCreateCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -350,8 +198,6 @@ public void testCreateCatalogWithInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); - ObjectMapper mapper = newObjectMapper(); - Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -359,12 +205,8 @@ public void testCreateCatalogWithInvalidName() { .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); } String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); @@ -386,16 +228,12 @@ public void testCreateCatalogWithInvalidName() { .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(mapper.writeValueAsString(catalog)))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(catalog))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); ErrorResponse errorResponse = response.readEntity(ErrorResponse.class); assertThat(errorResponse.message()).contains("Invalid value:"); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); } } } @@ -419,12 +257,10 @@ public void testCreateCatalogWithAzureStorageConfig() { .setStorageConfigInfo(azureConfigInfo) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + try (Response response = managementApi.request("v1/catalogs/my-catalog").get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -447,17 +283,15 @@ public void testCreateCatalogWithGcpStorageConfig() { Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("my-catalog") + .setName(client.newEntityName("my-catalog")) .setProperties(new CatalogProperties("gs://my-bucket/path/to/data")) .setStorageConfigInfo(gcpConfigModel) .build(); + + managementApi.createCatalog(catalog); + try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/my-catalog").get()) { + managementApi.request("v1/catalogs/{cat}", Map.of("cat", catalog.getName())).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog catResponse = response.readEntity(Catalog.class); assertThat(catResponse.getStorageConfigInfo()) @@ -486,9 +320,7 @@ public void testCreateCatalogWithNullBaseLocation() { catalogNode.set("properties", mapper.createObjectNode()); ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } @@ -513,9 +345,7 @@ public void testCreateCatalogWithoutProperties() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -528,12 +358,11 @@ public void testCreateCatalogWithoutProperties() { } @Test - public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingException { + public void testCreateCatalogWithoutStorageConfig() { String catalogString = "{\"catalog\": {\"type\":\"INTERNAL\",\"name\":\"my-catalog\",\"properties\":{\"default-base-location\":\"s3://my-bucket/path/to/data\"}}}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(catalogString))) { + managementApi.request("v1/catalogs").post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -546,11 +375,10 @@ public void testCreateCatalogWithoutStorageConfig() throws JsonProcessingExcepti } @Test - public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException { + public void testCreateCatalogWithUnparsableJson() { String catalogString = "{\"catalog\": {{\"bad data}"; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(catalogString))) { + managementApi.request("v1/catalogs").post(Entity.json(catalogString))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -559,7 +387,7 @@ public void testCreateCatalogWithUnparsableJson() throws JsonProcessingException } @Test - public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonProcessingException { + public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() { AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -577,16 +405,13 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro .setStorageConfigInfo(awsConfigModel) .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs", userToken) - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -603,10 +428,9 @@ public void testUpdateCatalogWithoutDefaultBaseLocationInUpdate() throws JsonPro fetchedCatalog.getEntityVersion(), Map.of("foo", "bar"), null /* storageConfigIno */); // Successfully update - Catalog updatedCatalog = null; + Catalog updatedCatalog; try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName, userToken) - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/" + catalogName).put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); updatedCatalog = response.readEntity(Catalog.class); @@ -628,7 +452,7 @@ public void testCreateExternalCatalog() { .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) .build(); - String catalogName = "my-external-catalog"; + String catalogName = client.newEntityName("my-external-catalog"); String remoteUrl = "http://localhost:8080"; Catalog catalog = ExternalCatalog.builder() @@ -638,10 +462,9 @@ public void testCreateExternalCatalog() { .setProperties(new CatalogProperties("s3://my-bucket/path/to/data")) .setStorageConfigInfo(awsConfigModel) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).get()) { + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); Catalog fetchedCatalog = response.readEntity(Catalog.class); assertThat(fetchedCatalog) @@ -656,11 +479,7 @@ public void testCreateExternalCatalog() { .returns("arn:aws:iam::123456789012:role/my-role", AwsStorageConfigInfo::getRoleArn); } - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + catalogName).delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + managementApi.deleteCatalog(catalogName); } @Test @@ -685,16 +504,14 @@ public void testCreateCatalogWithoutDefaultLocation() { ObjectNode requestNode = mapper.createObjectNode(); requestNode.set("catalog", catalogNode); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(requestNode))) { + try (Response response = managementApi.request("v1/catalogs").post(Entity.json(requestNode))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); } } @Test - public void serialization() throws JsonProcessingException { + public void serialization() { CatalogProperties properties = new CatalogProperties("s3://my-bucket/path/to/data"); ObjectMapper mapper = new ObjectMapper(); CatalogProperties translated = mapper.convertValue(properties, CatalogProperties.class); @@ -706,25 +523,25 @@ public void serialization() throws JsonProcessingException { public void testCreateAndUpdateAzureCatalog() { StorageConfigInfo storageConfig = new AzureStorageConfigInfo("azure:tenantid:12345", StorageConfigInfo.StorageTypeEnum.AZURE); + String catalogName = client.newEntityName("myazurecatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("myazurecatalog") + .setName(catalogName) .setStorageConfigInfo(storageConfig) .setProperties(new CatalogProperties("abfss://container1@acct1.dfs.core.windows.net/")) .build(); // 200 Successful create - createCatalog(catalog); + managementApi.createCatalog(catalog); // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); - assertThat(fetchedCatalog.getName()).isEqualTo("myazurecatalog"); + assertThat(fetchedCatalog.getName()).isEqualTo(catalogName); assertThat(fetchedCatalog.getProperties().toMap()) .isEqualTo( Map.of("default-base-location", "abfss://container1@acct1.dfs.core.windows.net/")); @@ -739,8 +556,7 @@ public void testCreateAndUpdateAzureCatalog() { Map.of("default-base-location", "abfss://newcontainer@acct1.dfs.core.windows.net/"), modifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(badUpdateRequest))) { + managementApi.request("v1/catalogs/" + catalogName).put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -759,8 +575,7 @@ public void testCreateAndUpdateAzureCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog") - .put(Entity.json(updateRequest))) { + managementApi.request("v1/catalogs/" + catalogName).put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -770,8 +585,7 @@ public void testCreateAndUpdateAzureCatalog() { } // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/myazurecatalog").delete()) { + try (Response response = managementApi.request("v1/catalogs/" + catalogName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -781,45 +595,44 @@ public void testCreateListUpdateAndDeleteCatalog() { StorageConfigInfo storageConfig = new AwsStorageConfigInfo( "arn:aws:iam::123456789011:role/role1", StorageConfigInfo.StorageTypeEnum.S3); + String catalogName = client.newEntityName("mycatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog") + .setName(catalogName) .setStorageConfigInfo(storageConfig) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { + managementApi.request("v1/catalogs").post(Entity.json(new CreateCatalogRequest(catalog)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Catalog fetchedCatalog = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + Catalog fetchedCatalog; + try (Response response = managementApi.request("v1/catalogs/" + catalogName).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); - assertThat(fetchedCatalog.getName()).isEqualTo("mycatalog"); + assertThat(fetchedCatalog.getName()).isEqualTo(catalogName); assertThat(fetchedCatalog.getProperties().toMap()) .isEqualTo(Map.of("default-base-location", "s3://bucket1/")); assertThat(fetchedCatalog.getEntityVersion()).isGreaterThan(0); } // Should list the catalog. - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) .extracting(Catalogs::getCatalogs) .asInstanceOf(InstanceOfAssertFactories.list(Catalog.class)) .filteredOn(cat -> !cat.getName().equalsIgnoreCase("ROOT")) - .satisfiesExactly(cat -> assertThat(cat).returns("mycatalog", Catalog::getName)); + .satisfiesExactly(cat -> assertThat(cat).returns(catalogName, Catalog::getName)); } // Reject update of fields that can't be currently updated @@ -832,7 +645,8 @@ public void testCreateListUpdateAndDeleteCatalog() { Map.of("default-base-location", "s3://newbucket/"), invalidModifiedStorageConfig); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") + managementApi + .request("v1/catalogs/{cat}", Map.of("cat", catalogName)) .put(Entity.json(badUpdateRequest))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -858,7 +672,8 @@ public void testCreateListUpdateAndDeleteCatalog() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog") + managementApi + .request("v1/catalogs/{cat}", Map.of("cat", catalogName)) .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -872,7 +687,7 @@ public void testCreateListUpdateAndDeleteCatalog() { // 200 GET after update should show new properties try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + managementApi.request("v1/catalogs/{cat}", Map.of("cat", catalogName)).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalog = response.readEntity(Catalog.class); @@ -882,18 +697,18 @@ public void testCreateListUpdateAndDeleteCatalog() { // 204 Successful delete try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { + managementApi.request("v1/catalogs/{cat}", Map.of("cat", catalogName)).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // NOT_FOUND after deletion try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + managementApi.request("v1/catalogs/{cat}", Map.of("cat", catalogName)).get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/catalogs").get()) { + try (Response response = managementApi.request("v1/catalogs").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Catalogs.class)) @@ -906,23 +721,10 @@ public void testCreateListUpdateAndDeleteCatalog() { } } - private static Invocation.Builder newRequest(String url, String token) { - return EXT.client() - .target(String.format(url, EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "Bearer " + token) - .header(REALM_PROPERTY_KEY, realm); - } - - private static Invocation.Builder newRequest(String url) { - return newRequest(url, userToken); - } - @Test public void testGetCatalogNotFound() { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").get()) { + try (Response response = managementApi.request("v1/catalogs/mycatalog").get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } } @@ -940,9 +742,7 @@ public void testGetCatalogInvalidName() { for (String invalidCatalogName : invalidCatalogNames) { // there's no catalog yet. Expect 404 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/" + invalidCatalogName) - .get()) { + try (Response response = managementApi.request("v1/catalogs/" + invalidCatalogName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -954,16 +754,17 @@ public void testGetCatalogInvalidName() { @Test public void testCatalogRoleInvalidName() { + String catalogName = client.newEntityName("mycatalog1"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog1") + .setName(catalogName) .setProperties(new CatalogProperties("s3://required/base/location")) .setStorageConfigInfo( new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidCatalogRoleNames = @@ -976,9 +777,8 @@ public void testCatalogRoleInvalidName() { for (String invalidCatalogRoleName : invalidCatalogRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/" - + invalidCatalogRoleName) + managementApi + .request("v1/catalogs/mycatalog1/catalog-roles/" + invalidCatalogRoleName) .get()) { assertThat(response) @@ -992,23 +792,9 @@ public void testCatalogRoleInvalidName() { @Test public void testListPrincipalsUnauthorized() { - Principal principal = new Principal("new_admin"); - String newToken = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(principal))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials creds = response.readEntity(PrincipalWithCredentials.class); - newToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - creds.getCredentials().getClientId(), - creds.getCredentials().getClientSecret(), - realm); - } - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + PrincipalWithCredentials principal = + managementApi.createPrincipal(client.newEntityName("new_admin")); + try (Response response = client.managementApi(principal).request("v1/principals").get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } @@ -1017,43 +803,31 @@ public void testListPrincipalsUnauthorized() { public void testCreatePrincipalAndRotateCredentials() { Principal principal = Principal.builder() - .setName("myprincipal") + .setName(client.newEntityName("myprincipal")) .setProperties(Map.of("custom-tag", "foo")) .build(); - PrincipalWithCredentialsCredentials creds = null; - Principal returnedPrincipal = null; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, true)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); - } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); - - String oldClientId = creds.getClientId(); - String oldSecret = creds.getClientSecret(); + PrincipalWithCredentials creds = + managementApi.createPrincipal(new CreatePrincipalRequest(principal, true)); + assertThat(creds.getCredentials().getClientId()).isEqualTo(creds.getPrincipal().getClientId()); // Now rotate the credentials. First, if we try to just use the adminToken to rotate the // newly created principal's credentials, we should fail; rotateCredentials is only // a "self" privilege that even admins can't inherit. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal/rotate") + managementApi + .request("v1/principals/{p}/rotate", Map.of("p", principal.getName())) .post(Entity.json(""))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } - // Get a fresh token associate with the principal itself. - String newPrincipalToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), EXT.getLocalPort(), oldClientId, oldSecret, realm); + String oldUserToken = client.obtainToken(creds); // Any call should initially fail with error indicating that rotation is needed. try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal", newPrincipalToken) + client + .managementApi(oldUserToken) + .request("v1/principals/{p}", Map.of("p", principal.getName())) .get()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); ErrorResponse error = response.readEntity(ErrorResponse.class); @@ -1065,21 +839,23 @@ public void testCreatePrincipalAndRotateCredentials() { } // Now try to rotate using the principal's token. + PrincipalWithCredentials newCreds; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal/rotate", - newPrincipalToken) + client + .managementApi(oldUserToken) + .request("v1/principals/{p}/rotate", Map.of("p", principal.getName())) .post(Entity.json(""))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - PrincipalWithCredentials parsed = response.readEntity(PrincipalWithCredentials.class); - creds = parsed.getCredentials(); - returnedPrincipal = parsed.getPrincipal(); + newCreds = response.readEntity(PrincipalWithCredentials.class); } - assertThat(creds.getClientId()).isEqualTo(returnedPrincipal.getClientId()); + assertThat(newCreds.getCredentials().getClientId()) + .isEqualTo(newCreds.getPrincipal().getClientId()); // ClientId shouldn't change - assertThat(creds.getClientId()).isEqualTo(oldClientId); - assertThat(creds.getClientSecret()).isNotEqualTo(oldSecret); + assertThat(newCreds.getCredentials().getClientId()) + .isEqualTo(creds.getCredentials().getClientId()); + assertThat(newCreds.getCredentials().getClientSecret()) + .isNotEqualTo(creds.getCredentials().getClientSecret()); // TODO: Test the validity of the old secret for getting tokens, here and then after a second // rotation that makes the old secret fall off retention. @@ -1089,42 +865,39 @@ public void testCreatePrincipalAndRotateCredentials() { public void testCreateListUpdateAndDeletePrincipal() { Principal principal = Principal.builder() - .setName("myprincipal") + .setName(client.newEntityName("myprincipal")) .setProperties(Map.of("custom-tag", "foo")) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + managementApi.createPrincipal(new CreatePrincipalRequest(principal, null)); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - Principal fetchedPrincipal = null; + Principal fetchedPrincipal; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + managementApi.request("v1/principals/{p}", Map.of("p", principal.getName())).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); - assertThat(fetchedPrincipal.getName()).isEqualTo("myprincipal"); + assertThat(fetchedPrincipal.getName()).isEqualTo(principal.getName()); assertThat(fetchedPrincipal.getProperties()).isEqualTo(Map.of("custom-tag", "foo")); assertThat(fetchedPrincipal.getEntityVersion()).isGreaterThan(0); } // Should list the principal. - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = managementApi.request("v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) .extracting(Principals::getPrincipals) .asInstanceOf(InstanceOfAssertFactories.list(Principal.class)) - .anySatisfy(pr -> assertThat(pr).returns("myprincipal", Principal::getName)); + .anySatisfy(pr -> assertThat(pr).returns(principal.getName(), Principal::getName)); } UpdatePrincipalRequest updateRequest = @@ -1133,7 +906,8 @@ public void testCreateListUpdateAndDeletePrincipal() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal") + managementApi + .request("v1/principals/{p}", Map.of("p", principal.getName())) .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); @@ -1143,33 +917,31 @@ public void testCreateListUpdateAndDeletePrincipal() { // 200 GET after update should show new properties try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + managementApi.request("v1/principals/{p}", Map.of("p", principal.getName())).get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipal = response.readEntity(Principal.class); assertThat(fetchedPrincipal.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); } - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + managementApi.deletePrincipal(principal); // NOT_FOUND after deletion try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal").get()) { + managementApi + .request("v1/principals/{prince}", Map.of("prince", principal.getName())) + .get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = newRequest("http://localhost:%d/api/management/v1/principals").get()) { + try (Response response = managementApi.request("v1/principals").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) .extracting(Principals::getPrincipals) .asInstanceOf(InstanceOfAssertFactories.list(Principal.class)) - .noneSatisfy(pr -> assertThat(pr).returns("myprincipal", Principal::getName)); + .noneSatisfy(pr -> assertThat(pr).returns(principal.getName(), Principal::getName)); } } @@ -1181,11 +953,7 @@ public void testCreatePrincipalWithInvalidName() { .setName(goodName) .setProperties(Map.of("custom-tag", "good_principal")) .build(); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(principal, null)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + managementApi.createPrincipal(new CreatePrincipalRequest(principal, null)); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidPrincipalNames = @@ -1205,7 +973,8 @@ public void testCreatePrincipalWithInvalidName() { .build(); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal, false)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1229,8 +998,7 @@ public void testGetPrincipalWithInvalidName() { for (String invalidPrincipalName : invalidPrincipalNames) { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/" + invalidPrincipalName) - .get()) { + managementApi.request("v1/principals/" + invalidPrincipalName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1243,40 +1011,44 @@ public void testGetPrincipalWithInvalidName() { @Test public void testCreateListUpdateAndDeletePrincipalRole() { PrincipalRole principalRole = - new PrincipalRole("myprincipalrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); - createPrincipalRole(principalRole); + new PrincipalRole( + client.newEntityName("myprincipalrole"), Map.of("custom-tag", "foo"), 0L, 0L, 1); + managementApi.createPrincipalRole(principalRole); // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + managementApi + .request("v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - PrincipalRole fetchedPrincipalRole = null; + PrincipalRole fetchedPrincipalRole; try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + managementApi + .request("v1/principal-roles/{pr}", Map.of("pr", principalRole.getName())) + .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); - assertThat(fetchedPrincipalRole.getName()).isEqualTo("myprincipalrole"); + assertThat(fetchedPrincipalRole.getName()).isEqualTo(principalRole.getName()); assertThat(fetchedPrincipalRole.getProperties()).isEqualTo(Map.of("custom-tag", "foo")); assertThat(fetchedPrincipalRole.getEntityVersion()).isGreaterThan(0); } // Should list the principalRole. - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = managementApi.request("v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(PrincipalRoles.class)) .extracting(PrincipalRoles::getRoles) .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) - .anySatisfy(pr -> assertThat(pr).returns("myprincipalrole", PrincipalRole::getName)); + .anySatisfy( + pr -> assertThat(pr).returns(principalRole.getName(), PrincipalRole::getName)); } UpdatePrincipalRoleRequest updateRequest = @@ -1285,7 +1057,8 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 successful update try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") + managementApi + .request("v1/principal-roles/{pr}", Map.of("pr", principalRole.getName())) .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); @@ -1295,38 +1068,35 @@ public void testCreateListUpdateAndDeletePrincipalRole() { // 200 GET after update should show new properties try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { + managementApi + .request("v1/principal-roles/{pr}", Map.of("pr", principalRole.getName())) + .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedPrincipalRole = response.readEntity(PrincipalRole.class); assertThat(fetchedPrincipalRole.getProperties()).isEqualTo(Map.of("custom-tag", "updated")); } - // 204 Successful delete - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + managementApi.deletePrincipalRole(principalRole); // NOT_FOUND after deletion try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole").get()) { - + managementApi + .request("v1/principal-roles/{pr}", Map.of("pr", principalRole.getName())) + .get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } // Empty list - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles").get()) { + try (Response response = managementApi.request("v1/principal-roles").get()) { assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(PrincipalRoles.class)) .extracting(PrincipalRoles::getRoles) .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) - .noneSatisfy(pr -> assertThat(pr).returns("myprincipalrole", PrincipalRole::getName)); + .noneSatisfy( + pr -> assertThat(pr).returns(principalRole.getName(), PrincipalRole::getName)); } } @@ -1335,7 +1105,7 @@ public void testCreatePrincipalRoleInvalidName() { String goodName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH, true, true); PrincipalRole principalRole = new PrincipalRole(goodName, Map.of("custom-tag", "good_principal_role"), 0L, 0L, 1); - createPrincipalRole(principalRole); + managementApi.createPrincipalRole(principalRole); String longInvalidName = RandomStringUtils.random(MAX_IDENTIFIER_LENGTH + 1, true, true); List invalidPrincipalRoleNames = @@ -1353,7 +1123,8 @@ public void testCreatePrincipalRoleInvalidName() { invalidPrincipalRoleName, Map.of("custom-tag", "bad_principal_role"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") + managementApi + .request("v1/principal-roles") .post(Entity.json(new CreatePrincipalRoleRequest(principalRole)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); @@ -1377,10 +1148,7 @@ public void testGetPrincipalRoleInvalidName() { for (String invalidPrincipalRoleName : invalidPrincipalRoleNames) { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + invalidPrincipalRoleName) - .get()) { + managementApi.request("v1/principal-roles/" + invalidPrincipalRoleName).get()) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus); assertThat(response.hasEntity()).isTrue(); @@ -1392,32 +1160,35 @@ public void testGetPrincipalRoleInvalidName() { @Test public void testCreateListUpdateAndDeleteCatalogRole() { + String catalogName = client.newEntityName("mycatalog1"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog1") + .setName(catalogName) .setProperties(new CatalogProperties("s3://required/base/location")) .setStorageConfigInfo( new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); + String catalogName2 = client.newEntityName("mycatalog2"); Catalog catalog2 = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog2") + .setName(catalogName2) .setStorageConfigInfo( new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://required/base/other_location")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); CatalogRole catalogRole = new CatalogRole("mycatalogrole", Map.of("custom-tag", "foo"), 0L, 0L, 1); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1425,17 +1196,18 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Second attempt to create the same entity should fail with CONFLICT. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CONFLICT.getStatusCode(), Response::getStatus); } // 200 successful GET after creation - CatalogRole fetchedCatalogRole = null; + CatalogRole fetchedCatalogRole; try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/{cat}/catalog-roles/mycatalogrole", Map.of("cat", catalogName)) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); @@ -1448,7 +1220,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Should list the catalogRole. try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) .get()) { assertThat(response) @@ -1461,7 +1234,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Empty list if listing in catalog2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName2)) .get()) { assertThat(response) @@ -1483,8 +1257,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 successful update try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/{cat}/catalog-roles/mycatalogrole", Map.of("cat", catalogName)) .put(Entity.json(updateRequest))) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1494,8 +1268,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 200 GET after update should show new properties try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/{cat}/catalog-roles/mycatalogrole", Map.of("cat", catalogName)) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); fetchedCatalogRole = response.readEntity(CatalogRole.class); @@ -1505,8 +1279,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 204 Successful delete try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/{cat}/catalog-roles/mycatalogrole", Map.of("cat", catalogName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1514,8 +1288,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // NOT_FOUND after deletion try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles/mycatalogrole") + managementApi + .request("v1/catalogs/{cat}/catalog-roles/mycatalogrole", Map.of("cat", catalogName)) .get()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); @@ -1523,7 +1297,8 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalogName)) .get()) { assertThat(response) @@ -1536,15 +1311,13 @@ public void testCreateListUpdateAndDeleteCatalogRole() { // 204 Successful delete mycatalog try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog1").delete()) { - + managementApi.request("v1/catalogs/{cat}", Map.of("cat", catalogName)).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete mycatalog2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog2").delete()) { - + managementApi.request("v1/catalogs/{cat}", Map.of("cat", catalogName2)).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -1552,37 +1325,42 @@ public void testCreateListUpdateAndDeleteCatalogRole() { @Test public void testAssignListAndRevokePrincipalRoles() { // Create two Principals - Principal principal1 = new Principal("myprincipal1"); + Principal principal1 = new Principal(client.newEntityName("myprincipal1")); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal1, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } - Principal principal2 = new Principal("myprincipal2"); + Principal principal2 = new Principal(client.newEntityName("myprincipal2")); try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") + managementApi + .request("v1/principals") .post(Entity.json(new CreatePrincipalRequest(principal2, false)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // One PrincipalRole - PrincipalRole principalRole = new PrincipalRole("myprincipalrole"); - createPrincipalRole(principalRole); + PrincipalRole principalRole = new PrincipalRole(client.newEntityName("myprincipalrole")); + managementApi.createPrincipalRole(principalRole); // Assign the role to myprincipal1 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + managementApi + .request( + "v1/principals/{prince}/principal-roles", Map.of("prince", principal1.getName())) .put(Entity.json(principalRole))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } // Should list myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + managementApi + .request( + "v1/principals/{prince}/principal-roles", Map.of("prince", principal1.getName())) .get()) { assertThat(response) @@ -1592,13 +1370,13 @@ public void testAssignListAndRevokePrincipalRoles() { .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) .hasSize(1) .satisfiesExactly( - pr -> assertThat(pr).returns("myprincipalrole", PrincipalRole::getName)); + pr -> assertThat(pr).returns(principalRole.getName(), PrincipalRole::getName)); } // Should list myprincipal1 if listing assignees of myprincipalrole try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") + managementApi + .request("v1/principal-roles/{pr}/principals", Map.of("pr", principalRole.getName())) .get()) { assertThat(response) @@ -1607,12 +1385,14 @@ public void testAssignListAndRevokePrincipalRoles() { .extracting(Principals::getPrincipals) .asInstanceOf(InstanceOfAssertFactories.list(Principal.class)) .hasSize(1) - .satisfiesExactly(pr -> assertThat(pr).returns("myprincipal1", Principal::getName)); + .satisfiesExactly(pr -> assertThat(pr).returns(principal1.getName(), Principal::getName)); } // Empty list if listing in principal2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2/principal-roles") + managementApi + .request( + "v1/principals/{prince}/principal-roles", Map.of("prince", principal2.getName())) .get()) { assertThat(response) @@ -1623,8 +1403,10 @@ public void testAssignListAndRevokePrincipalRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles/myprincipalrole") + managementApi + .request( + "v1/principals/{prince}/principal-roles/{pr}", + Map.of("prince", principal1.getName(), "pr", principalRole.getName())) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1632,7 +1414,9 @@ public void testAssignListAndRevokePrincipalRoles() { // Empty list try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1/principal-roles") + managementApi + .request( + "v1/principals/{prince}/principal-roles", Map.of("prince", principal1.getName())) .get()) { assertThat(response) @@ -1641,10 +1425,9 @@ public void testAssignListAndRevokePrincipalRoles() { .returns(List.of(), PrincipalRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/myprincipalrole/principals") + managementApi + .request("v1/principal-roles/{pr}/principals", Map.of("pr", principalRole.getName())) .get()) { - assertThat(response) .returns(Response.Status.OK.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(Principals.class)) @@ -1653,23 +1436,26 @@ public void testAssignListAndRevokePrincipalRoles() { // 204 Successful delete myprincipal1 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal1").delete()) { - + managementApi + .request("v1/principals/{prince}", Map.of("prince", principal1.getName())) + .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipal2 try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals/myprincipal2").delete()) { + managementApi + .request("v1/principals/{prince}", Map.of("prince", principal2.getName())) + .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // 204 Successful delete myprincipalrole try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/myprincipalrole") + managementApi + .request("v1/principal-roles/{pr}", Map.of("pr", principalRole.getName())) .delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -1677,27 +1463,28 @@ public void testAssignListAndRevokePrincipalRoles() { @Test public void testAssignListAndRevokeCatalogRoles() { // Create two PrincipalRoles - PrincipalRole principalRole1 = new PrincipalRole("mypr1"); - createPrincipalRole(principalRole1); + PrincipalRole principalRole1 = new PrincipalRole(client.newEntityName("mypr1")); + managementApi.createPrincipalRole(principalRole1); - PrincipalRole principalRole2 = new PrincipalRole("mypr2"); - createPrincipalRole(principalRole2); + PrincipalRole principalRole2 = new PrincipalRole(client.newEntityName("mypr2")); + managementApi.createPrincipalRole(principalRole2); // One CatalogRole Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("mycatalog") + .setName(client.newEntityName("mycatalog")) .setStorageConfigInfo( new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); CatalogRole catalogRole = new CatalogRole("mycr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", catalog.getName())) .post(Entity.json(new CreateCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1707,17 +1494,18 @@ public void testAssignListAndRevokeCatalogRoles() { Catalog otherCatalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName("othercatalog") + .setName(client.newEntityName("othercatalog")) .setProperties(new CatalogProperties("s3://path/to/data")) .setStorageConfigInfo( new AwsStorageConfigInfo( "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .build(); - createCatalog(otherCatalog); + managementApi.createCatalog(otherCatalog); CatalogRole otherCatalogRole = new CatalogRole("myothercr"); try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles") + managementApi + .request("v1/catalogs/{cat}/catalog-roles", Map.of("cat", otherCatalog.getName())) .post(Entity.json(new CreateCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1725,15 +1513,19 @@ public void testAssignListAndRevokeCatalogRoles() { // Assign both the roles to mypr1 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + managementApi + .request( + "v1/principal-roles/{pr}/catalog-roles/{cat}", + Map.of("pr", principalRole1.getName(), "cat", catalog.getName())) .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/othercatalog") + managementApi + .request( + "v1/principal-roles/{pr}/catalog-roles/{cat}", + Map.of("pr", principalRole1.getName(), "cat", otherCatalog.getName())) .put(Entity.json(new GrantCatalogRoleRequest(otherCatalogRole)))) { assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); @@ -1741,8 +1533,10 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list only mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + managementApi + .request( + "v1/principal-roles/{pr}/catalog-roles/{cat}", + Map.of("pr", principalRole1.getName(), "cat", catalog.getName())) .get()) { assertThat(response) @@ -1756,8 +1550,10 @@ public void testAssignListAndRevokeCatalogRoles() { // Should list mypr1 if listing assignees of mycr try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") + managementApi + .request( + "v1/catalogs/{cat}/catalog-roles/mycr/principal-roles", + Map.of("cat", catalog.getName())) .get()) { assertThat(response) @@ -1766,13 +1562,16 @@ public void testAssignListAndRevokeCatalogRoles() { .extracting(PrincipalRoles::getRoles) .asInstanceOf(InstanceOfAssertFactories.list(PrincipalRole.class)) .hasSize(1) - .satisfiesExactly(pr -> assertThat(pr).returns("mypr1", PrincipalRole::getName)); + .satisfiesExactly( + pr -> assertThat(pr).returns(principalRole1.getName(), PrincipalRole::getName)); } // Empty list if listing in principalRole2 try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr2/catalog-roles/mycatalog") + managementApi + .request( + "v1/principal-roles/{pr}/catalog-roles/{cat}", + Map.of("pr", principalRole2.getName(), "cat", catalog.getName())) .get()) { assertThat(response) @@ -1783,8 +1582,10 @@ public void testAssignListAndRevokeCatalogRoles() { // 204 Successful revoke try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog/mycr") + managementApi + .request( + "v1/principal-roles/{pr}/catalog-roles/{cat}/mycr", + Map.of("pr", principalRole1.getName(), "cat", catalog.getName())) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1792,8 +1593,10 @@ public void testAssignListAndRevokeCatalogRoles() { // Empty list try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/mypr1/catalog-roles/mycatalog") + managementApi + .request( + "v1/principal-roles/{pr}/catalog-roles/{cat}", + Map.of("pr", principalRole1.getName(), "cat", catalog.getName())) .get()) { assertThat(response) @@ -1802,8 +1605,10 @@ public void testAssignListAndRevokeCatalogRoles() { .returns(List.of(), CatalogRoles::getRoles); } try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr/principal-roles") + managementApi + .request( + "v1/catalogs/{cat}/catalog-roles/mycr/principal-roles", + Map.of("cat", catalog.getName())) .get()) { assertThat(response) @@ -1812,61 +1617,24 @@ public void testAssignListAndRevokeCatalogRoles() { .returns(List.of(), PrincipalRoles::getRoles); } - // 204 Successful delete mypr1 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr1").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete mypr2 - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles/mypr2").delete()) { + managementApi.deletePrincipalRole(principalRole1); + managementApi.deletePrincipalRole(principalRole2); - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + managementApi.deleteCatalogRole(catalog.getName(), "mycr"); + managementApi.deleteCatalog(catalog.getName()); - // 204 Successful delete mycr - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog/catalog-roles/mycr") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete mycatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/mycatalog").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete myothercr - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/othercatalog/catalog-roles/myothercr") - .delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } - - // 204 Successful delete othercatalog - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs/othercatalog").delete()) { - - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + managementApi.deleteCatalogRole(otherCatalog.getName(), otherCatalogRole.getName()); + managementApi.deleteCatalog(otherCatalog.getName()); } @Test public void testCatalogAdminGrantAndRevokeCatalogRoles() { // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal // role - String principalRoleName = "mypr33"; - PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + String principalRoleName = client.newEntityName("mypr33"); + managementApi.createPrincipalRole(principalRoleName); - String catalogName = "myuniquetestcatalog"; + String catalogName = client.newEntityName("myuniquetestcatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -1876,56 +1644,55 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); + PrincipalWithCredentials catalogAdminPrincipal = + managementApi.createPrincipal(client.newEntityName("principal1")); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRoleName); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String catalogAdminToken = client.obtainToken(catalogAdminPrincipal); // Create a second principal role. Use the catalog admin principal to list principal roles and // grant a catalog role to the new principal role String principalRoleName2 = "mypr2"; PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // create a catalog role and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName, catalogRoleName, catalogAdminToken); + client.managementApi(catalogAdminToken).createCatalogRole(catalogName, catalogRoleName); CatalogPrivilege privilege = CatalogPrivilege.CATALOG_MANAGE_CONTENT; - grantPrivilegeToCatalogRole( - catalogName, - catalogRoleName, - new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG), - catalogAdminToken, - Response.Status.CREATED); + client + .managementApi(catalogAdminToken) + .addGrant( + catalogName, + catalogRoleName, + new CatalogGrant(privilege, GrantResource.TypeEnum.CATALOG)); // The catalog admin can grant the new catalog role to the mypr2 principal role - grantCatalogRoleToPrincipalRole( - principalRoleName2, catalogName, new CatalogRole(catalogRoleName), catalogAdminToken); + client + .managementApi(catalogAdminToken) + .grantCatalogRoleToPrincipalRole( + principalRoleName2, catalogName, new CatalogRole(catalogRoleName)); // But the catalog admin cannot revoke the role because it requires // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + client + .managementApi(catalogAdminToken) + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" - + catalogRoleName, - catalogAdminToken) + + catalogRoleName) .delete()) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } @@ -1933,14 +1700,14 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { // The service admin can revoke the role because it has the // PRINCIPAL_ROLE_MANAGE_GRANTS_FOR_GRANTEE privilege try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + managementApi + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName + "/" - + catalogRoleName, - userToken) + + catalogRoleName) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1950,11 +1717,11 @@ public void testCatalogAdminGrantAndRevokeCatalogRoles() { public void testServiceAdminCanTransferCatalogAdmin() { // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal // role - String principalRoleName = "mypr33"; + String principalRoleName = client.newEntityName("mypr33"); PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); - String catalogName = "myothertestcatalog"; + String catalogName = client.newEntityName("myothertestcatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -1964,31 +1731,27 @@ public void testServiceAdminCanTransferCatalogAdmin() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); + PrincipalWithCredentials catalogAdminPrincipal = + managementApi.createPrincipal(client.newEntityName("principal1")); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String catalogAdminToken = client.obtainToken(catalogAdminPrincipal); // service_admin revokes the catalog_admin privilege from its principal role try { try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/service_admin/catalog-roles/" + managementApi + .request( + "v1/principal-roles/service_admin/catalog-roles/" + catalogName - + "/catalog_admin", - userToken) + + "/catalog_admin") .delete()) { assertThat(response) .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); @@ -1996,21 +1759,23 @@ public void testServiceAdminCanTransferCatalogAdmin() { // the service_admin can not revoke the catalog_admin privilege from the new principal role try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" + client + .managementApi(catalogAdminToken) + .request( + "v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName - + "/catalog_admin", - catalogAdminToken) + + "/catalog_admin") .delete()) { assertThat(response) .returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } } finally { // grant the admin role back to service_admin so that cleanup can happen - grantCatalogRoleToPrincipalRole( - "service_admin", catalogName, catalogAdminRole, catalogAdminToken); + client + .managementApi(catalogAdminToken) + .grantCatalogRoleToPrincipalRole("service_admin", catalogName, catalogAdminRole); } } @@ -2018,12 +1783,12 @@ public void testServiceAdminCanTransferCatalogAdmin() { public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { // Create a PrincipalRole and a new catalog. Grant the catalog_admin role to the new principal // role - String principalRoleName = "mypr33"; + String principalRoleName = client.newEntityName("mypr33"); PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); // create a catalog - String catalogName = "mytestcatalog"; + String catalogName = client.newEntityName("mytestcatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -2033,10 +1798,10 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a second catalog - String catalogName2 = "anothercatalog"; + String catalogName2 = client.newEntityName("anothercatalog"); Catalog catalog2 = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -2046,42 +1811,35 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); // create a catalog role *in the second catalog* and grant it manage_content privilege String catalogRoleName = "mycr1"; - createCatalogRole(catalogName2, catalogRoleName, userToken); + managementApi.createCatalogRole(catalogName2, catalogRoleName); // Get the catalog admin role from the *first* catalog and grant that role to the principal role - CatalogRole catalogAdminRole = readCatalogRole(catalogName, "catalog_admin"); - grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole, userToken); + CatalogRole catalogAdminRole = managementApi.getCatalogRole(catalogName, "catalog_admin"); + managementApi.grantCatalogRoleToPrincipalRole(principalRoleName, catalogName, catalogAdminRole); // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("principal1"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + PrincipalWithCredentials catalogAdminPrincipal = + managementApi.createPrincipal(client.newEntityName("principal1")); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String catalogAdminToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String catalogAdminToken = client.obtainToken(catalogAdminPrincipal); // Create a second principal role. - String principalRoleName2 = "mypr2"; + String principalRoleName2 = client.newEntityName("mypr2"); PrincipalRole principalRole2 = new PrincipalRole(principalRoleName2); - createPrincipalRole(principalRole2); + managementApi.createPrincipalRole(principalRole2); // The catalog admin cannot grant the new catalog role to the mypr2 principal role because the // catalog role is in the wrong catalog try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName2, - catalogAdminToken) + client + .managementApi(catalogAdminToken) + .request("v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName2) .put(Entity.json(new GrantCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } @@ -2090,12 +1848,12 @@ public void testCatalogAdminGrantAndRevokeCatalogRolesFromWrongCatalog() { @Test public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // Create a PrincipalRole and a new catalog. - String principalRoleName = "mypr33"; + String principalRoleName = client.newEntityName("mypr33"); PrincipalRole principalRole1 = new PrincipalRole(principalRoleName); - createPrincipalRole(principalRole1); + managementApi.createPrincipalRole(principalRole1); // create a catalog - String catalogName = "mytablemanagecatalog"; + String catalogName = client.newEntityName("mytablemanagecatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -2105,13 +1863,13 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a valid target CatalogRole in this catalog - createCatalogRole(catalogName, "target_catalog_role", userToken); + managementApi.createCatalogRole(catalogName, "target_catalog_role"); // create a second catalog - String catalogName2 = "anothertablemanagecatalog"; + String catalogName2 = client.newEntityName("anothertablemanagecatalog"); Catalog catalog2 = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -2121,59 +1879,53 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog2); + managementApi.createCatalog(catalog2); // create an *invalid* target CatalogRole in second catalog - createCatalogRole(catalogName2, "invalid_target_catalog_role", userToken); + managementApi.createCatalogRole(catalogName2, "invalid_target_catalog_role"); // create the namespace "c" in *both* namespaces String namespaceName = "c"; - createNamespace(catalogName, namespaceName); - createNamespace(catalogName2, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName2, namespaceName); // create a catalog role *in the first catalog* and grant it manage_content privilege at the // namespace level // grant that role to the PrincipalRole String catalogRoleName = "ns_manage_access_role"; - createCatalogRole(catalogName, catalogRoleName, userToken); - grantPrivilegeToCatalogRole( + managementApi.createCatalogRole(catalogName, catalogRoleName); + managementApi.addGrant( catalogName, catalogRoleName, new NamespaceGrant( List.of(namespaceName), NamespacePrivilege.CATALOG_MANAGE_ACCESS, - GrantResource.TypeEnum.NAMESPACE), - userToken, - Response.Status.CREATED); + GrantResource.TypeEnum.NAMESPACE)); - grantCatalogRoleToPrincipalRole( - principalRoleName, catalogName, new CatalogRole(catalogRoleName), userToken); + managementApi.grantCatalogRoleToPrincipalRole( + principalRoleName, catalogName, new CatalogRole(catalogRoleName)); // Create a principal and grant the principal role to it - PrincipalWithCredentials catalogAdminPrincipal = createPrincipal("ns_manage_access_user"); - grantPrincipalRoleToPrincipal(catalogAdminPrincipal.getPrincipal().getName(), principalRole1); + PrincipalWithCredentials catalogAdminPrincipal = + managementApi.createPrincipal(client.newEntityName("ns_manage_access_user")); + managementApi.assignPrincipalRole( + catalogAdminPrincipal.getPrincipal().getName(), principalRole1.getName()); - String manageAccessUserToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - catalogAdminPrincipal.getCredentials().getClientId(), - catalogAdminPrincipal.getCredentials().getClientSecret(), - realm); + String manageAccessUserToken = client.obtainToken(catalogAdminPrincipal); // Use the ns_manage_access_user to grant TABLE_CREATE access to the target catalog role // This works because the user has CATALOG_MANAGE_ACCESS within the namespace and the target // catalog role is in // the same catalog - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.CREATED); + client + .managementApi(manageAccessUserToken) + .addGrant( + catalogName, + "target_catalog_role", + new NamespaceGrant( + List.of(namespaceName), + NamespacePrivilege.TABLE_CREATE, + GrantResource.TypeEnum.NAMESPACE)); // Even though the ns_manage_access_role can grant privileges to the catalog role, it cannot // grant the target @@ -2181,45 +1933,59 @@ public void testTableManageAccessCanGrantAndRevokeFromCatalogRoles() { // on the catalog role // as a securable try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - manageAccessUserToken) + client + .managementApi(manageAccessUserToken) + .request("v1/principal-roles/" + principalRoleName + "/catalog-roles/" + catalogName) .put( Entity.json(new GrantCatalogRoleRequest(new CatalogRole("target_catalog_role"))))) { assertThat(response).returns(Response.Status.FORBIDDEN.getStatusCode(), Response::getStatus); } // The user cannot grant catalog-level privileges to the catalog role - grantPrivilegeToCatalogRole( - catalogName, - "target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + client + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName, "role", "target_catalog_role")) + .put( + Entity.json( + new CatalogGrant( + CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } // even though the namespace "c" exists in both catalogs, the ns_manage_access_role can only // grant privileges for // the namespace in its own catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new NamespaceGrant( - List.of(namespaceName), - NamespacePrivilege.TABLE_CREATE, - GrantResource.TypeEnum.NAMESPACE), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + client + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName2, "role", "invalid_target_catalog_role")) + .put( + Entity.json( + new NamespaceGrant( + List.of(namespaceName), + NamespacePrivilege.TABLE_CREATE, + GrantResource.TypeEnum.NAMESPACE)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } // nor can it grant privileges to the catalog role in the second catalog - grantPrivilegeToCatalogRole( - catalogName2, - "invalid_target_catalog_role", - new CatalogGrant(CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG), - manageAccessUserToken, - Response.Status.FORBIDDEN); + try (Response response = + client + .managementApi(manageAccessUserToken) + .request( + "v1/catalogs/{cat}/catalog-roles/{role}/grants", + Map.of("cat", catalogName2, "role", "invalid_target_catalog_role")) + .put( + Entity.json( + new CatalogGrant( + CatalogPrivilege.TABLE_CREATE, GrantResource.TypeEnum.CATALOG)))) { + assertThat(response).returns(FORBIDDEN.getStatusCode(), Response::getStatus); + } } @Test @@ -2230,13 +1996,13 @@ public void testTokenExpiry() { .withExpiresAt(Instant.now().plus(1, ChronoUnit.SECONDS)) .sign(Algorithm.HMAC256("polaris")); Awaitility.await("expected list of records should be produced") - .atMost(Duration.ofSeconds(2)) + .atMost(Duration.ofSeconds(20)) .pollDelay(Duration.ofSeconds(1)) .pollInterval(Duration.ofSeconds(1)) .untilAsserted( () -> { try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2248,8 +2014,7 @@ public void testTokenInactive() { // InvalidClaimException - if a claim contained a different value than the expected one. String newToken = defaultJwt().withClaim(CLAIM_KEY_ACTIVE, false).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2259,8 +2024,7 @@ public void testTokenInactive() { public void testTokenInvalidSignature() { // SignatureVerificationException - if the signature is invalid. String newToken = defaultJwt().sign(Algorithm.HMAC256("invalid_secret")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2270,8 +2034,7 @@ public void testTokenInvalidSignature() { public void testTokenInvalidPrincipalId() { String newToken = defaultJwt().withClaim(CLAIM_KEY_PRINCIPAL_ID, 0).sign(Algorithm.HMAC256("polaris")); - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals", newToken).get()) { + try (Response response = client.managementApi(newToken).request("v1/principals").get()) { assertThat(response) .returns(Response.Status.UNAUTHORIZED.getStatusCode(), Response::getStatus); } @@ -2280,7 +2043,7 @@ public void testTokenInvalidPrincipalId() { @Test public void testNamespaceExistsStatus() { // create a catalog - String catalogName = "mytablemanagecatalog"; + String catalogName = client.newEntityName("mytablemanagecatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -2290,21 +2053,15 @@ public void testNamespaceExistsStatus() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a namespace String namespaceName = "c"; - createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); // check if a namespace existed try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) - .head()) { + catalogApi.request("v1/" + catalogName + "/namespaces/" + namespaceName).head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -2312,7 +2069,7 @@ public void testNamespaceExistsStatus() { @Test public void testDropNamespaceStatus() { // create a catalog - String catalogName = "mytablemanagecatalog"; + String catalogName = client.newEntityName("mytablemanagecatalog"); Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -2322,21 +2079,15 @@ public void testDropNamespaceStatus() { "arn:aws:iam::012345678901:role/jdoe", StorageConfigInfo.StorageTypeEnum.S3)) .setProperties(new CatalogProperties("s3://bucket1/")) .build(); - createCatalog(catalog); + managementApi.createCatalog(catalog); // create a namespace String namespaceName = "c"; - createNamespace(catalogName, namespaceName); + catalogApi.createNamespace(catalogName, namespaceName); // drop a namespace try (Response response = - newRequest( - "http://localhost:%d/api/catalog/v1/" - + catalogName - + "/namespaces/" - + namespaceName, - userToken) - .delete()) { + catalogApi.request("v1/" + catalogName + "/namespaces/" + namespaceName).delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } } @@ -2350,119 +2101,8 @@ public static JWTCreator.Builder defaultJwt() { .withExpiresAt(now.plus(10, ChronoUnit.SECONDS)) .withJWTId(UUID.randomUUID().toString()) .withClaim(CLAIM_KEY_ACTIVE, true) - .withClaim(CLAIM_KEY_CLIENT_ID, clientId) + .withClaim(CLAIM_KEY_CLIENT_ID, rootCredentials.clientId()) .withClaim(CLAIM_KEY_PRINCIPAL_ID, 1) - .withClaim(CLAIM_KEY_SCOPE, BasePolarisAuthenticator.PRINCIPAL_ROLE_ALL); - } - - private static void createNamespace(String catalogName, String namespaceName) { - try (Response response = - newRequest("http://localhost:%d/api/catalog/v1/" + catalogName + "/namespaces", userToken) - .post( - Entity.json( - CreateNamespaceRequest.builder() - .withNamespace(Namespace.of(namespaceName)) - .build()))) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalog(Catalog catalog) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/catalogs") - .post(Entity.json(new CreateCatalogRequest(catalog)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void grantPrivilegeToCatalogRole( - String catalogName, - String catalogRoleName, - GrantResource grant, - String catalogAdminToken, - Response.Status expectedStatus) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + catalogRoleName - + "/grants", - catalogAdminToken) - .put(Entity.json(new AddGrantRequest(grant)))) { - assertThat(response).returns(expectedStatus.getStatusCode(), Response::getStatus); - } - } - - private static void createCatalogRole( - String catalogName, String catalogRoleName, String catalogAdminToken) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName + "/catalog-roles", - catalogAdminToken) - .post(Entity.json(new CreateCatalogRoleRequest(new CatalogRole(catalogRoleName))))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static void grantPrincipalRoleToPrincipal( - String principalName, PrincipalRole principalRole) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principals/" - + principalName - + "/principal-roles") - .put(Entity.json(new GrantPrincipalRoleRequest(principalRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static PrincipalWithCredentials createPrincipal(String principalName) { - PrincipalWithCredentials catalogAdminPrincipal; - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principals") - .post(Entity.json(new CreatePrincipalRequest(new Principal(principalName), false)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - catalogAdminPrincipal = response.readEntity(PrincipalWithCredentials.class); - } - return catalogAdminPrincipal; - } - - private static void grantCatalogRoleToPrincipalRole( - String principalRoleName, String catalogName, CatalogRole catalogRole, String token) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/principal-roles/" - + principalRoleName - + "/catalog-roles/" - + catalogName, - token) - .put(Entity.json(new GrantCatalogRoleRequest(catalogRole)))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private static CatalogRole readCatalogRole(String catalogName, String roleName) { - try (Response response = - newRequest( - "http://localhost:%d/api/management/v1/catalogs/" - + catalogName - + "/catalog-roles/" - + roleName) - .get()) { - - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - return response.readEntity(CatalogRole.class); - } - } - - private static void createPrincipalRole(PrincipalRole principalRole1) { - try (Response response = - newRequest("http://localhost:%d/api/management/v1/principal-roles") - .post(Entity.json(new CreatePrincipalRoleRequest(principalRole1)))) { - - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + .withClaim(CLAIM_KEY_SCOPE, PRINCIPAL_ROLE_ALL); } } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java similarity index 62% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 948f6d581..64f3426c6 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -16,22 +16,19 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.google.common.collect.ImmutableMap; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.reflect.Method; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,35 +61,32 @@ import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogGrant; import org.apache.polaris.core.admin.model.CatalogPrivilege; -import org.apache.polaris.core.admin.model.CatalogRole; +import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GrantResource; import org.apache.polaris.core.admin.model.GrantResources; import org.apache.polaris.core.admin.model.NamespaceGrant; import org.apache.polaris.core.admin.model.NamespacePrivilege; import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.admin.model.TableGrant; import org.apache.polaris.core.admin.model.TablePrivilege; -import org.apache.polaris.core.admin.model.UpdateCatalogRequest; import org.apache.polaris.core.admin.model.ViewGrant; import org.apache.polaris.core.admin.model.ViewPrivilege; import org.apache.polaris.core.entity.CatalogEntity; import org.apache.polaris.core.entity.PolarisEntityConstants; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.auth.TokenUtils; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension.PolarisToken; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension; -import org.apache.polaris.service.dropwizard.test.SnowmanCredentialsExtension.SnowmanCredentials; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.assertj.core.api.Assertions; import org.assertj.core.api.InstanceOfAssertFactories; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -103,12 +97,7 @@ * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog * client. */ -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class, - SnowmanCredentialsExtension.class -}) +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisRestCatalogIntegrationTest extends CatalogTests { private static final String TEST_ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) @@ -116,22 +105,18 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static final String S3_BUCKET_BASE = Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH")) .orElse("file:///tmp/buckets/my-bucket"); - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; + private static String principalRoleName; + private static ClientCredentials adminCredentials; + private static PrincipalWithCredentials principalCredentials; + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ManagementApi managementApi; + private static CatalogApi catalogApi; private RESTCatalog restCatalog; private String currentCatalogName; - private String userToken; - private String realm; private final String catalogBaseLocation = S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data"; @@ -157,93 +142,92 @@ String[] properties() default { } @BeforeAll - public static void setup(@PolarisRealm String realm) throws IOException { - // Set up test location - PolarisConnectionExtension.createTestDir(realm); + static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + adminCredentials = credentials; + endpoints = apiEndpoints; + client = polarisClient(endpoints); + managementApi = client.managementApi(credentials); + String principalName = client.newEntityName("snowman-rest"); + principalRoleName = client.newEntityName("rest-admin"); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); + catalogApi = client.catalogApi(principalCredentials); + } + + @AfterAll + static void close() throws Exception { + client.close(); } @BeforeEach - public void before( - TestInfo testInfo, - PolarisToken adminToken, - SnowmanCredentials snowmanCredentials, - @PolarisRealm String realm) { - this.realm = realm; - userToken = - TokenUtils.getTokenFromSecrets( - EXT.client(), - EXT.getLocalPort(), - snowmanCredentials.clientId(), - snowmanCredentials.clientSecret(), - realm); - testInfo - .getTestMethod() - .ifPresent( - method -> { - currentCatalogName = method.getName() + UUID.randomUUID(); - AwsStorageConfigInfo awsConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn(TEST_ROLE_ARN) - .setExternalId("externalId") - .setUserArn("a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) - .build(); - Optional catalogConfig = - testInfo - .getTestMethod() - .flatMap(m -> Optional.ofNullable(m.getAnnotation(CatalogConfig.class))); - - org.apache.polaris.core.admin.model.CatalogProperties.Builder catalogPropsBuilder = - org.apache.polaris.core.admin.model.CatalogProperties.builder( - catalogBaseLocation); - String[] properties = - catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); - for (int i = 0; i < properties.length; i += 2) { - catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); - } - if (!S3_BUCKET_BASE.startsWith("file:/")) { - catalogPropsBuilder.addProperty( - CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); - } - Catalog catalog = - PolarisCatalog.builder() - .setType( - catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) - .setName(currentCatalogName) - .setProperties(catalogPropsBuilder.build()) - .setStorageConfigInfo( - S3_BUCKET_BASE.startsWith("file:/") - ? new FileStorageConfigInfo( - StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) - : awsConfigModel) - .build(); - - Optional restCatalogConfig = - testInfo - .getTestMethod() - .flatMap( - m -> - Optional.ofNullable( - m.getAnnotation( - PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); - ImmutableMap.Builder extraPropertiesBuilder = - ImmutableMap.builder(); - restCatalogConfig.ifPresent( - config -> { - for (int i = 0; i < config.value().length; i += 2) { - extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); - } - }); - restCatalog = - TestUtil.createSnowmanManagedCatalog( - EXT, - adminToken, - snowmanCredentials, - realm, - catalog, - extraPropertiesBuilder.build()); - }); + public void before(TestInfo testInfo) { + String principalName = "snowman-rest-" + UUID.randomUUID(); + principalRoleName = "rest-admin-" + UUID.randomUUID(); + PrincipalWithCredentials principalCredentials = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + + catalogApi = client.catalogApi(principalCredentials); + + Method method = testInfo.getTestMethod().orElseThrow(); + currentCatalogName = client.newEntityName(method.getName()); + AwsStorageConfigInfo awsConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn(TEST_ROLE_ARN) + .setExternalId("externalId") + .setUserArn("a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://my-old-bucket/path/to/data")) + .build(); + Optional catalogConfig = + Optional.ofNullable(method.getAnnotation(CatalogConfig.class)); + + CatalogProperties.Builder catalogPropsBuilder = CatalogProperties.builder(catalogBaseLocation); + String[] properties = + catalogConfig.map(CatalogConfig::properties).orElse(DEFAULT_CATALOG_PROPERTIES); + for (int i = 0; i < properties.length; i += 2) { + catalogPropsBuilder.addProperty(properties[i], properties[i + 1]); + } + if (!S3_BUCKET_BASE.startsWith("file:/")) { + catalogPropsBuilder.addProperty( + CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:"); + } + Catalog catalog = + PolarisCatalog.builder() + .setType(catalogConfig.map(CatalogConfig::value).orElse(Catalog.TypeEnum.INTERNAL)) + .setName(currentCatalogName) + .setProperties(catalogPropsBuilder.build()) + .setStorageConfigInfo( + S3_BUCKET_BASE.startsWith("file:/") + ? new FileStorageConfigInfo( + StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://")) + : awsConfigModel) + .build(); + + managementApi.createCatalog(principalRoleName, catalog); + + Optional restCatalogConfig = + testInfo + .getTestMethod() + .flatMap( + m -> + Optional.ofNullable( + m.getAnnotation( + PolarisRestCatalogIntegrationTest.RestCatalogConfig.class))); + ImmutableMap.Builder extraPropertiesBuilder = ImmutableMap.builder(); + restCatalogConfig.ifPresent( + config -> { + for (int i = 0; i < config.value().length; i += 2) { + extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); + } + }); + + restCatalog = + IcebergHelper.restCatalog( + endpoints, principalCredentials, currentCatalogName, extraPropertiesBuilder.build()); + } + + @AfterEach + public void cleanUp() { + client.cleanUp(adminCredentials); } @Override @@ -271,37 +255,6 @@ protected boolean overridesRequestedLocation() { return true; } - private void createCatalogRole(String catalogRoleName) { - CatalogRole catalogRole = new CatalogRole(catalogRoleName); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalogRole))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - - private void addGrant(String catalogRoleName, GrantResource grant) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, catalogRoleName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .put(Entity.json(grant))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } - } - @Test public void testListGrantsOnCatalogObjectsToCatalogRoles() { restCatalog.createNamespace(Namespace.of("ns1")); @@ -396,8 +349,8 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { ViewPrivilege.VIEW_WRITE_PROPERTIES, GrantResource.TypeEnum.VIEW); - createCatalogRole("catalogrole1"); - createCatalogRole("catalogrole2"); + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + managementApi.createCatalogRole(currentCatalogName, "catalogrole2"); List role1Grants = List.of( @@ -409,7 +362,7 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { tableGrant2, viewGrant1, viewGrant2); - role1Grants.forEach(grant -> addGrant("catalogrole1", grant)); + role1Grants.forEach(grant -> managementApi.addGrant(currentCatalogName, "catalogrole1", grant)); List role2Grants = List.of( catalogGrant1, @@ -420,45 +373,19 @@ public void testListGrantsOnCatalogObjectsToCatalogRoles() { tableGrant3, viewGrant1, viewGrant3); - role2Grants.forEach(grant -> addGrant("catalogrole2", grant)); + role2Grants.forEach(grant -> managementApi.addGrant(currentCatalogName, "catalogrole2", grant)); // List grants for catalogrole1 - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole1")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactlyInAnyOrder(role1Grants.toArray(new GrantResource[0])); // List grants for catalogrole2 - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole2")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole2")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactlyInAnyOrder(role2Grants.toArray(new GrantResource[0])); } @Test @@ -478,8 +405,8 @@ public void testListGrantsAfterRename() { TablePrivilege.TABLE_FULL_METADATA, GrantResource.TypeEnum.TABLE); - createCatalogRole("catalogrole1"); - addGrant("catalogrole1", tableGrant1); + managementApi.createCatalogRole(currentCatalogName, "catalogrole1"); + managementApi.addGrant(currentCatalogName, "catalogrole1", tableGrant1); // Grants will follow the table through the rename restCatalog.renameTable( @@ -493,60 +420,19 @@ public void testListGrantsAfterRename() { TablePrivilege.TABLE_FULL_METADATA, GrantResource.TypeEnum.TABLE); - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s/catalog-roles/%s/grants", - EXT.getLocalPort(), currentCatalogName, "catalogrole1")) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response) - .returns(Response.Status.OK.getStatusCode(), Response::getStatus) - .extracting(r -> r.readEntity(GrantResources.class)) - .extracting(GrantResources::getGrants) - .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) - .containsExactly(expectedGrant); - } + assertThat(managementApi.listGrants(currentCatalogName, "catalogrole1")) + .extracting(GrantResources::getGrants) + .asInstanceOf(InstanceOfAssertFactories.list(GrantResource.class)) + .containsExactly(expectedGrant); } @Test - public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocation() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -569,41 +455,12 @@ public void testCreateTableWithOverriddenBaseLocation(PolarisToken adminToken) { } @Test - public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( - PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -635,41 +492,12 @@ public void testCreateTableWithOverriddenBaseLocationCannotOverlapSibling( } @Test - public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory( - PolarisToken adminToken) { - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .get()) { - assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - Catalog catalog = response.readEntity(Catalog.class); - Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); - catalogProps.put( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); - try (Response updateResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/%s", - EXT.getLocalPort(), catalog.getName())) - .request("application/json") - .header("Authorization", "Bearer " + adminToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .put( - Entity.json( - new UpdateCatalogRequest( - catalog.getEntityVersion(), - catalogProps, - catalog.getStorageConfigInfo())))) { - assertThat(updateResponse).returns(Response.Status.OK.getStatusCode(), Response::getStatus); - } - } + public void testCreateTableWithOverriddenBaseLocationMustResideInNsDirectory() { + Catalog catalog = managementApi.getCatalog(currentCatalogName); + Map catalogProps = new HashMap<>(catalog.getProperties().toMap()); + catalogProps.put( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "false"); + managementApi.updateCatalog(catalog, catalogProps); restCatalog.createNamespace(Namespace.of("ns1")); restCatalog.createNamespace( @@ -790,27 +618,20 @@ public void testLoadTableWithAccessDelegationForExternalCatalogWithConfigEnabled @Test public void testSendNotificationInternalCatalog() { - NotificationRequest notification = new NotificationRequest(); - notification.setNotificationType(NotificationType.CREATE); - notification.setPayload( - new TableUpdateNotification( - "tbl1", - System.currentTimeMillis(), - UUID.randomUUID().toString(), - "s3://my-bucket/path/to/metadata.json", - null)); + Map payload = + ImmutableMap.builder() + .put("table-name", "tbl1") + .put("timestamps", "" + System.currentTimeMillis()) + .put("table-uuid", UUID.randomUUID().toString()) + .put("metadata-location", "s3://my-bucket/path/to/metadata.json") + .build(); restCatalog.createNamespace(Namespace.of("ns1")); - String notificationUrl = - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/ns1/tables/tbl1/notifications", - EXT.getLocalPort(), currentCatalogName); + Invocation.Builder notificationEndpoint = + catalogApi.request( + "v1/{cat}/namespaces/ns1/tables/tbl1/notifications", Map.of("cat", currentCatalogName)); try (Response response = - EXT.client() - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(notification))) { + notificationEndpoint.post( + Entity.json(Map.of("notification-type", "CREATE", "payload", payload)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(ErrorResponse.class)) @@ -818,14 +639,9 @@ public void testSendNotificationInternalCatalog() { } // NotificationType.VALIDATE should also surface the same error. - notification.setNotificationType(NotificationType.VALIDATE); try (Response response = - EXT.client() - .target(notificationUrl) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(notification))) { + notificationEndpoint.post( + Entity.json(Map.of("notification-type", "VALIDATE", "payload", payload)))) { assertThat(response) .returns(Response.Status.BAD_REQUEST.getStatusCode(), Response::getStatus) .extracting(r -> r.readEntity(ErrorResponse.class)) @@ -1033,14 +849,10 @@ public void testTableExistsStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "table", tableName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1059,14 +871,10 @@ public void testDropTableStatus() { catalog().buildTable(identifier, SCHEMA).create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), currentCatalogName, namespace.toString(), tableName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "table", tableName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1093,14 +901,10 @@ public void testViewExistsStatus() { .create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1127,14 +931,10 @@ public void testDropViewStatus() { .create(); try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .delete()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } @@ -1168,42 +968,28 @@ public void testRenameViewStatus() { // Perform view rename try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/views/rename", - EXT.getLocalPort(), currentCatalogName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request("v1/{cat}/views/rename", Map.of("cat", currentCatalogName)) .post(Entity.json(payload))) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } // Original view should no longer exists try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, viewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", viewName)) .head()) { assertThat(response).returns(Response.Status.NOT_FOUND.getStatusCode(), Response::getStatus); } - // New view should exists + // New view should exist try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/views/%s", - EXT.getLocalPort(), currentCatalogName, namespace, newViewName)) - .request("application/json") - .header("Authorization", "Bearer " + userToken) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/views/{view}", + Map.of("cat", currentCatalogName, "ns", namespace.toString(), "view", newViewName)) .head()) { assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); } diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java index b7fdbee57..d6b7e39cb 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAwsIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAwsIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.Optional; @@ -27,7 +27,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on AWS. */ public class PolarisRestCatalogViewAwsIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String ROLE_ARN = Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) // Backward compatibility .orElse(System.getenv("INTEGRATION_TEST_S3_ROLE_ARN")); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java index be755526f..4a4eef984 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewAzureIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewAzureIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.stream.Stream; @@ -26,7 +26,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on Azure. */ public class PolarisRestCatalogViewAzureIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID"); public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_AZURE_PATH"); diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java similarity index 93% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java index c7853df97..6b6bc5548 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewFileIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewFileIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import org.apache.polaris.core.admin.model.FileStorageConfigInfo; @@ -24,7 +24,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */ public class PolarisRestCatalogViewFileIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket"; @Override diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java similarity index 94% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java index b74677275..f1c4a762a 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisRestCatalogViewGcpIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewGcpIntegrationTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; import java.util.List; import java.util.stream.Stream; @@ -26,7 +26,7 @@ /** Runs PolarisRestCatalogViewIntegrationTest on GCP. */ public class PolarisRestCatalogViewGcpIntegrationTest - extends PolarisRestCatalogViewIntegrationTest { + extends PolarisRestCatalogViewIntegrationBase { public static final String SERVICE_ACCOUNT = System.getenv("INTEGRATION_TEST_GCS_SERVICE_ACCOUNT"); public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_GCS_PATH"); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java new file mode 100644 index 000000000..4f78413e0 --- /dev/null +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.it.test; + +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; + +import java.lang.reflect.Method; +import java.util.Map; +import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.view.ViewCatalogTests; +import org.apache.polaris.core.PolarisConfiguration; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.PrincipalWithCredentials; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.apache.polaris.core.entity.CatalogEntity; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.IcebergHelper; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.ExtendWith; + +/** + * Import the full core Iceberg catalog tests by hitting the REST service via the RESTCatalog + * client. + */ +@ExtendWith(PolarisIntegrationTestExtension.class) +public abstract class PolarisRestCatalogViewIntegrationBase extends ViewCatalogTests { + + private static ClientCredentials adminCredentials; + private static PolarisApiEndpoints endpoints; + private static PolarisClient client; + private static ManagementApi managementApi; + + private RESTCatalog restCatalog; + + @BeforeAll + static void setup(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + adminCredentials = credentials; + endpoints = apiEndpoints; + client = polarisClient(endpoints); + managementApi = client.managementApi(credentials); + } + + @AfterAll + static void close() throws Exception { + client.close(); + } + + @BeforeEach + public void before(TestInfo testInfo) { + Assumptions.assumeFalse(shouldSkip()); + + String principalName = client.newEntityName("snowman-rest"); + String principalRoleName = client.newEntityName("rest-admin"); + PrincipalWithCredentials principalCredentials = + managementApi.createPrincipalWithRole(principalName, principalRoleName); + + Method method = testInfo.getTestMethod().orElseThrow(); + String catalogName = client.newEntityName(method.getName()); + + StorageConfigInfo storageConfig = getStorageConfigInfo(); + String defaultBaseLocation = + storageConfig.getAllowedLocations().getFirst() + + "/" + + System.getenv("USER") + + "/path/to/data"; + + CatalogProperties props = + CatalogProperties.builder(defaultBaseLocation) + .addProperty( + CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY, "file:") + .addProperty(PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .build(); + Catalog catalog = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName(catalogName) + .setProperties(props) + .setStorageConfigInfo(storageConfig) + .build(); + managementApi.createCatalog(principalRoleName, catalog); + + restCatalog = IcebergHelper.restCatalog(endpoints, principalCredentials, catalogName, Map.of()); + } + + @AfterEach + public void cleanUp() { + client.cleanUp(adminCredentials); + } + + /** + * @return The catalog's storage config. + */ + protected abstract StorageConfigInfo getStorageConfigInfo(); + + /** + * @return Whether the tests should be skipped, for example due to environment variables not being + * specified. + */ + protected abstract boolean shouldSkip(); + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected org.apache.iceberg.catalog.Catalog tableCatalog() { + return restCatalog; + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } + + @Override + protected boolean overridesRequestedLocation() { + return true; + } +} diff --git a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java similarity index 63% rename from dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java rename to integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java index 173e2bda6..dc7c45ac0 100644 --- a/dropwizard/service/src/test/java/org/apache/polaris/service/dropwizard/catalog/PolarisSparkIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisSparkIntegrationTest.java @@ -16,17 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.polaris.service.dropwizard.catalog; +package org.apache.polaris.service.it.test; -import static org.apache.polaris.service.context.DefaultRealmContextResolver.REALM_PROPERTY_KEY; +import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import com.adobe.testing.s3mock.testcontainers.S3MockContainer; -import io.dropwizard.testing.ConfigOverride; -import io.dropwizard.testing.ResourceHelpers; -import io.dropwizard.testing.junit5.DropwizardAppExtension; -import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import com.google.common.collect.ImmutableMap; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.Response; import java.io.IOException; @@ -41,14 +38,12 @@ import org.apache.polaris.core.admin.model.ExternalCatalog; import org.apache.polaris.core.admin.model.PolarisCatalog; import org.apache.polaris.core.admin.model.StorageConfigInfo; -import org.apache.polaris.service.dropwizard.PolarisApplication; -import org.apache.polaris.service.dropwizard.config.PolarisApplicationConfig; -import org.apache.polaris.service.dropwizard.test.PolarisConnectionExtension; -import org.apache.polaris.service.dropwizard.test.PolarisRealm; -import org.apache.polaris.service.dropwizard.test.TestEnvironmentExtension; -import org.apache.polaris.service.types.NotificationRequest; -import org.apache.polaris.service.types.NotificationType; -import org.apache.polaris.service.types.TableUpdateNotification; +import org.apache.polaris.service.it.env.CatalogApi; +import org.apache.polaris.service.it.env.ClientCredentials; +import org.apache.polaris.service.it.env.ManagementApi; +import org.apache.polaris.service.it.env.PolarisApiEndpoints; +import org.apache.polaris.service.it.env.PolarisClient; +import org.apache.polaris.service.it.ext.PolarisIntegrationTestExtension; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.SparkSession; @@ -61,39 +56,23 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.slf4j.LoggerFactory; -@ExtendWith({ - DropwizardExtensionsSupport.class, - TestEnvironmentExtension.class, - PolarisConnectionExtension.class -}) +@ExtendWith(PolarisIntegrationTestExtension.class) public class PolarisSparkIntegrationTest { - private static final DropwizardAppExtension EXT = - new DropwizardAppExtension<>( - PolarisApplication.class, - ResourceHelpers.resourceFilePath("polaris-server-integrationtest.yml"), - ConfigOverride.config( - "server.applicationConnectors[0].port", - "0"), // Bind to random port to support parallelism - ConfigOverride.config( - "server.adminConnectors[0].port", "0")); // Bind to random port to support parallelism - - public static final String CATALOG_NAME = "mycatalog"; - public static final String EXTERNAL_CATALOG_NAME = "external_catalog"; + private static final S3MockContainer s3Container = new S3MockContainer("3.11.0").withInitialBuckets("my-bucket,my-old-bucket"); - private static PolarisConnectionExtension.PolarisToken polarisToken; private static SparkSession spark; - private String realm; + private PolarisApiEndpoints endpoints; + private PolarisClient client; + private ManagementApi managementApi; + private CatalogApi catalogApi; + private String sparkToken; + private String catalogName; + private String externalCatalogName; @BeforeAll - public static void setup( - PolarisConnectionExtension.PolarisToken polarisToken, @PolarisRealm String realm) - throws IOException { + public static void setup() throws IOException { s3Container.start(); - PolarisSparkIntegrationTest.polarisToken = polarisToken; - - // Set up test location - PolarisConnectionExtension.createTestDir(realm); } @AfterAll @@ -102,8 +81,16 @@ public static void cleanup() { } @BeforeEach - public void before(@PolarisRealm String realm) { - this.realm = realm; + public void before(PolarisApiEndpoints apiEndpoints, ClientCredentials credentials) { + endpoints = apiEndpoints; + client = polarisClient(endpoints); + sparkToken = client.obtainToken(credentials); + managementApi = client.managementApi(credentials); + catalogApi = client.catalogApi(credentials); + + catalogName = client.newEntityName("spark_catalog"); + externalCatalogName = client.newEntityName("spark_ext_catalog"); + AwsStorageConfigInfo awsConfigModel = AwsStorageConfigInfo.builder() .setRoleArn("arn:aws:iam::123456789012:role/my-role") @@ -134,21 +121,12 @@ public void before(@PolarisRealm String realm) { Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) - .setName(CATALOG_NAME) + .setName(catalogName) .setProperties(props) .setStorageConfigInfo(awsConfigModel) .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(catalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + managementApi.createCatalog(catalog); CatalogProperties externalProps = new CatalogProperties("s3://my-bucket/path/to/data"); externalProps.putAll( @@ -172,21 +150,14 @@ public void before(@PolarisRealm String realm) { Catalog externalCatalog = ExternalCatalog.builder() .setType(Catalog.TypeEnum.EXTERNAL) - .setName(EXTERNAL_CATALOG_NAME) + .setName(externalCatalogName) .setProperties(externalProps) .setStorageConfigInfo(awsConfigModel) .setRemoteUrl("http://dummy_url") .build(); - try (Response response = - EXT.client() - .target( - String.format("http://localhost:%d/api/management/v1/catalogs", EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .post(Entity.json(externalCatalog))) { - assertThat(response).returns(Response.Status.CREATED.getStatusCode(), Response::getStatus); - } + + managementApi.createCatalog(externalCatalog); + SparkSession.Builder sessionBuilder = SparkSession.builder() .master("local[1]") @@ -202,9 +173,9 @@ public void before(@PolarisRealm String realm) { .config("spark.ui.showConsoleProgress", false) .config("spark.ui.enabled", "false"); spark = - withCatalog(withCatalog(sessionBuilder, CATALOG_NAME), EXTERNAL_CATALOG_NAME).getOrCreate(); + withCatalog(withCatalog(sessionBuilder, catalogName), externalCatalogName).getOrCreate(); - onSpark("USE " + CATALOG_NAME); + onSpark("USE " + catalogName); } private SparkSession.Builder withCatalog(SparkSession.Builder builder, String catalogName) { @@ -215,11 +186,11 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca .config(String.format("spark.sql.catalog.%s.type", catalogName), "rest") .config( String.format("spark.sql.catalog.%s.uri", catalogName), - "http://localhost:" + EXT.getLocalPort() + "/api/catalog") + endpoints.catalogApiEndpoint().toString()) .config(String.format("spark.sql.catalog.%s.warehouse", catalogName), catalogName) .config(String.format("spark.sql.catalog.%s.scope", catalogName), "PRINCIPAL_ROLE:ALL") - .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), realm) - .config(String.format("spark.sql.catalog.%s.token", catalogName), polarisToken.token()) + .config(String.format("spark.sql.catalog.%s.header.realm", catalogName), endpoints.realm()) + .config(String.format("spark.sql.catalog.%s.token", catalogName), sparkToken) .config(String.format("spark.sql.catalog.%s.s3.access-key-id", catalogName), "fakekey") .config( String.format("spark.sql.catalog.%s.s3.secret-access-key", catalogName), "fakesecret") @@ -227,9 +198,9 @@ private SparkSession.Builder withCatalog(SparkSession.Builder builder, String ca } @AfterEach - public void after() { - cleanupCatalog(CATALOG_NAME); - cleanupCatalog(EXTERNAL_CATALOG_NAME); + public void after() throws Exception { + cleanupCatalog(catalogName); + cleanupCatalog(externalCatalogName); try { SparkSession.clearDefaultSession(); SparkSession.clearActiveSession(); @@ -237,6 +208,8 @@ public void after() { } catch (Exception e) { LoggerFactory.getLogger(getClass()).error("Unable to close spark session", e); } + + client.close(); } private void cleanupCatalog(String catalogName) { @@ -253,18 +226,8 @@ private void cleanupCatalog(String catalogName) { } onSpark("DROP NAMESPACE " + namespace.getString(0)); } - try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/management/v1/catalogs/" + catalogName, - EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) - .delete()) { - assertThat(response).returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); - } + + managementApi.deleteCatalog(catalogName); } @Test @@ -292,7 +255,7 @@ public void testCreateAndUpdateExternalTable() { long recordCount = onSpark("SELECT * FROM tb1").count(); assertThat(recordCount).isEqualTo(3); - onSpark("USE " + EXTERNAL_CATALOG_NAME); + onSpark("USE " + externalCatalogName); List existingNamespaces = onSpark("SHOW NAMESPACES").collectAsList(); assertThat(existingNamespaces).isEmpty(); @@ -301,18 +264,10 @@ public void testCreateAndUpdateExternalTable() { List existingTables = onSpark("SHOW TABLES").collectAsList(); assertThat(existingTables).isEmpty(); - LoadTableResponse tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); + LoadTableResponse tableResponse = loadTable(catalogName, "ns1", "tb1"); try (Response registerResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/" - + EXTERNAL_CATALOG_NAME - + "/namespaces/externalns1/register", - EXT.getLocalPort())) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request("v1/{cat}/namespaces/externalns1/register", Map.of("cat", externalCatalogName)) .post( Entity.json( ImmutableRegisterTableRequest.builder() @@ -331,30 +286,30 @@ public void testCreateAndUpdateExternalTable() { assertThatThrownBy(() -> onSpark("INSERT INTO mytb1 VALUES (20, 'new_text')")) .isInstanceOf(Exception.class); - onSpark("INSERT INTO " + CATALOG_NAME + ".ns1.tb1 VALUES (20, 'new_text')"); - tableResponse = loadTable(CATALOG_NAME, "ns1", "tb1"); - TableUpdateNotification updateNotification = - new TableUpdateNotification( - "mytb1", - Instant.now().toEpochMilli(), - tableResponse.tableMetadata().uuid(), - tableResponse.metadataLocation(), - tableResponse.tableMetadata()); - NotificationRequest notificationRequest = new NotificationRequest(); - notificationRequest.setPayload(updateNotification); - notificationRequest.setNotificationType(NotificationType.UPDATE); + onSpark("INSERT INTO " + catalogName + ".ns1.tb1 VALUES (20, 'new_text')"); + tableResponse = loadTable(catalogName, "ns1", "tb1"); + Map updateNotification = + ImmutableMap.builder() + .put("table-name", "mytb1") + .put("timestamp", "" + Instant.now().toEpochMilli()) + .put("table-uuid", tableResponse.tableMetadata().uuid()) + .put("metadata-location", tableResponse.metadataLocation()) + .put("metadata", tableResponse.tableMetadata()) + .build(); + Map notificationRequest = + ImmutableMap.builder() + .put("payload", updateNotification) + .put("notification-type", "UPDATE") + .build(); try (Response notifyResponse = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/externalns1/tables/mytb1/notifications", - EXT.getLocalPort(), EXTERNAL_CATALOG_NAME)) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/externalns1/tables/mytb1/notifications", + Map.of("cat", externalCatalogName)) .post(Entity.json(notificationRequest))) { assertThat(notifyResponse) - .returns(Response.Status.NO_CONTENT.getStatusCode(), Response::getStatus); + .extracting(Response::getStatus) + .isEqualTo(Response.Status.NO_CONTENT.getStatusCode()); } // refresh the table so it queries for the latest metadata.json onSpark("REFRESH TABLE mytb1"); @@ -378,14 +333,10 @@ public void testCreateView() { private LoadTableResponse loadTable(String catalog, String namespace, String table) { try (Response response = - EXT.client() - .target( - String.format( - "http://localhost:%d/api/catalog/v1/%s/namespaces/%s/tables/%s", - EXT.getLocalPort(), catalog, namespace, table)) - .request("application/json") - .header("Authorization", "BEARER " + polarisToken.token()) - .header(REALM_PROPERTY_KEY, realm) + catalogApi + .request( + "v1/{cat}/namespaces/{ns}/tables/{table}", + Map.of("cat", catalog, "ns", namespace, "table", table)) .get()) { assertThat(response).returns(Response.Status.OK.getStatusCode(), Response::getStatus); return response.readEntity(LoadTableResponse.class);