From 9e3137164c0072cfc693641b42353d8f4a8e84f0 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 2 Jan 2025 16:59:10 +0100 Subject: [PATCH] Polaris Admin Tool --- LICENSE-BINARY-DIST | 2 + ...pseLinkPolarisMetaStoreManagerFactory.java | 8 +- gradle/projects.main.properties | 1 + .../LocalPolarisMetaStoreManagerFactory.java | 48 +++++------ .../persistence/MetaStoreManagerFactory.java | 3 +- .../PolarisCredentialsBootstrap.java | 60 +++++++------ .../PolarisCredentialsBootstrapTest.java | 22 +++++ quarkus/admin/README.md | 27 ++++++ quarkus/admin/build.gradle.kts | 86 +++++++++++++++++++ quarkus/admin/src/main/docker/Dockerfile.jvm | 42 +++++++++ .../service/quarkus/admin/BaseCommand.java | 54 ++++++++++++ .../quarkus/admin/BootstrapCommand.java | 80 +++++++++++++++++ .../quarkus/admin/PolarisAdminTool.java | 54 ++++++++++++ .../quarkus/admin/PolarisVersionProvider.java | 39 +++++++++ .../service/quarkus/admin/PurgeCommand.java | 49 +++++++++++ .../src/main/resources/application.properties | 27 ++++++ .../service/quarkus/admin/version.properties | 20 +++++ .../quarkus/admin/BootstrapCommandTest.java | 47 ++++++++++ .../quarkus/admin/PurgeCommandTest.java | 36 ++++++++ quarkus/server/README.md | 12 ++- quarkus/server/build.gradle.kts | 17 ++++ .../src/main/resources/application.properties | 1 + .../catalog/BasePolarisCatalogTest.java | 4 +- .../test/PolarisIntegrationTestFixture.java | 4 +- ...nMemoryPolarisMetaStoreManagerFactory.java | 11 ++- 25 files changed, 694 insertions(+), 60 deletions(-) create mode 100644 quarkus/admin/README.md create mode 100644 quarkus/admin/build.gradle.kts create mode 100644 quarkus/admin/src/main/docker/Dockerfile.jvm create mode 100644 quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java create mode 100644 quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java create mode 100644 quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java create mode 100644 quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java create mode 100644 quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java create mode 100644 quarkus/admin/src/main/resources/application.properties create mode 100644 quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties create mode 100644 quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java create mode 100644 quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java diff --git a/LICENSE-BINARY-DIST b/LICENSE-BINARY-DIST index 1165f96e49..205a3eedc1 100644 --- a/LICENSE-BINARY-DIST +++ b/LICENSE-BINARY-DIST @@ -280,6 +280,7 @@ commons-io:commons-io commons-logging:commons-logging commons-net:commons-net dev.failsafe:failsafe +info.picocli:picocli io.airlift:aircompressor io.grpc:grpc-alts io.grpc:grpc-api @@ -400,6 +401,7 @@ io.quarkus:quarkus-micrometer-registry-prometheus io.quarkus:quarkus-mutiny io.quarkus:quarkus-netty io.quarkus:quarkus-opentelemetry +io.quarkus:quarkus-picocli io.quarkus:quarkus-reactive-routes io.quarkus:quarkus-rest io.quarkus:quarkus-rest-common diff --git a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java index f7a1197612..3767a0645a 100644 --- a/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java +++ b/extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/EclipseLinkPolarisMetaStoreManagerFactory.java @@ -20,12 +20,14 @@ import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.nio.file.Path; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.storage.PolarisStorageIntegrationProvider; @@ -50,14 +52,16 @@ protected PolarisEclipseLinkStore createBackingStore(@Nonnull PolarisDiagnostics @Override protected PolarisMetaStoreSession createMetaStoreSession( - @Nonnull PolarisEclipseLinkStore store, @Nonnull RealmContext realmContext) { + @Nonnull PolarisEclipseLinkStore store, + @Nonnull RealmContext realmContext, + @Nullable PolarisCredentialsBootstrap credentialsBootstrap) { return new PolarisEclipseLinkMetaStoreSessionImpl( store, storageIntegrationProvider, realmContext, configurationFile(), persistenceUnitName(), - secretsGenerator(realmContext)); + secretsGenerator(realmContext, credentialsBootstrap)); } private String configurationFile() { diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index fe632405fc..32705debf6 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -25,6 +25,7 @@ polaris-api-management-service=api/management-service polaris-service-common=service/common polaris-quarkus-service=quarkus/service polaris-quarkus-server=quarkus/server +polaris-quarkus-admin=quarkus/admin polaris-eclipselink=extension/persistence/eclipselink polaris-jpa-model=extension/persistence/jpa-model aggregated-license-report=aggregated-license-report diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java index 6c9294418c..6c2d6565aa 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java @@ -19,6 +19,7 @@ package org.apache.polaris.core.persistence; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -57,50 +58,49 @@ public abstract class LocalPolarisMetaStoreManagerFactory private static final Logger LOGGER = LoggerFactory.getLogger(LocalPolarisMetaStoreManagerFactory.class); - private boolean bootstrap; - protected abstract StoreType createBackingStore(@Nonnull PolarisDiagnostics diagnostics); protected abstract PolarisMetaStoreSession createMetaStoreSession( - @Nonnull StoreType store, @Nonnull RealmContext realmContext); + @Nonnull StoreType store, + @Nonnull RealmContext realmContext, + @Nullable PolarisCredentialsBootstrap credentialsBootstrap); - protected PrincipalSecretsGenerator secretsGenerator(RealmContext realmContext) { - if (bootstrap) { - return PrincipalSecretsGenerator.bootstrap(realmContext.getRealmIdentifier()); + protected PrincipalSecretsGenerator secretsGenerator( + RealmContext realmContext, @Nullable PolarisCredentialsBootstrap credentialsBootstrap) { + if (credentialsBootstrap != null) { + return PrincipalSecretsGenerator.bootstrap( + realmContext.getRealmIdentifier(), credentialsBootstrap); } else { return PrincipalSecretsGenerator.RANDOM_SECRETS; } } - private void initializeForRealm(RealmContext realmContext) { + private void initializeForRealm( + RealmContext realmContext, PolarisCredentialsBootstrap credentialsBootstrap) { final StoreType backingStore = createBackingStore(diagServices); backingStoreMap.put(realmContext.getRealmIdentifier(), backingStore); sessionSupplierMap.put( realmContext.getRealmIdentifier(), - () -> createMetaStoreSession(backingStore, realmContext)); + () -> createMetaStoreSession(backingStore, realmContext, credentialsBootstrap)); PolarisMetaStoreManager metaStoreManager = new PolarisMetaStoreManagerImpl(); metaStoreManagerMap.put(realmContext.getRealmIdentifier(), metaStoreManager); } @Override - public synchronized Map bootstrapRealms(List realms) { + public synchronized Map bootstrapRealms( + List realms, PolarisCredentialsBootstrap credentialsBootstrap) { Map results = new HashMap<>(); - bootstrap = true; - try { - for (String realm : realms) { - RealmContext realmContext = () -> realm; - if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext); - PrincipalSecretsResult secretsResult = - bootstrapServiceAndCreatePolarisPrincipalForRealm( - realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); - results.put(realmContext.getRealmIdentifier(), secretsResult); - } + for (String realm : realms) { + RealmContext realmContext = () -> realm; + if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { + initializeForRealm(realmContext, credentialsBootstrap); + PrincipalSecretsResult secretsResult = + bootstrapServiceAndCreatePolarisPrincipalForRealm( + realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); + results.put(realmContext.getRealmIdentifier(), secretsResult); } - } finally { - bootstrap = false; } return results; @@ -126,7 +126,7 @@ public void purgeRealms(List realms) { public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( RealmContext realmContext) { if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext); + initializeForRealm(realmContext, null); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } @@ -137,7 +137,7 @@ public synchronized PolarisMetaStoreManager getOrCreateMetaStoreManager( public synchronized Supplier getOrCreateSessionSupplier( RealmContext realmContext) { if (!sessionSupplierMap.containsKey(realmContext.getRealmIdentifier())) { - initializeForRealm(realmContext); + initializeForRealm(realmContext, null); checkPolarisServiceBootstrappedForRealm( realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier())); } else { diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java index 5d4691a554..4e37f034c8 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java @@ -37,7 +37,8 @@ public interface MetaStoreManagerFactory { EntityCache getOrCreateEntityCache(RealmContext realmContext); - Map bootstrapRealms(List realms); + Map bootstrapRealms( + List realms, PolarisCredentialsBootstrap credentialsBootstrap); /** Purge all metadata for the realms provided */ void purgeRealms(List realms); diff --git a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrap.java b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrap.java index fb70281278..dc4f5b8758 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrap.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrap.java @@ -35,6 +35,9 @@ */ public class PolarisCredentialsBootstrap { + public static final PolarisCredentialsBootstrap EMPTY = + new PolarisCredentialsBootstrap(new HashMap<>()); + /** * Parse credentials from the system property {@code polaris.bootstrap.credentials} or the * environment variable {@code POLARIS_BOOTSTRAP_CREDENTIALS}, whichever is set. @@ -55,35 +58,38 @@ public static PolarisCredentialsBootstrap fromEnvironment() { * */ public static PolarisCredentialsBootstrap fromString(@Nullable String credentialsString) { + return credentialsString != null && !credentialsString.isBlank() + ? fromList(Splitter.on(';').trimResults().splitToList(credentialsString)) + : EMPTY; + } + + /** + * Parse a list of credentials; each element should be in the format: {@code + * realm,principal,clientId,clientSecret}. + */ + public static PolarisCredentialsBootstrap fromList(List credentialsList) { Map>> credentials = new HashMap<>(); - if (credentialsString != null && !credentialsString.isBlank()) { - Splitter.on(';') - .trimResults() - .splitToList(credentialsString) - .forEach( - quadruple -> { - if (!quadruple.isBlank()) { - List parts = Splitter.on(',').trimResults().splitToList(quadruple); - if (parts.size() != 4) { - throw new IllegalArgumentException("Invalid credentials format: " + quadruple); - } - String realmName = parts.get(0); - String principalName = parts.get(1); - String clientId = parts.get(2); - String clientSecret = parts.get(3); - credentials - .computeIfAbsent(realmName, k -> new HashMap<>()) - .merge( - principalName, - new SimpleEntry<>(clientId, clientSecret), - (a, b) -> { - throw new IllegalArgumentException( - "Duplicate principal: " + principalName); - }); - } - }); + for (String quadruple : credentialsList) { + if (!quadruple.isBlank()) { + List parts = Splitter.on(',').trimResults().splitToList(quadruple); + if (parts.size() != 4) { + throw new IllegalArgumentException("Invalid credentials format: " + quadruple); + } + String realmName = parts.get(0); + String principalName = parts.get(1); + String clientId = parts.get(2); + String clientSecret = parts.get(3); + credentials + .computeIfAbsent(realmName, k -> new HashMap<>()) + .merge( + principalName, + new SimpleEntry<>(clientId, clientSecret), + (a, b) -> { + throw new IllegalArgumentException("Duplicate principal: " + principalName); + }); + } } - return new PolarisCredentialsBootstrap(credentials); + return credentials.isEmpty() ? EMPTY : new PolarisCredentialsBootstrap(credentials); } @VisibleForTesting final Map>> credentials; diff --git a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrapTest.java b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrapTest.java index bd4c57c99e..4cba20a740 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrapTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/persistence/PolarisCredentialsBootstrapTest.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Comparator; +import java.util.List; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; import org.junit.jupiter.api.Test; @@ -87,6 +88,27 @@ void getSecretsValidString() { .contains(new PolarisPrincipalSecrets(123, "client2a", "secret2a", "secret2a")); } + @Test + void getSecretsValidList() { + PolarisCredentialsBootstrap credentials = + PolarisCredentialsBootstrap.fromList( + List.of( + "realm1,user1a,client1a,secret1a", + "realm1,user1b,client1b,secret1b", + "realm2,user2a,client2a,secret2a")); + assertThat(credentials.getSecrets("realm1", 123, "nonexistent")).isEmpty(); + assertThat(credentials.getSecrets("nonexistent", 123, "user1a")).isEmpty(); + assertThat(credentials.getSecrets("realm1", 123, "user1a")) + .usingValueComparator(comparator) + .contains(new PolarisPrincipalSecrets(123, "client1a", "secret1a", "secret1a")); + assertThat(credentials.getSecrets("realm1", 123, "user1b")) + .usingValueComparator(comparator) + .contains(new PolarisPrincipalSecrets(123, "client1b", "secret1b", "secret1b")); + assertThat(credentials.getSecrets("realm2", 123, "user2a")) + .usingValueComparator(comparator) + .contains(new PolarisPrincipalSecrets(123, "client2a", "secret2a", "secret2a")); + } + @Test void getSecretsValidSystemProperty() { PolarisCredentialsBootstrap credentials = PolarisCredentialsBootstrap.fromEnvironment(); diff --git a/quarkus/admin/README.md b/quarkus/admin/README.md new file mode 100644 index 0000000000..8639200669 --- /dev/null +++ b/quarkus/admin/README.md @@ -0,0 +1,27 @@ +# Polaris Admin Tool + +This module contains a maintenance tool for performing administrative tasks on the Polaris database. +It is a Quarkus application that can be used to perform various maintenance tasks targeting the +Polaris database directly. + +Building this module will create a runnable uber-jar that can be executed from the command line. + +To also build the Docker image, you can use the following command: + +```shell +./gradlew :polaris-quarkus-admin:assemble -Dquarkus.container-image.build=true +``` + +## Running the Admin Tool + +The admin tool can be run from the command line using the following command: + +```shell +java -jar polaris-quarkus-admin--runner.jar --help +``` + +Using the Docker image, you can run the admin tool with the following command: + +```shell +docker run --rm -it polaris-admin-tool: --help +``` \ No newline at end of file diff --git a/quarkus/admin/build.gradle.kts b/quarkus/admin/build.gradle.kts new file mode 100644 index 0000000000..9cf4c94000 --- /dev/null +++ b/quarkus/admin/build.gradle.kts @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import io.quarkus.gradle.tasks.QuarkusBuild + +plugins { + alias(libs.plugins.quarkus) + alias(libs.plugins.openapi.generator) + id("polaris-server") + id("polaris-license-report") +} + +dependencies { + implementation(project(":polaris-core")) + implementation(project(":polaris-api-management-service")) + implementation(project(":polaris-api-iceberg-service")) + implementation(project(":polaris-service-common")) + implementation(project(":polaris-quarkus-service")) + + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-picocli") + implementation("io.quarkus:quarkus-container-image-docker") + + implementation("org.jboss.slf4j:slf4j-jboss-logmanager") + + // override dnsjava version in dependencies due to https://github.com/dnsjava/dnsjava/issues/329 + implementation(platform(libs.dnsjava)) + + testImplementation(enforcedPlatform(libs.quarkus.bom)) + testImplementation("io.quarkus:quarkus-junit5") + + testImplementation(platform(libs.junit.bom)) + testImplementation(libs.bundles.junit.testing) +} + +tasks.withType().configureEach { + from("src/main/resources") { + expand("polarisVersion" to version) + duplicatesStrategy = DuplicatesStrategy.INCLUDE + } +} + +quarkus { + quarkusBuildProperties.put("quarkus.package.type", "uber-jar") + // Pull manifest attributes from the "main" `jar` task to get the + // release-information into the jars generated by Quarkus. + quarkusBuildProperties.putAll( + provider { + tasks + .named("jar", Jar::class.java) + .get() + .manifest + .attributes + .map { e -> "quarkus.package.jar.manifest.attributes.\"${e.key}\"" to e.value.toString() } + .toMap() + } + ) +} + +publishing { + publications { + named("maven") { + val quarkusBuild = tasks.getByName("quarkusBuild") + artifact(quarkusBuild.runnerJar) { + classifier = "runner" + builtBy(quarkusBuild) + } + } + } +} diff --git a/quarkus/admin/src/main/docker/Dockerfile.jvm b/quarkus/admin/src/main/docker/Dockerfile.jvm new file mode 100644 index 0000000000..9f992a30a4 --- /dev/null +++ b/quarkus/admin/src/main/docker/Dockerfile.jvm @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +FROM registry.access.redhat.com/ubi9/openjdk-21:1.20-2.1726695192 + +LABEL org.opencontainers.image.source=https://github.com/apache/polaris +LABEL org.opencontainers.image.description="Apache Polaris (incubating) Admin Tool" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +ENV LANGUAGE='en_US:en' + +USER root +RUN groupadd --gid 10001 polaris \ + && useradd --uid 10000 --gid polaris polaris \ + && chown -R polaris:polaris /opt/jboss/container \ + && chown -R polaris:polaris /deployments + +USER polaris +WORKDIR /deployments +ENV USER=polaris +ENV UID=10000 +ENV HOME=/home/polaris +ENV PWD=/deployments + +COPY --chown=polaris:polaris build/*-runner.jar /deployments/polaris-server-admin-tool.jar + +ENTRYPOINT [ "java", "-jar", "/deployments/polaris-server-admin-tool.jar" ] \ No newline at end of file diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java new file mode 100644 index 0000000000..6966ef6574 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BaseCommand.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import jakarta.inject.Inject; +import java.util.concurrent.Callable; +import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.service.quarkus.persistence.QuarkusPersistenceConfiguration; +import picocli.CommandLine.Model.CommandSpec; +import picocli.CommandLine.Spec; + +public abstract class BaseCommand implements Callable { + + public static final Integer EXIT_CODE_BOOTSTRAP_ERROR = 2; + public static final Integer EXIT_CODE_PURGE_ERROR = 3; + + @Inject QuarkusPersistenceConfiguration persistenceConfiguration; + + @Inject MetaStoreManagerFactory metaStoreManagerFactory; + + @Spec CommandSpec spec; + + protected void warnOnInMemory() { + if (persistenceConfiguration.type().equalsIgnoreCase("in-memory")) { + spec.commandLine() + .getErr() + .println( + spec.commandLine() + .getColorScheme() + .errorText( + """ + ********************************************************************************************* + ** Running the Admin Tool on a Polaris instance with in-memory persistence is meaningless! ** + ********************************************************************************************* + """)); + } + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java new file mode 100644 index 0000000000..5d9f5bec4d --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/BootstrapCommand.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import java.util.List; +import java.util.Map; +import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; +import picocli.CommandLine; + +@CommandLine.Command( + name = "bootstrap", + mixinStandardHelpOptions = true, + description = "Bootstraps realms and principal credentials.") +public class BootstrapCommand extends BaseCommand { + + @CommandLine.Option( + names = {"-r", "--realm"}, + required = true, + description = "The name of a realm to bootstrap.") + List realms; + + @CommandLine.Option( + names = {"-c", "--credential"}, + description = + "Principal credentials to bootstrap. Must be of the form 'realm,userName,clientId,clientSecret'.") + List credentials; + + @Override + public Integer call() { + warnOnInMemory(); + + PolarisCredentialsBootstrap credentialsBootstrap = + credentials == null || credentials.isEmpty() + ? PolarisCredentialsBootstrap.EMPTY + : PolarisCredentialsBootstrap.fromList(credentials); + + // Execute the bootstrap + Map results = + metaStoreManagerFactory.bootstrapRealms(realms, credentialsBootstrap); + + // Log any errors: + boolean success = true; + for (Map.Entry result : results.entrySet()) { + if (!result.getValue().isSuccess()) { + String realm = result.getKey(); + spec.commandLine() + .getErr() + .printf( + "Bootstrapping '%s' failed: %s%n", + realm, result.getValue().getReturnStatus().toString()); + success = false; + } + } + + if (success) { + spec.commandLine().getOut().println("Bootstrap completed successfully."); + return 0; + } else { + spec.commandLine().getErr().println("Bootstrap encountered errors during operation."); + return EXIT_CODE_BOOTSTRAP_ERROR; + } + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java new file mode 100644 index 0000000000..3b33273516 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisAdminTool.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import io.quarkus.picocli.runtime.annotations.TopCommand; +import java.io.PrintWriter; +import picocli.CommandLine.Command; +import picocli.CommandLine.HelpCommand; + +@TopCommand +@Command( + name = "polaris-quarkus-admin-runner.jar", + mixinStandardHelpOptions = true, + versionProvider = PolarisVersionProvider.class, + description = "Polaris Admin Tool", + subcommands = { + HelpCommand.class, + BootstrapCommand.class, + PurgeCommand.class, + }) +public class PolarisAdminTool extends BaseCommand { + + @Override + public Integer call() { + return info(); + } + + private int info() { + warnOnInMemory(); + + PrintWriter out = spec.commandLine().getOut(); + + out.println("Polaris administration & maintenance tool."); + out.println("Use the 'help' command."); + out.println(); + return 0; + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java new file mode 100644 index 0000000000..090948b187 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PolarisVersionProvider.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import java.io.InputStream; +import java.net.URL; +import java.util.Objects; +import java.util.Properties; +import picocli.CommandLine.IVersionProvider; + +public class PolarisVersionProvider implements IVersionProvider { + + @Override + public String[] getVersion() throws Exception { + URL resource = + Objects.requireNonNull(PolarisVersionProvider.class.getResource("version.properties")); + try (InputStream input = resource.openConnection().getInputStream()) { + Properties props = new Properties(); + props.load(input); + return new String[] {props.getProperty("polaris.version")}; + } + } +} diff --git a/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java new file mode 100644 index 0000000000..63963c12a7 --- /dev/null +++ b/quarkus/admin/src/main/java/org/apache/polaris/service/quarkus/admin/PurgeCommand.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import java.util.List; +import picocli.CommandLine; + +@CommandLine.Command( + name = "purge", + mixinStandardHelpOptions = true, + description = "Purge principal credentials.") +public class PurgeCommand extends BaseCommand { + + @CommandLine.Option( + names = {"-r", "--realm"}, + required = true, + description = "The name of a realm to purge.") + List realms; + + @Override + public Integer call() { + warnOnInMemory(); + + try { + metaStoreManagerFactory.purgeRealms(realms); + spec.commandLine().getOut().println("Purge completed successfully."); + return 0; + } catch (Exception e) { + spec.commandLine().getErr().println("Purge encountered errors during operation."); + return EXIT_CODE_PURGE_ERROR; + } + } +} diff --git a/quarkus/admin/src/main/resources/application.properties b/quarkus/admin/src/main/resources/application.properties new file mode 100644 index 0000000000..8f963cc536 --- /dev/null +++ b/quarkus/admin/src/main/resources/application.properties @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +quarkus.application.name=Apache Polaris Admin Tool (incubating) + +quarkus.container-image.build=false +quarkus.container-image.push=false +quarkus.container-image.registry=docker.io +quarkus.container-image.group=apache +quarkus.container-image.name=polaris-admin-tool +quarkus.container-image.additional-tags=latest diff --git a/quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties b/quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties new file mode 100644 index 0000000000..b36811eef4 --- /dev/null +++ b/quarkus/admin/src/main/resources/org/apache/polaris/service/quarkus/admin/version.properties @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +polaris.version=${polarisVersion} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java new file mode 100644 index 0000000000..0e964c07f6 --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/BootstrapCommandTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.main.Launch; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainTest; +import org.junit.jupiter.api.Test; + +@QuarkusMainTest +class BootstrapCommandTest { + + @Test + @Launch( + value = { + "bootstrap", + "-r", + "realm1", + "-r", + "realm2", + "-c", + "realm1,root,root,s3cr3t", + "-c", + "realm2,root,root,s3cr3t" + }) + public void testBootstrap(LaunchResult result) { + assertThat(result.getOutput()).contains("Bootstrap completed successfully."); + } +} diff --git a/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java new file mode 100644 index 0000000000..902cf8a10b --- /dev/null +++ b/quarkus/admin/src/test/java/org/apache/polaris/service/quarkus/admin/PurgeCommandTest.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.quarkus.admin; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.quarkus.test.junit.main.Launch; +import io.quarkus.test.junit.main.LaunchResult; +import io.quarkus.test.junit.main.QuarkusMainTest; +import org.junit.jupiter.api.Test; + +@QuarkusMainTest +class PurgeCommandTest { + + @Test + @Launch(value = {"purge", "-r", "realm1", "-r", "realm2"}) + public void testPurge(LaunchResult result) { + assertThat(result.getOutput()).contains("Purge completed successfully."); + } +} diff --git a/quarkus/server/README.md b/quarkus/server/README.md index 7fb773ece8..bac32962b1 100644 --- a/quarkus/server/README.md +++ b/quarkus/server/README.md @@ -8,5 +8,15 @@ To also build the Docker image, you can use the following command (a running Doc required): ```shell -./gradlew :polaris-quarkus-server:build -Dquarkus.container-image.build=true +./gradlew :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true ``` + +If you need to customize the Docker image, for example to push to a local registry, you can use the +following command: + +```shell +./gradlew :polaris-quarkus-server:build -Dquarkus.container-image.build=true \ + -Dquarkus.container-image.registry=localhost:5001 \ + -Dquarkus.container-image.group=apache \ + -Dquarkus.container-image.name=polaris-local +``` \ No newline at end of file diff --git a/quarkus/server/build.gradle.kts b/quarkus/server/build.gradle.kts index c27c77e47f..e4b0f9e000 100644 --- a/quarkus/server/build.gradle.kts +++ b/quarkus/server/build.gradle.kts @@ -39,6 +39,23 @@ dependencies { implementation(platform(libs.dnsjava)) } +quarkus { + quarkusBuildProperties.put("quarkus.package.type", "fast-jar") + // Pull manifest attributes from the "main" `jar` task to get the + // release-information into the jars generated by Quarkus. + quarkusBuildProperties.putAll( + provider { + tasks + .named("jar", Jar::class.java) + .get() + .manifest + .attributes + .map { e -> "quarkus.package.jar.manifest.attributes.\"${e.key}\"" to e.value.toString() } + .toMap() + } + ) +} + tasks.named("distZip") { dependsOn("quarkusBuild") } tasks.named("distTar") { dependsOn("quarkusBuild") } diff --git a/quarkus/server/src/main/resources/application.properties b/quarkus/server/src/main/resources/application.properties index 3f5018a776..3c628e380c 100644 --- a/quarkus/server/src/main/resources/application.properties +++ b/quarkus/server/src/main/resources/application.properties @@ -24,3 +24,4 @@ quarkus.container-image.push=false quarkus.container-image.registry=docker.io quarkus.container-image.group=apache quarkus.container-image.name=polaris +quarkus.container-image.additional-tags=latest diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index ae9f22496c..b0b5d6fe96 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -78,6 +78,7 @@ import org.apache.polaris.core.entity.PrincipalEntity; import org.apache.polaris.core.entity.TaskEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisEntityManager; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; @@ -304,7 +305,8 @@ public EntityCache getOrCreateEntityCache(RealmContext realmContext) { } @Override - public Map bootstrapRealms(List realms) { + public Map bootstrapRealms( + List realms, PolarisCredentialsBootstrap credentialsBootstrap) { throw new NotImplementedException("Bootstrapping realms is not supported"); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java index 963c917c65..74f1cc4cfe 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/test/PolarisIntegrationTestFixture.java @@ -42,6 +42,7 @@ import org.apache.polaris.core.entity.PolarisEntitySubType; import org.apache.polaris.core.entity.PolarisEntityType; import org.apache.polaris.core.entity.PolarisPrincipalSecrets; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.service.persistence.InMemoryPolarisMetaStoreManagerFactory; @@ -97,7 +98,8 @@ public PolarisIntegrationTestFixture( private PolarisPrincipalSecrets fetchAdminSecrets() { if (!(helper.metaStoreManagerFactory instanceof InMemoryPolarisMetaStoreManagerFactory)) { - helper.metaStoreManagerFactory.bootstrapRealms(List.of(realm)); + helper.metaStoreManagerFactory.bootstrapRealms( + List.of(realm), PolarisCredentialsBootstrap.fromEnvironment()); } RealmContext realmContext = diff --git a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java index 023fc7342a..db4f241e67 100644 --- a/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java +++ b/service/common/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java @@ -20,6 +20,7 @@ import io.smallrye.common.annotation.Identifier; import jakarta.annotation.Nonnull; +import jakarta.annotation.Nullable; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import java.util.Collections; @@ -31,6 +32,7 @@ import org.apache.polaris.core.auth.PolarisSecretsManager.PrincipalSecretsResult; import org.apache.polaris.core.context.RealmContext; import org.apache.polaris.core.persistence.LocalPolarisMetaStoreManagerFactory; +import org.apache.polaris.core.persistence.PolarisCredentialsBootstrap; import org.apache.polaris.core.persistence.PolarisMetaStoreManager; import org.apache.polaris.core.persistence.PolarisMetaStoreSession; import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl; @@ -67,9 +69,11 @@ protected PolarisTreeMapStore createBackingStore(@Nonnull PolarisDiagnostics dia @Override protected PolarisMetaStoreSession createMetaStoreSession( - @Nonnull PolarisTreeMapStore store, @Nonnull RealmContext realmContext) { + @Nonnull PolarisTreeMapStore store, + @Nonnull RealmContext realmContext, + @Nullable PolarisCredentialsBootstrap credentialsBootstrap) { return new PolarisTreeMapMetaStoreSessionImpl( - store, storageIntegration, secretsGenerator(realmContext)); + store, storageIntegration, secretsGenerator(realmContext, credentialsBootstrap)); } @Override @@ -94,7 +98,8 @@ public synchronized Supplier getOrCreateSessionSupplier private void bootstrapRealmAndPrintCredentials(String realmId) { Map results = - this.bootstrapRealms(Collections.singletonList(realmId)); + this.bootstrapRealms( + Collections.singletonList(realmId), PolarisCredentialsBootstrap.fromEnvironment()); bootstrappedRealms.add(realmId); PrincipalSecretsResult principalSecrets = results.get(realmId);