Skip to content

Commit

Permalink
Merge pull request #3617 from ingef/feature/api-formbackend-management
Browse files Browse the repository at this point in the history
Feature/api formbackend management
  • Loading branch information
thoniTUB authored Nov 12, 2024
2 parents 9c76e36 + 568dd00 commit e0dc145
Show file tree
Hide file tree
Showing 13 changed files with 535 additions and 132 deletions.
57 changes: 52 additions & 5 deletions backend/pom.xml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://maven.apache.org/POM/4.0.0"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

Expand All @@ -15,6 +15,8 @@
<apache-poi.version>5.2.2</apache-poi.version>
<apache-arrow.version>16.0.0</apache-arrow.version>
<apache-parquet.version>1.12.3</apache-parquet.version>
<prometheus-metrics.version>1.2.1</prometheus-metrics.version>
<openapi-generator.version>7.9.0</openapi-generator.version>
</properties>

<build>
Expand Down Expand Up @@ -75,6 +77,36 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator-maven-plugin</artifactId>
<version>${openapi-generator.version}</version>
<executions>
<execution>
<id>openapi-admin</id>
<goals>
<goal>generate</goal>
</goals>
<configuration>
<inputSpec>${project.basedir}/src/main/resources/openapi/admin-openapi.json</inputSpec>
<!-- Don't use jaxrs-jersey here because it does not allow dependency injection -->
<generatorName>jaxrs-spec</generatorName>
<modelsToGenerate>ApiResponse</modelsToGenerate>
<packageName>com.bakdata.conquery</packageName>
<!-- This package name is used by EndpointTestHelper to filter out openapi specs -->
<apiPackage>com.bakdata.conquery.models.api.openapi</apiPackage>
<modelPackage>com.bakdata.conquery.models.config</modelPackage>
<configOptions>
<!-- The implementation of these interfaces actually allows us to use DI (not possible with jaxrs-jersey) -->
<interfaceOnly>true</interfaceOnly>
<sourceFolder>src/main/java</sourceFolder>
<generateSupportingFiles>false</generateSupportingFiles>
<useJakartaEe>true</useJakartaEe>
</configOptions>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

Expand Down Expand Up @@ -383,17 +415,32 @@
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>prometheus-metrics-simpleclient-bridge</artifactId>
<version>1.2.1</version>
<version>${prometheus-metrics.version}</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>prometheus-metrics-exporter-servlet-jakarta</artifactId>
<version>1.2.1</version>
<version>${prometheus-metrics.version}</version>
</dependency>
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>prometheus-metrics-core</artifactId>
<version>1.2.1</version>
<version>${prometheus-metrics.version}</version>
</dependency>
<dependency>
<groupId>org.openapitools</groupId>
<artifactId>openapi-generator-maven-plugin</artifactId>
<version>${openapi-generator.version}</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-integration-jakarta</artifactId>
<version>2.2.25</version>
</dependency>
<dependency>
<groupId>io.swagger.core.v3</groupId>
<artifactId>swagger-jaxrs2-jakarta</artifactId>
<version>2.2.25</version>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import com.bakdata.conquery.tasks.QueryCleanupTask;
import com.bakdata.conquery.tasks.ReloadMetaStorageTask;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Throwables;
import io.dropwizard.core.setup.Environment;
import io.dropwizard.lifecycle.Managed;
import lombok.Getter;
Expand All @@ -53,12 +52,11 @@ public class ManagerNode implements Managed {
public static final String DEFAULT_NAME = "manager";

private final String name;

private final List<ResourcesProvider> providers = new ArrayList<>();
private Validator validator;
private AdminServlet admin;
private AuthorizationController authController;
private ScheduledExecutorService maintenanceService;
private final List<ResourcesProvider> providers = new ArrayList<>();
@Delegate(excludes = Managed.class)
private Manager manager;

Expand Down Expand Up @@ -88,7 +86,8 @@ public void run(Manager manager) throws InterruptedException {
formScanner = new FormScanner(config);


config.initialize(this);
// Init all plugins
config.getPlugins().forEach(pluginConfig -> pluginConfig.initialize(this));


// Initialization of internationalization
Expand Down Expand Up @@ -127,34 +126,11 @@ public void run(Manager manager) throws InterruptedException {
}
}

try {
formScanner.execute(null, null);
}
catch (Exception e) {
Throwables.throwIfUnchecked(e);
throw new RuntimeException(e);
}
formScanner.execute(null, null);

registerTasks(manager, environment, config);
}

private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) {
environment.admin().addTask(formScanner);
environment.admin().addTask(
new QueryCleanupTask(getMetaStorage(), Duration.of(
config.getQueries().getOldQueriesTime().getQuantity(),
config.getQueries().getOldQueriesTime().getUnit().toChronoUnit()
)));

environment.admin().addTask(new PermissionCleanupTask(getMetaStorage()));
manager.getAdminTasks().forEach(environment.admin()::addTask);
environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage()));

final ShutdownTask shutdown = new ShutdownTask();
environment.admin().addTask(shutdown);
environment.lifecycle().addServerLifecycleListener(shutdown);
}

private void configureApiServlet(ConqueryConfig config, Environment environment) {
ResourceConfig jerseyConfig = environment.jersey().getResourceConfig();
RESTServer.configure(config, jerseyConfig);
Expand All @@ -171,14 +147,6 @@ protected void configure() {
jerseyConfig.register(PathParamInjector.class);
}

private void loadMetaStorage() {
log.info("Opening MetaStorage");
getMetaStorage().openStores(getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage()), getEnvironment().metrics());
log.info("Loading MetaStorage");
getMetaStorage().loadData();
log.info("MetaStorage loaded {}", getMetaStorage());
}

@SneakyThrows(InterruptedException.class)
public void loadNamespaces() {

Expand All @@ -203,6 +171,31 @@ public void loadNamespaces() {
}
}

private void loadMetaStorage() {
log.info("Opening MetaStorage");
getMetaStorage().openStores(getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage()), getEnvironment().metrics());
log.info("Loading MetaStorage");
getMetaStorage().loadData();
log.info("MetaStorage loaded {}", getMetaStorage());
}

private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) {
environment.admin().addTask(formScanner);
environment.admin().addTask(
new QueryCleanupTask(getMetaStorage(), Duration.of(
config.getQueries().getOldQueriesTime().getQuantity(),
config.getQueries().getOldQueriesTime().getUnit().toChronoUnit()
)));

environment.admin().addTask(new PermissionCleanupTask(getMetaStorage()));
manager.getAdminTasks().forEach(environment.admin()::addTask);
environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage()));

final ShutdownTask shutdown = new ShutdownTask();
environment.admin().addTask(shutdown);
environment.lifecycle().addServerLifecycleListener(shutdown);
}

@Override
public void start() throws Exception {
manager.start();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
import java.util.Map;
import java.util.UUID;
import java.util.function.Function;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.client.Invocation;
import jakarta.ws.rs.client.WebTarget;
import jakarta.ws.rs.core.GenericType;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import com.bakdata.conquery.apiv1.forms.ExternalForm;
import com.bakdata.conquery.models.auth.entities.User;
Expand All @@ -16,16 +23,11 @@
import com.codahale.metrics.health.HealthCheck;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.dropwizard.health.check.http.HttpHealthCheck;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.Entity;
import jakarta.ws.rs.client.Invocation;
import jakarta.ws.rs.client.WebTarget;
import jakarta.ws.rs.core.GenericType;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;

@Slf4j
@Getter
public class ExternalFormBackendApi {

public final static String TASK_ID = "task-id";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;

import com.bakdata.conquery.commands.ManagerNode;
import com.bakdata.conquery.io.jackson.Injectable;
import com.bakdata.conquery.io.jackson.MutableInjectableValues;
import com.bakdata.conquery.io.jackson.serializer.CDateSetDeserializer;
Expand All @@ -31,8 +32,6 @@
import io.dropwizard.client.JerseyClientConfiguration;
import io.dropwizard.core.Configuration;
import io.dropwizard.validation.ValidationMethod;
import jakarta.validation.Valid;
import jakarta.validation.constraints.NotNull;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.NoArgsConstructor;
Expand Down Expand Up @@ -149,10 +148,6 @@ boolean isPluginUnique() {
return true;
}

public void initialize(ManagerNode node) {
plugins.forEach(config -> config.initialize((node)));
}

public <T extends PluginConfig> Optional<T> getPluginConfig(Class<T> type) {
return plugins.stream()
.filter(c -> type.isAssignableFrom(c.getClass()))
Expand Down
Loading

0 comments on commit e0dc145

Please sign in to comment.