From 897ba54e4397bc084646fab396362d054baf892b Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Fri, 17 May 2024 15:45:51 +0200 Subject: [PATCH] Implement support for multiple DB tenants in SQL mode (#3444) --- .../mode/local/LocalManagerProvider.java | 37 +++------- .../mode/local/LocalNamespaceHandler.java | 34 +++++++-- .../models/config/DatabaseConfig.java | 23 ++++++ .../models/config/SqlConnectorConfig.java | 22 +++--- .../models/worker/LocalNamespace.java | 40 +++++++++-- .../conquery/sql/DSLContextWrapper.java | 29 ++++++++ .../conquery/sql/DslContextFactory.java | 16 +++-- .../com/bakdata/conquery/sql/SqlContext.java | 11 --- .../sql/conquery/SqlExecutionManager.java | 5 +- .../sql/conversion/NodeConversions.java | 9 +-- .../conquery/sql/conversion/SqlConverter.java | 7 +- .../cqelement/ConversionContext.java | 4 +- .../conversion/dialect/HanaSqlDialect.java | 13 +--- .../conversion/dialect/PostgreSqlDialect.java | 13 +--- .../sql/conversion/dialect/SqlDialect.java | 8 +-- .../conversion/dialect/SqlDialectFactory.java | 14 ++++ .../conquery/util/TablePrimaryColumnUtil.java | 4 +- .../com/bakdata/conquery/util/io/Cloner.java | 11 +-- .../integration/IntegrationTests.java | 9 +-- .../integration/sql/CsvTableImporter.java | 10 +-- .../integration/sql/SqlStandaloneCommand.java | 18 +---- .../sql/dialect/HanaSqlIntegrationTests.java | 72 +++++++++---------- .../dialect/PostgreSqlIntegrationTests.java | 50 ++++++------- .../sql/dialect/TestContextProvider.java | 10 +-- .../sql/dialect/TestSqlConnectorConfig.java | 23 ++++++ .../sql/dialect/TestSqlDialectFactory.java | 16 +++++ 26 files changed, 310 insertions(+), 198 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 91588c3bf5..a6d5f63277 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -10,39 +10,30 @@ import com.bakdata.conquery.mode.ManagerProvider; import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; -import com.bakdata.conquery.sql.DslContextFactory; -import com.bakdata.conquery.sql.SqlContext; -import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; -import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; -import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; -import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; -import com.bakdata.conquery.sql.execution.SqlExecutionService; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; import io.dropwizard.core.setup.Environment; -import org.jooq.DSLContext; public class LocalManagerProvider implements ManagerProvider { private static final Supplier> EMPTY_NODE_PROVIDER = Collections::emptyList; - public DelegateManager provideManager(ConqueryConfig config, Environment environment) { + private final SqlDialectFactory dialectFactory; - InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + public LocalManagerProvider() { + this.dialectFactory = new SqlDialectFactory(); + } - SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); - DSLContext dslContext = DslContextFactory.create(sqlConnectorConfig); - SqlDialect sqlDialect = createSqlDialect(sqlConnectorConfig, dslContext); - SqlContext sqlContext = new SqlContext(sqlConnectorConfig, sqlDialect); + public LocalManagerProvider(SqlDialectFactory dialectFactory) { + this.dialectFactory = dialectFactory; + } - SqlExecutionService sqlExecutionService = new SqlExecutionService( - sqlDialect.getDSLContext(), - ResultSetProcessorFactory.create(sqlDialect) - ); + public DelegateManager provideManager(ConqueryConfig config, Environment environment) { - NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext, sqlExecutionService); + InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); DatasetRegistry datasetRegistry = ManagerProvider.createLocalDatasetRegistry(namespaceHandler, config, creator); creator.init(datasetRegistry); @@ -59,10 +50,4 @@ public DelegateManager provideManager(ConqueryConfig config, Env ); } - protected SqlDialect createSqlDialect(SqlConnectorConfig sqlConnectorConfig, DSLContext dslContext) { - return switch (sqlConnectorConfig.getDialect()) { - case POSTGRESQL -> new PostgreSqlDialect(dslContext); - case HANA -> new HanaSqlDialect(dslContext); - }; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index ce79a76c6c..52909c50ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -6,33 +6,57 @@ import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.mode.NamespaceSetupData; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.worker.LocalNamespace; -import com.bakdata.conquery.sql.SqlContext; +import com.bakdata.conquery.sql.DSLContextWrapper; +import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; +import com.bakdata.conquery.sql.conversion.SqlConverter; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; +import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; @RequiredArgsConstructor +@Slf4j public class LocalNamespaceHandler implements NamespaceHandler { private final ConqueryConfig config; private final InternalObjectMapperCreator mapperCreator; - private final SqlContext sqlContext; - private final SqlExecutionService sqlExecutionService; + private final SqlDialectFactory dialectFactory; @Override public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) { + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService); - ExecutionManager executionManager = new SqlExecutionManager(sqlContext, sqlExecutionService, metaStorage); + + SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); + DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset()); + + DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig); + DSLContext dslContext = dslContextWrapper.getDslContext(); + SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect()); + + SqlConverter sqlConverter = new SqlConverter(sqlDialect, dslContext, databaseConfig); + SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect)); + ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage); + SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); + return new LocalNamespace( namespaceData.getPreprocessMapper(), namespaceData.getCommunicationMapper(), namespaceStorage, executionManager, - sqlExecutionService, + dslContextWrapper, + sqlStorageHandler, namespaceData.getJobManager(), namespaceData.getFilterSearch(), namespaceData.getIndexService(), diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java new file mode 100644 index 0000000000..b42aa9bf0e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.models.config; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class DatabaseConfig { + + private static final String DEFAULT_PRIMARY_COLUMN = "pid"; + + private Dialect dialect; + + private String databaseUsername; + + private String databasePassword; + + private String jdbcConnectionUrl; + + @Builder.Default + private String primaryColumn = DEFAULT_PRIMARY_COLUMN; + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index 7393a58f01..2a65aa5ca6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -1,8 +1,13 @@ package com.bakdata.conquery.models.config; +import java.util.Map; + +import com.bakdata.conquery.models.datasets.Dataset; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; @Data @@ -11,22 +16,21 @@ @AllArgsConstructor public class SqlConnectorConfig { - public static final String DEFAULT_PRIMARY_COLUMN = "pid"; - boolean enabled; - private Dialect dialect; - /** * Determines if generated SQL should be formatted. */ private boolean withPrettyPrinting; - private String databaseUsername; - - private String databasePassword; + /** + * Keys must match the name of existing {@link Dataset}s. + */ + @Getter(AccessLevel.PRIVATE) + private Map databaseConfigs; - private String jdbcConnectionUrl; + public DatabaseConfig getDatabaseConfig(Dataset dataset) { + return databaseConfigs.get(dataset.getName()); + } - private String primaryColumn = DEFAULT_PRIMARY_COLUMN; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index 5663baedc5..e5a2c8f5aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.worker; +import java.io.IOException; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -13,31 +14,34 @@ import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.FilterSearch; -import com.bakdata.conquery.sql.execution.SqlExecutionService; +import com.bakdata.conquery.sql.DSLContextWrapper; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; @Getter +@Slf4j public class LocalNamespace extends Namespace { - private final SqlExecutionService sqlExecutionService; - + private final DSLContextWrapper dslContextWrapper; private final SqlStorageHandler storageHandler; public LocalNamespace( ObjectMapper preprocessMapper, ObjectMapper communicationMapper, NamespaceStorage storage, - ExecutionManager executionManager, - SqlExecutionService sqlExecutionService, + ExecutionManager executionManager, + DSLContextWrapper dslContextWrapper, + SqlStorageHandler storageHandler, JobManager jobManager, FilterSearch filterSearch, IndexService indexService, List injectables ) { super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); - this.sqlExecutionService = sqlExecutionService; - this.storageHandler = new SqlStorageHandler(sqlExecutionService); + this.dslContextWrapper = dslContextWrapper; + this.storageHandler = storageHandler; } @Override @@ -52,4 +56,26 @@ void registerColumnValuesInSearch(Set columns) { getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet())); } } + + @Override + public void close() { + closeDslContextWrapper(); + super.close(); + } + + @Override + public void remove() { + closeDslContextWrapper(); + super.remove(); + } + + private void closeDslContextWrapper() { + try { + dslContextWrapper.close(); + } + catch (IOException e) { + log.warn("Could not close namespace's {} DSLContext/Datasource directly", getDataset().getId(), e); + } + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java new file mode 100644 index 0000000000..cf031678ad --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql; + +import java.io.Closeable; +import java.io.IOException; + +import com.zaxxer.hikari.HikariDataSource; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +/** + * Provides access to a configured {@link DSLContext} and enables closing the underlying connection pool. + */ +@RequiredArgsConstructor +public class DSLContextWrapper implements Closeable { + + @Getter + private final DSLContext dslContext; + + private final HikariDataSource dataSource; + + @Override + public void close() throws IOException { + // Hikari opens a connection pool under the hood which we won't be able to close after passing it to the DSLContext. + // That's why we keep the HikariDataSource reference. + dataSource.close(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java index 980aa597bb..b7ea1916d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.sql; -import javax.sql.DataSource; - +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -12,25 +11,28 @@ public class DslContextFactory { - public static DSLContext create(SqlConnectorConfig config) { + public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig connectorConfig) { + HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(config.getJdbcConnectionUrl()); hikariConfig.setUsername(config.getDatabaseUsername()); hikariConfig.setPassword(config.getDatabasePassword()); - DataSource dataSource = new HikariDataSource(hikariConfig); + HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig); Settings settings = new Settings() - .withRenderFormatted(config.isWithPrettyPrinting()) + .withRenderFormatted(connectorConfig.isWithPrettyPrinting()) // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED); - return DSL.using( - dataSource, + DSLContext dslContext = DSL.using( + hikariDataSource, config.getDialect().getJooqDialect(), settings ); + + return new DSLContextWrapper(dslContext, hikariDataSource); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java b/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java deleted file mode 100644 index bca5ca6f2e..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.bakdata.conquery.sql; - -import com.bakdata.conquery.models.config.SqlConnectorConfig; -import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; -import lombok.Value; - -@Value -public class SqlContext { - SqlConnectorConfig config; - SqlDialect sqlDialect; -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index a0c1616396..5fb2e663ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.sql.SqlContext; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.execution.SqlExecutionResult; @@ -29,10 +28,10 @@ public class SqlExecutionManager extends ExecutionManager { private final SqlConverter converter; private final Map> runningExecutions; - public SqlExecutionManager(final SqlContext context, SqlExecutionService sqlExecutionService, MetaStorage storage) { + public SqlExecutionManager(SqlConverter sqlConverter, SqlExecutionService sqlExecutionService, MetaStorage storage) { super(storage); + this.converter = sqlConverter; this.executionService = sqlExecutionService; - this.converter = new SqlConverter(context.getSqlDialect(), context.getConfig()); this.runningExecutions = new HashMap<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 81b936e558..769fe973ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.sql.conversion; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import org.jooq.DSLContext; /** * Entry point for converting {@link QueryDescription} to an SQL query. @@ -13,10 +14,10 @@ public class NodeConversions extends Conversions { private final SqlDialect dialect; - private final SqlConnectorConfig config; + private final DatabaseConfig config; - public NodeConversions(SqlDialect dialect, SqlConnectorConfig config) { - super(dialect.getNodeConverters()); + public NodeConversions(SqlDialect dialect, DSLContext dslContext, DatabaseConfig config) { + super(dialect.getNodeConverters(dslContext)); this.dialect = dialect; this.config = config; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index 261195c7a7..af2a6fa681 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -1,17 +1,18 @@ package com.bakdata.conquery.sql.conversion; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import org.jooq.DSLContext; public class SqlConverter { private final NodeConversions nodeConversions; - public SqlConverter(SqlDialect dialect, SqlConnectorConfig config) { - this.nodeConversions = new NodeConversions(dialect, config); + public SqlConverter(SqlDialect dialect, DSLContext dslContext, DatabaseConfig config) { + this.nodeConversions = new NodeConversions(dialect, dslContext, config); } public SqlQuery convert(QueryDescription queryDescription) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index ad10c04235..f55754e70f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.NodeConversions; @@ -25,7 +25,7 @@ @Builder(toBuilder = true) public class ConversionContext implements Context { - SqlConnectorConfig config; + DatabaseConfig config; NodeConversions nodeConversions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java index 17e54c5d84..72228209df 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java @@ -16,29 +16,22 @@ public class HanaSqlDialect implements SqlDialect { private final IntervalPacker hanaIntervalPacker; private final SqlDateAggregator hanaSqlDateAggregator; private final DefaultSqlCDateSetParser defaultNotationParser; - private final DSLContext dslContext; - public HanaSqlDialect(DSLContext dslContext) { - this.dslContext = dslContext; + public HanaSqlDialect() { this.hanaSqlFunctionProvider = new HanaSqlFunctionProvider(); this.hanaIntervalPacker = new AnsiSqlIntervalPacker(); this.hanaSqlDateAggregator = new AnsiSqlDateAggregator(this.hanaIntervalPacker); this.defaultNotationParser = new DefaultSqlCDateSetParser(); } - @Override - public DSLContext getDSLContext() { - return this.dslContext; - } - @Override public SqlCDateSetParser getCDateSetParser() { return this.defaultNotationParser; } @Override - public List> getNodeConverters() { - return getDefaultNodeConverters(); + public List> getNodeConverters(DSLContext dslContext) { + return getDefaultNodeConverters(dslContext); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java index 014308bbe2..b5f385343a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java @@ -16,21 +16,14 @@ public class PostgreSqlDialect implements SqlDialect { private final IntervalPacker postgresqlIntervalPacker; private final SqlDateAggregator postgresqlDateAggregator; private final DefaultSqlCDateSetParser defaultNotationParser; - private final DSLContext dslContext; - public PostgreSqlDialect(DSLContext dslContext) { - this.dslContext = dslContext; + public PostgreSqlDialect() { this.postgresqlFunctionProvider = new PostgreSqlFunctionProvider(); this.postgresqlIntervalPacker = new PostgreSqlIntervalPacker(this.postgresqlFunctionProvider); this.postgresqlDateAggregator = new PostgreSqlDateAggregator(this.postgresqlFunctionProvider); this.defaultNotationParser = new DefaultSqlCDateSetParser(); } - @Override - public DSLContext getDSLContext() { - return this.dslContext; - } - @Override public SqlCDateSetParser getCDateSetParser() { return this.defaultNotationParser; @@ -42,8 +35,8 @@ public boolean supportsSingleColumnRanges() { } @Override - public List> getNodeConverters() { - return getDefaultNodeConverters(); + public List> getNodeConverters(DSLContext dslContext) { + return getDefaultNodeConverters(dslContext); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 733112ac18..d201bdf720 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -38,9 +38,7 @@ public interface SqlDialect { SqlDateAggregator getDateAggregator(); - List> getNodeConverters(); - - DSLContext getDSLContext(); + List> getNodeConverters(DSLContext context); SqlCDateSetParser getCDateSetParser(); @@ -52,9 +50,9 @@ default boolean supportsSingleColumnRanges() { return false; } - default List> getDefaultNodeConverters() { + default List> getDefaultNodeConverters(DSLContext dslContext) { - QueryStepTransformer queryStepTransformer = new QueryStepTransformer(getDSLContext()); + QueryStepTransformer queryStepTransformer = new QueryStepTransformer(dslContext); FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); return List.of( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java new file mode 100644 index 0000000000..3480926432 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import com.bakdata.conquery.models.config.Dialect; + +public class SqlDialectFactory { + + public SqlDialect createSqlDialect(Dialect dialect) { + return switch (dialect) { + case POSTGRESQL -> new PostgreSqlDialect(); + case HANA -> new HanaSqlDialect(); + }; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java index 491bc04338..8ed9cafaf9 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.util; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.datasets.Table; import org.jooq.Field; import org.jooq.impl.DSL; public class TablePrimaryColumnUtil { - public static Field findPrimaryColumn(Table table, SqlConnectorConfig sqlConfig) { + public static Field findPrimaryColumn(Table table, DatabaseConfig sqlConfig) { String primaryColumnName = table.getPrimaryColum() == null ? sqlConfig.getPrimaryColumn() : table.getPrimaryColum().getName(); diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java index 30829dc1c7..9c217e30d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java @@ -25,6 +25,7 @@ public static ConqueryConfig clone(ConqueryConfig config, Map, T> i ConqueryConfig.class ); clone.setLoggingFactory(config.getLoggingFactory()); + clone.setSqlConnectorConfig(config.getSqlConnectorConfig()); return clone; } catch (IOException e) { @@ -35,11 +36,11 @@ public static ConqueryConfig clone(ConqueryConfig config, Map, T> i public static T clone(T element, Injectable injectable, Class valueType) { try { return injectable - .injectIntoNew(Jackson.BINARY_MAPPER) - .readValue( - Jackson.BINARY_MAPPER.writeValueAsBytes(element), - valueType - ); + .injectIntoNew(Jackson.BINARY_MAPPER) + .readValue( + Jackson.BINARY_MAPPER.writeValueAsBytes(element), + valueType + ); } catch (IOException e) { throw new IllegalStateException("Failed to clone the CQElement " + element, e); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 5034162860..d48f99401c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -23,13 +23,14 @@ import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.integration.json.WorkerTestDataImporter; +import com.bakdata.conquery.integration.sql.dialect.TestSqlConnectorConfig; import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.util.support.ConfigOverride; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestConquery; @@ -132,18 +133,18 @@ public Stream programmaticTests(TestDataImporter testImporter, Stan } @SneakyThrows - public Stream sqlProgrammaticTests(SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public Stream sqlProgrammaticTests(DatabaseConfig databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL); } @SneakyThrows - public List sqlQueryTests(SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public List sqlQueryTests(DatabaseConfig databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); ResourceTree tree = scanForResources(testRoot, SQL_TEST_PATTERN); - return collectTestTree(tree, testRoot, testDataImporter, sqlConfig.getDialect()); + return collectTestTree(tree, testRoot, testDataImporter, databaseConfig.getDialect()); } private List collectTestTree(ResourceTree tree, String testRoot, TestDataImporter testImporter, Dialect sqlDialect) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java index 046887dbae..010d6ee112 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; import com.google.common.base.Strings; @@ -43,14 +43,14 @@ public class CsvTableImporter { private final DateRangeParser dateRangeParser; private final CsvParser csvReader; private final TestSqlDialect testSqlDialect; - private final SqlConnectorConfig sqlConnectorConfig; + private final DatabaseConfig databaseConfig; - public CsvTableImporter(DSLContext dslContext, TestSqlDialect testSqlDialect, SqlConnectorConfig sqlConnectorConfig) { + public CsvTableImporter(DSLContext dslContext, TestSqlDialect testSqlDialect, DatabaseConfig databaseConfig) { this.dslContext = dslContext; this.dateRangeParser = new DateRangeParser(new ConqueryConfig()); this.csvReader = new CSVConfig().withSkipHeader(true).createParser(); this.testSqlDialect = testSqlDialect; - this.sqlConnectorConfig = sqlConnectorConfig; + this.databaseConfig = databaseConfig; } /** @@ -121,7 +121,7 @@ private Field createField(RequiredColumn requiredColumn) { }; // Set all columns except 'pid' to nullable, important for ClickHouse compatibility - if (!requiredColumn.getName().equals(sqlConnectorConfig.getPrimaryColumn())) { + if (!requiredColumn.getName().equals(databaseConfig.getPrimaryColumn())) { dataType = dataType.nullable(true); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java index 37c74be44b..6504e804a2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java @@ -7,14 +7,11 @@ import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; -import com.bakdata.conquery.integration.sql.dialect.HanaSqlIntegrationTests; -import com.bakdata.conquery.integration.sql.dialect.PostgreSqlIntegrationTests; +import com.bakdata.conquery.integration.sql.dialect.TestSqlDialectFactory; import com.bakdata.conquery.mode.DelegateManager; import com.bakdata.conquery.mode.local.LocalManagerProvider; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.worker.LocalNamespace; -import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.util.io.ConqueryMDC; import io.dropwizard.core.cli.ServerCommand; import io.dropwizard.core.setup.Bootstrap; @@ -22,7 +19,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import net.sourceforge.argparse4j.inf.Namespace; -import org.jooq.DSLContext; @Slf4j @Getter @@ -42,7 +38,7 @@ public SqlStandaloneCommand(Conquery conquery) { public void startStandalone(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception { ConqueryMDC.setLocation("ManagerNode"); log.debug("Starting ManagerNode"); - this.manager = new TestLocalManagerProvider().provideManager(config, environment); + this.manager = new LocalManagerProvider(new TestSqlDialectFactory()).provideManager(config, environment); this.conquery.setManagerNode(managerNode); this.conquery.run(manager); // starts the Jersey Server @@ -74,14 +70,6 @@ public void run(Bootstrap bootstrap, Namespace namespace, Conque startStandalone(environment, namespace, configuration); } - private static class TestLocalManagerProvider extends LocalManagerProvider { - @Override - protected SqlDialect createSqlDialect(SqlConnectorConfig sqlConnectorConfig, DSLContext dslContext) { - return switch (sqlConnectorConfig.getDialect()) { - case POSTGRESQL -> new PostgreSqlIntegrationTests.TestPostgreSqlDialect(dslContext); - case HANA -> new HanaSqlIntegrationTests.TestHanaDialect(dslContext); - }; - } - } + } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java index 5ca44ad7d6..9ec5ea4d3e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java @@ -21,8 +21,9 @@ import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.integration.sql.CsvTableImporter; import com.bakdata.conquery.integration.sql.testcontainer.hana.HanaContainer; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; @@ -50,6 +51,7 @@ public class HanaSqlIntegrationTests extends IntegrationTests { private final static DockerImageName HANA_IMAGE = DockerImageName.parse("saplabs/hanaexpress:2.00.072.00.20231123.1"); private static final Path TMP_HANA_MOUNT_DIR = Paths.get("/tmp/data/hana"); private static boolean useLocalHanaDb = true; + private static DSLContextWrapper dslContextWrapper; static { final String USE_LOCAL_HANA_DB = System.getenv("USE_LOCAL_HANA_DB"); @@ -72,14 +74,15 @@ public Stream sqlBackendTests() { log.info("Running HANA tests with %s.".formatted(provider.getClass().getSimpleName())); - DSLContext dslContext = provider.getDslContext(); - SqlConnectorConfig config = provider.getSqlConnectorConfig(); - TestHanaDialect testHanaDialect = new TestHanaDialect(dslContext); - TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testHanaDialect, config)); + dslContextWrapper = provider.getDslContextWrapper(); + DatabaseConfig databaseConfig = provider.getDatabaseConfig(); + TestSqlConnectorConfig config = provider.getSqlConnectorConfig(); + TestHanaDialect testHanaDialect = new TestHanaDialect(); + TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testHanaDialect, databaseConfig)); return Stream.concat( - super.sqlProgrammaticTests(config, testDataImporter), - super.sqlQueryTests(config, testDataImporter).stream() + super.sqlProgrammaticTests(databaseConfig, config, testDataImporter), + super.sqlQueryTests(databaseConfig, config, testDataImporter).stream() ); } @@ -102,6 +105,9 @@ public static void prepareTmpHanaDir() { @SneakyThrows @AfterAll public static void tearDownClass() { + + dslContextWrapper.close(); + if (!Files.exists(TMP_HANA_MOUNT_DIR)) { return; } @@ -116,11 +122,6 @@ public static class TestHanaDialect extends HanaSqlDialect implements TestSqlDia public static final MockDateNowSupplier DATE_NOW_SUPPLIER = new MockDateNowSupplier(); - - public TestHanaDialect(DSLContext dslContext) { - super(dslContext); - } - @Override public DateNowSupplier getDateNowSupplier() { return DATE_NOW_SUPPLIER; @@ -157,8 +158,9 @@ public String createDropTableStatement(Table table, DSLContext dslContex @Getter private static class HanaTestcontainerContextProvider implements TestContextProvider { - private final DSLContext dslContext; - private final SqlConnectorConfig sqlConnectorConfig; + private final DSLContextWrapper dslContextWrapper; + private final DatabaseConfig databaseConfig; + private final TestSqlConnectorConfig sqlConnectorConfig; @Container private final HanaContainer hanaContainer; @@ -167,17 +169,14 @@ public HanaTestcontainerContextProvider() { this.hanaContainer = new HanaContainer<>(HANA_IMAGE) .withFileSystemBind(TMP_HANA_MOUNT_DIR.toString(), "/home/secrets"); this.hanaContainer.start(); - - this.sqlConnectorConfig = SqlConnectorConfig.builder() - .enabled(true) - .dialect(Dialect.HANA) - .withPrettyPrinting(true) - .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) - .databaseUsername(hanaContainer.getUsername()) - .databasePassword(hanaContainer.getPassword()) - .primaryColumn("pid") - .build(); - this.dslContext = DslContextFactory.create(sqlConnectorConfig); + this.databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) + .databaseUsername(hanaContainer.getUsername()) + .databasePassword(hanaContainer.getPassword()) + .build(); + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); + this.dslContextWrapper = DslContextFactory.create(this.databaseConfig, sqlConnectorConfig); } } @@ -190,20 +189,19 @@ private static class RemoteHanaContextProvider implements TestContextProvider { private final static String CONNECTION_URL = "jdbc:sap://%s:%s/databaseName=HXE&encrypt=true&validateCertificate=false".formatted(HOST, PORT); private final static String USERNAME = System.getenv("CONQUERY_SQL_USER"); private final static String PASSWORD = System.getenv("CONQUERY_SQL_PASSWORD"); - private final DSLContext dslContext; - private final SqlConnectorConfig sqlConnectorConfig; + private final DSLContextWrapper dslContextWrapper; + private final DatabaseConfig databaseConfig; + private final TestSqlConnectorConfig sqlConnectorConfig; public RemoteHanaContextProvider() { - this.sqlConnectorConfig = SqlConnectorConfig.builder() - .enabled(true) - .dialect(Dialect.HANA) - .withPrettyPrinting(true) - .jdbcConnectionUrl(CONNECTION_URL) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .primaryColumn("pid") - .build(); - this.dslContext = DslContextFactory.create(sqlConnectorConfig); + this.databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(CONNECTION_URL) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); + this.dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 6e7dfaa61e..3a31b73d7c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.integration.sql.dialect; +import java.io.IOException; import java.util.stream.Stream; import com.bakdata.conquery.TestTags; @@ -7,10 +8,11 @@ import com.bakdata.conquery.integration.IntegrationTests; import com.bakdata.conquery.integration.json.SqlTestDataImporter; import com.bakdata.conquery.integration.sql.CsvTableImporter; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.i18n.I18n; +import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; @@ -20,7 +22,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; -import org.jooq.DSLContext; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.Tag; @@ -39,8 +41,10 @@ public class PostgreSqlIntegrationTests extends IntegrationTests { private static final String DATABASE_NAME = "test"; private static final String USERNAME = "user"; private static final String PASSWORD = "pass"; - private static DSLContext dslContext; - private static SqlConnectorConfig sqlConfig; + + private static DSLContextWrapper dslContextWrapper; + private static DatabaseConfig databaseConfig; + private static TestSqlConnectorConfig sqlConfig; private static TestSqlDialect testSqlDialect; private static SqlTestDataImporter testDataImporter; @@ -58,18 +62,21 @@ public PostgreSqlIntegrationTests() { @BeforeAll static void before() { POSTGRESQL_CONTAINER.start(); - sqlConfig = SqlConnectorConfig.builder() - .enabled(true) - .dialect(Dialect.POSTGRESQL) - .jdbcConnectionUrl(POSTGRESQL_CONTAINER.getJdbcUrl()) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .withPrettyPrinting(true) - .primaryColumn("pid") - .build(); - dslContext = DslContextFactory.create(sqlConfig); - testSqlDialect = new TestPostgreSqlDialect(dslContext); - testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testSqlDialect, sqlConfig)); + databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(POSTGRESQL_CONTAINER.getJdbcUrl()) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + sqlConfig = new TestSqlConnectorConfig(databaseConfig); + dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConfig); + testSqlDialect = new TestPostgreSqlDialect(); + testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testSqlDialect, databaseConfig)); + } + + @AfterAll + static void after() throws IOException { + dslContextWrapper.close(); } @Test @@ -77,7 +84,7 @@ static void before() { public void shouldThrowException() { // This can be removed as soon as we switch to a full integration test including the REST API I18n.init(); - SqlExecutionService executionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(testSqlDialect)); + SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), ResultSetProcessorFactory.create(testSqlDialect)); SqlQuery validQuery = new TestSqlQuery("SELECT 1"); Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery)); @@ -92,8 +99,8 @@ public void shouldThrowException() { @Tag(TestTags.INTEGRATION_SQL_BACKEND) public Stream sqlBackendTests() { return Stream.concat( - super.sqlProgrammaticTests(sqlConfig, testDataImporter), - super.sqlQueryTests(sqlConfig, testDataImporter).stream() + super.sqlProgrammaticTests(databaseConfig, sqlConfig, testDataImporter), + super.sqlQueryTests(databaseConfig, sqlConfig, testDataImporter).stream() ); } @@ -101,11 +108,6 @@ public static class TestPostgreSqlDialect extends PostgreSqlDialect implements T public static final MockDateNowSupplier DATE_NOW_SUPPLIER = new MockDateNowSupplier(); - - public TestPostgreSqlDialect(DSLContext dslContext) { - super(dslContext); - } - @Override public DateNowSupplier getDateNowSupplier() { return DATE_NOW_SUPPLIER; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java index c111f83d8b..d5cf8eb140 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java @@ -1,12 +1,14 @@ package com.bakdata.conquery.integration.sql.dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; -import org.jooq.DSLContext; +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.sql.DSLContextWrapper; public interface TestContextProvider { - SqlConnectorConfig getSqlConnectorConfig(); + DatabaseConfig getDatabaseConfig(); - DSLContext getDslContext(); + TestSqlConnectorConfig getSqlConnectorConfig(); + + DSLContextWrapper getDslContextWrapper(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java new file mode 100644 index 0000000000..ec5dccddc8 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import java.util.Map; + +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.Dataset; + +public class TestSqlConnectorConfig extends SqlConnectorConfig { + + private final DatabaseConfig databaseConfig; + + public TestSqlConnectorConfig(DatabaseConfig databaseConfig) { + super(true, true, Map.of()); + this.databaseConfig = databaseConfig; + } + + @Override + public DatabaseConfig getDatabaseConfig(Dataset dataset) { + return databaseConfig; + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java new file mode 100644 index 0000000000..cadd8cb746 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java @@ -0,0 +1,16 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; + +public class TestSqlDialectFactory extends SqlDialectFactory { + + @Override + public SqlDialect createSqlDialect(Dialect dialect) { + return switch (dialect) { + case POSTGRESQL -> new PostgreSqlIntegrationTests.TestPostgreSqlDialect(); + case HANA -> new HanaSqlIntegrationTests.TestHanaDialect(); + }; + } +}