Skip to content

Commit

Permalink
added tiling options to CLI
Browse files Browse the repository at this point in the history
  • Loading branch information
@clausnagel
clausnagel committed Oct 30, 2024
1 parent 07874a9 commit 441cc04
Show file tree
Hide file tree
Showing 9 changed files with 522 additions and 72 deletions.
14 changes: 7 additions & 7 deletions citydb-cli/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,16 @@ plugins {
}

dependencies {
api project(':citydb-config')
api project(':citydb-database')
api project(':citydb-io')
api project(':citydb-logging')
api project(':citydb-operation')
api project(':citydb-plugin')
api project(':citydb-query')
api project(':citydb-tiling')
api 'info.picocli:picocli:4.7.6'
implementation project(':citydb-config')
implementation project(':citydb-io')
implementation project(':citydb-io-citygml')
implementation project(':citydb-logging')
implementation project(':citydb-database')
implementation project(':citydb-database-postgres')
implementation project(':citydb-operation')
implementation project(':citydb-query')
}

processResources {
Expand Down
15 changes: 8 additions & 7 deletions citydb-cli/src/main/java/module-info.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
module org.citydb.cli {
requires org.citydb.config;
requires org.citygml4j.core;
requires org.citydb.io;
requires org.citydb.io.citygml;
requires org.citydb.logging;
requires org.citydb.database;
requires org.citydb.operation;
requires org.citydb.query;
requires transitive org.citydb.config;
requires transitive org.citygml4j.core;
requires transitive org.citydb.database;
requires transitive org.citydb.io;
requires transitive org.citydb.logging;
requires transitive org.citydb.operation;
requires transitive org.citydb.plugin;
requires transitive org.citydb.query;
requires transitive org.citydb.tiling;
requires transitive info.picocli;

exports org.citydb.cli;
Expand Down
1 change: 1 addition & 0 deletions citydb-cli/src/main/java/org/citydb/cli/Launcher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
scope = CommandLine.ScopeType.INHERIT,
description = "Command-line interface for the 3D City Database.",
synopsisSubcommandLabel = "COMMAND",
abbreviateSynopsis = true,
mixinStandardHelpOptions = true,
versionProvider = Launcher.class,
showAtFileInUsageHelp = true,
Expand Down
2 changes: 1 addition & 1 deletion citydb-cli/src/main/java/org/citydb/cli/deleter/DeleteCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ public Integer call() throws ExecutionException {
}

try {
logger.info("Querying features matching the request...");
logger.debug("Querying features matching the request...");
logger.trace("Using SQL query:\n{}", () -> helper.getFormattedSql(executor.getSelect(),
databaseManager.getAdapter()));

Expand Down
169 changes: 113 additions & 56 deletions citydb-cli/src/main/java/org/citydb/cli/exporter/ExportController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
import org.citydb.cli.ExecutionException;
import org.citydb.cli.common.*;
import org.citydb.cli.exporter.options.QueryOptions;
import org.citydb.cli.exporter.options.TilingOptions;
import org.citydb.cli.exporter.util.SequentialWriter;
import org.citydb.cli.exporter.util.TilingHelper;
import org.citydb.cli.util.CommandHelper;
import org.citydb.cli.util.FeatureStatistics;
import org.citydb.config.Config;
Expand All @@ -52,6 +54,9 @@
import org.citydb.query.executor.QueryResult;
import org.citydb.query.filter.encoding.FilterParseException;
import org.citydb.query.util.QueryHelper;
import org.citydb.tiling.Tile;
import org.citydb.tiling.TileIterator;
import org.citydb.tiling.Tiling;
import picocli.CommandLine;

import java.nio.file.Path;
Expand All @@ -77,7 +82,11 @@ public abstract class ExportController implements Command {

@CommandLine.ArgGroup(exclusive = false, order = Integer.MAX_VALUE,
heading = "Query and filter options:%n")
private QueryOptions queryOptions;
protected QueryOptions queryOptions;

@CommandLine.ArgGroup(exclusive = false, order = Integer.MAX_VALUE,
heading = "Tiling options:%n")
protected TilingOptions tilingOptions;

@CommandLine.ArgGroup(exclusive = false, order = Integer.MAX_VALUE,
heading = "Database connection options:%n")
Expand Down Expand Up @@ -119,71 +128,97 @@ protected boolean doExport() throws ExecutionException {
WriteOptions writeOptions = getWriteOptions(exportOptions, databaseManager.getAdapter());
writeOptions.getFormatOptions().set(getFormatOptions(writeOptions.getFormatOptions()));

Query query = getQuery(exportOptions);
QueryExecutor executor = helper.getQueryExecutor(query,
SqlBuildOptions.defaults().omitDistinct(true),
tempDirectory,
databaseManager.getAdapter());

FeatureStatistics statistics = new FeatureStatistics(databaseManager.getAdapter());
helper.logIndexStatus(Level.INFO, databaseManager.getAdapter());
initialize(exportOptions, writeOptions, databaseManager);

try (OutputFile outputFile = builder.newOutputFile(outputFileOptions.getFile());
FeatureWriter writer = createWriter(query, ioAdapter)) {
Exporter exporter = Exporter.newInstance();
exportOptions.setOutputFile(outputFile);

AtomicLong counter = new AtomicLong();

logger.info("Exporting to {} file {}.", ioManager.getFileFormat(ioAdapter), outputFile.getFile());
writer.initialize(outputFile, writeOptions);

logger.info("Querying features matching the request...");
logger.trace("Using SQL query:\n{}", () -> helper.getFormattedSql(executor.getSelect(),
databaseManager.getAdapter()));

long sequenceId = 1;
try (QueryResult result = executor.executeQuery()) {
exporter.startSession(databaseManager.getAdapter(), exportOptions);
while (shouldRun && result.hasNext()) {
long id = result.getId();
exporter.exportFeature(id, sequenceId++).whenComplete((feature, t) -> {
if (feature != null) {
try {
writer.write(feature, (success, e) -> {
if (success == Boolean.TRUE) {
statistics.add(feature);
long count = counter.incrementAndGet();
if (count % 1000 == 0) {
logger.info("{} features exported.", count);
}
} else {
Query query = getQuery(exportOptions);
Tiling tiling = getTiling(exportOptions);
FeatureStatistics statistics = new FeatureStatistics(databaseManager.getAdapter());
AtomicLong counter = new AtomicLong();

try {
TilingHelper tilingHelper = TilingHelper.of(tiling, query, databaseManager.getAdapter());
if (tilingHelper.isUseTiling()) {
logger.info("Creating {} tile(s) based on provided tiling scheme.",
tilingHelper.getTileMatrix().size());
}

TileIterator iterator = tilingHelper.getTileMatrix().getTileIterator();
while (iterator.hasNext()) {
Tile tile = iterator.next();
QueryExecutor executor = helper.getQueryExecutor(tilingHelper.getTileQuery(tile),
SqlBuildOptions.defaults()
.omitDistinct(true)
.withColumn(tilingHelper.isUseTiling() ? "envelope" : null),
tempDirectory,
databaseManager.getAdapter());

Path file = tilingHelper.getOutputFile(outputFileOptions.getFile(), tile);
FeatureStatistics tileStatistics = new FeatureStatistics(databaseManager.getAdapter());

try (OutputFile outputFile = builder.newOutputFile(file);
FeatureWriter writer = createWriter(query, ioAdapter)) {
Exporter exporter = Exporter.newInstance();
exportOptions.setOutputFile(outputFile);

logger.info("{}Exporting to {} file {}.", getTileCounter(tilingHelper, tile),
ioManager.getFileFormat(ioAdapter), outputFile.getFile());
writer.initialize(outputFile, writeOptions);

logger.debug("Querying features matching the request...");
logger.trace("Using SQL query:\n{}", () -> helper.getFormattedSql(executor.getSelect(),
databaseManager.getAdapter()));

long sequenceId = 1;
try (QueryResult result = executor.executeQuery()) {
exporter.startSession(databaseManager.getAdapter(), exportOptions);
while (shouldRun && result.hasNext()) {
long id = result.getId();

if (tilingHelper.isUseTiling() && !tile.isOnTile(databaseManager.getAdapter()
.getGeometryAdapter()
.getEnvelope(result.get(rs -> rs.getObject("envelope"))))) {
continue;
}

exporter.exportFeature(id, sequenceId++).whenComplete((feature, t) -> {
if (feature != null) {
try {
writer.write(feature, (success, e) -> {
if (success == Boolean.TRUE) {
tileStatistics.add(feature);
long count = counter.incrementAndGet();
if (count % 1000 == 0) {
logger.info("{} features exported.", count);
}
} else {
abort(feature, id, e);
}
});
} catch (Throwable e) {
abort(feature, id, e);
}
});
} catch (Throwable e) {
abort(feature, id, e);
}
} else {
abort(null, id, t);
} else {
abort(null, id, t);
}
});
}
});
} finally {
exporter.closeSession();
}
} catch (Throwable e) {
logger.warn("Database export aborted due to an error.");
throw new ExecutionException("A fatal error has occurred during export.", e);
} finally {
statistics.merge(tileStatistics);
if (tilingHelper.isUseTiling()) {
logStatistics(tileStatistics, "Tile export summary:", Level.DEBUG);
}
}
} finally {
exporter.closeSession();
}
} catch (Throwable e) {
logger.warn("Database export aborted due to an error.");
throw new ExecutionException("A fatal error has occurred during export.", e);
} finally {
databaseManager.disconnect();
if (!statistics.isEmpty()) {
logger.info("Export summary:");
statistics.logFeatureSummary(Level.INFO);
} else {
logger.info("No features exported.");
}
logStatistics(statistics, "Export summary:", Level.INFO);
}

return shouldRun;
Expand All @@ -206,6 +241,12 @@ protected Query getQuery(ExportOptions exportOptions) throws ExecutionException
}
}

protected Tiling getTiling(ExportOptions exportOptions) throws ExecutionException {
return tilingOptions != null ?
tilingOptions.getTiling() :
exportOptions.getTiling().orElseGet(TilingHelper::noTiling);
}

protected ExportOptions getExportOptions() throws ExecutionException {
ExportOptions exportOptions;
try {
Expand Down Expand Up @@ -270,6 +311,22 @@ protected WriteOptions getWriteOptions(ExportOptions exportOptions, DatabaseAdap
return writeOptions;
}

private String getTileCounter(TilingHelper helper, Tile tile) {
return helper.isUseTiling() ?
"[" + (tile.getRow() * helper.getTileMatrix().getColumns() + tile.getColumn() + 1) + "|" +
helper.getTileMatrix().size() + "] " :
"";
}

private void logStatistics(FeatureStatistics statistics, String title, Level level) {
if (!statistics.isEmpty()) {
logger.log(level, title);
statistics.logFeatureSummary(level);
} else {
logger.log(level, "No features exported.");
}
}

private void abort(Feature feature, long id, Throwable e) {
synchronized (lock) {
if (shouldRun) {
Expand Down
11 changes: 11 additions & 0 deletions citydb-cli/src/main/java/org/citydb/cli/exporter/ExportOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@

import org.citydb.config.SerializableConfig;
import org.citydb.query.Query;
import org.citydb.tiling.Tiling;

import java.util.Optional;

@SerializableConfig(name = "exportOptions")
public class ExportOptions extends org.citydb.operation.exporter.ExportOptions {
private Query query;
private Tiling tiling;

public Optional<Query> getQuery() {
return Optional.ofNullable(query);
Expand All @@ -38,4 +40,13 @@ public ExportOptions setQuery(Query query) {
this.query = query;
return this;
}

public Optional<Tiling> getTiling() {
return Optional.ofNullable(tiling);
}

public ExportOptions setTiling(Tiling tiling) {
this.tiling = tiling;
return this;
}
}
Loading

0 comments on commit 441cc04

Please sign in to comment.