Add tiling module and CLI options

citydb-cli/build.gradle

@@ -5,16 +5,17 @@ plugins {
 }
 
 dependencies {
+    api project(':citydb-database')
+    api project(':citydb-io')
+    api project(':citydb-operation')
     api project(':citydb-plugin')
+    api project(':citydb-query')
+    api project(':citydb-tiling')
     api 'info.picocli:picocli:4.7.6'
     implementation project(':citydb-config')
-    implementation project(':citydb-io')
+    implementation project(':citydb-database-postgres')
     implementation project(':citydb-io-citygml')
     implementation project(':citydb-logging')
-    implementation project(':citydb-database')
-    implementation project(':citydb-database-postgres')
-    implementation project(':citydb-operation')
-    implementation project(':citydb-query')
 }
 
 processResources {

citydb-cli/src/main/java/module-info.java

@@ -1,13 +1,14 @@
 module org.citydb.cli {
     requires org.citydb.config;
     requires org.citygml4j.core;
-    requires org.citydb.io;
     requires org.citydb.io.citygml;
     requires org.citydb.logging;
-    requires org.citydb.database;
-    requires org.citydb.operation;
-    requires org.citydb.query;
+    requires transitive org.citydb.database;
+    requires transitive org.citydb.io;
+    requires transitive org.citydb.operation;
     requires transitive org.citydb.plugin;
+    requires transitive org.citydb.query;
+    requires transitive org.citydb.tiling;
     requires transitive info.picocli;
 
     exports org.citydb.cli;

citydb-cli/src/main/java/org/citydb/cli/Launcher.java

@@ -61,6 +61,7 @@
         scope = CommandLine.ScopeType.INHERIT,
         description = "Command-line interface for the 3D City Database.",
         synopsisSubcommandLabel = "COMMAND",
+        abbreviateSynopsis = true,
         mixinStandardHelpOptions = true,
         versionProvider = Launcher.class,
         showAtFileInUsageHelp = true,

citydb-cli/src/main/java/org/citydb/cli/deleter/DeleteCommand.java

@@ -130,7 +130,7 @@ public Integer call() throws ExecutionException {
         }
 
         try {
-            logger.info("Querying features matching the request...");
+            logger.debug("Querying features matching the request...");
             logger.trace("Using SQL query:\n{}", () -> helper.getFormattedSql(executor.getSelect(),
                     databaseManager.getAdapter()));
 

citydb-cli/src/main/java/org/citydb/cli/exporter/ExportController.java

@@ -26,7 +26,9 @@
 import org.citydb.cli.ExecutionException;
 import org.citydb.cli.common.*;
 import org.citydb.cli.exporter.options.QueryOptions;
+import org.citydb.cli.exporter.options.TilingOptions;
 import org.citydb.cli.exporter.util.SequentialWriter;
+import org.citydb.cli.exporter.util.TilingHelper;
 import org.citydb.cli.util.CommandHelper;
 import org.citydb.cli.util.FeatureStatistics;
 import org.citydb.config.Config;
@@ -52,6 +54,9 @@
 import org.citydb.query.executor.QueryResult;
 import org.citydb.query.filter.encoding.FilterParseException;
 import org.citydb.query.util.QueryHelper;
+import org.citydb.tiling.Tile;
+import org.citydb.tiling.TileIterator;
+import org.citydb.tiling.Tiling;
 import picocli.CommandLine;
 
 import java.nio.file.Path;
@@ -77,7 +82,11 @@ public abstract class ExportController implements Command {
 
     @CommandLine.ArgGroup(exclusive = false, order = Integer.MAX_VALUE,
             heading = "Query and filter options:%n")
-    private QueryOptions queryOptions;
+    protected QueryOptions queryOptions;
+
+    @CommandLine.ArgGroup(exclusive = false, order = Integer.MAX_VALUE,
+            heading = "Tiling options:%n")
+    protected TilingOptions tilingOptions;
 
     @CommandLine.ArgGroup(exclusive = false, order = Integer.MAX_VALUE,
             heading = "Database connection options:%n")
@@ -119,71 +128,97 @@ protected boolean doExport() throws ExecutionException {
         WriteOptions writeOptions = getWriteOptions(exportOptions, databaseManager.getAdapter());
         writeOptions.getFormatOptions().set(getFormatOptions(writeOptions.getFormatOptions()));
 
-        Query query = getQuery(exportOptions);
-        QueryExecutor executor = helper.getQueryExecutor(query,
-                SqlBuildOptions.defaults().omitDistinct(true),
-                tempDirectory,
-                databaseManager.getAdapter());
-
-        FeatureStatistics statistics = new FeatureStatistics(databaseManager.getAdapter());
         helper.logIndexStatus(Level.INFO, databaseManager.getAdapter());
         initialize(exportOptions, writeOptions, databaseManager);
 
-        try (OutputFile outputFile = builder.newOutputFile(outputFileOptions.getFile());
-             FeatureWriter writer = createWriter(query, ioAdapter)) {
-            Exporter exporter = Exporter.newInstance();
-            exportOptions.setOutputFile(outputFile);
-
-            AtomicLong counter = new AtomicLong();
-
-            logger.info("Exporting to {} file {}.", ioManager.getFileFormat(ioAdapter), outputFile.getFile());
-            writer.initialize(outputFile, writeOptions);
-
-            logger.info("Querying features matching the request...");
-            logger.trace("Using SQL query:\n{}", () -> helper.getFormattedSql(executor.getSelect(),
-                    databaseManager.getAdapter()));
-
-            long sequenceId = 1;
-            try (QueryResult result = executor.executeQuery()) {
-                exporter.startSession(databaseManager.getAdapter(), exportOptions);
-                while (shouldRun && result.hasNext()) {
-                    long id = result.getId();
-                    exporter.exportFeature(id, sequenceId++).whenComplete((feature, t) -> {
-                        if (feature != null) {
-                            try {
-                                writer.write(feature, (success, e) -> {
-                                    if (success == Boolean.TRUE) {
-                                        statistics.add(feature);
-                                        long count = counter.incrementAndGet();
-                                        if (count % 1000 == 0) {
-                                            logger.info("{} features exported.", count);
-                                        }
-                                    } else {
+        Query query = getQuery(exportOptions);
+        Tiling tiling = getTiling(exportOptions);
+        FeatureStatistics statistics = new FeatureStatistics(databaseManager.getAdapter());
+        AtomicLong counter = new AtomicLong();
+
+        try {
+            TilingHelper tilingHelper = TilingHelper.of(tiling, query, databaseManager.getAdapter());
+            if (tilingHelper.isUseTiling()) {
+                logger.info("Creating {} tile(s) based on provided tiling scheme.",
+                        tilingHelper.getTileMatrix().size());
+            }
+
+            TileIterator iterator = tilingHelper.getTileMatrix().getTileIterator();
+            while (iterator.hasNext()) {
+                Tile tile = iterator.next();
+                QueryExecutor executor = helper.getQueryExecutor(tilingHelper.getTileQuery(tile),
+                        SqlBuildOptions.defaults()
+                                .omitDistinct(true)
+                                .withColumn(tilingHelper.isUseTiling() ? "envelope" : null),
+                        tempDirectory,
+                        databaseManager.getAdapter());
+
+                Path file = tilingHelper.getOutputFile(outputFileOptions.getFile(), tile);
+                FeatureStatistics tileStatistics = new FeatureStatistics(databaseManager.getAdapter());
+
+                try (OutputFile outputFile = builder.newOutputFile(file);
+                     FeatureWriter writer = createWriter(query, ioAdapter)) {
+                    Exporter exporter = Exporter.newInstance();
+                    exportOptions.setOutputFile(outputFile);
+
+                    logger.info("{}Exporting to {} file {}.", getTileCounter(tilingHelper, tile),
+                            ioManager.getFileFormat(ioAdapter), outputFile.getFile());
+                    writer.initialize(outputFile, writeOptions);
+
+                    logger.debug("Querying features matching the request...");
+                    logger.trace("Using SQL query:\n{}", () -> helper.getFormattedSql(executor.getSelect(),
+                            databaseManager.getAdapter()));
+
+                    long sequenceId = 1;
+                    try (QueryResult result = executor.executeQuery()) {
+                        exporter.startSession(databaseManager.getAdapter(), exportOptions);
+                        while (shouldRun && result.hasNext()) {
+                            long id = result.getId();
+
+                            if (tilingHelper.isUseTiling() && !tile.isOnTile(databaseManager.getAdapter()
+                                    .getGeometryAdapter()
+                                    .getEnvelope(result.get(rs -> rs.getObject("envelope"))))) {
+                                continue;
+                            }
+
+                            exporter.exportFeature(id, sequenceId++).whenComplete((feature, t) -> {
+                                if (feature != null) {
+                                    try {
+                                        writer.write(feature, (success, e) -> {
+                                            if (success == Boolean.TRUE) {
+                                                tileStatistics.add(feature);
+                                                long count = counter.incrementAndGet();
+                                                if (count % 1000 == 0) {
+                                                    logger.info("{} features exported.", count);
+                                                }
+                                            } else {
+                                                abort(feature, id, e);
+                                            }
+                                        });
+                                    } catch (Throwable e) {
                                         abort(feature, id, e);
                                     }
-                                });
-                            } catch (Throwable e) {
-                                abort(feature, id, e);
-                            }
-                        } else {
-                            abort(null, id, t);
+                                } else {
+                                    abort(null, id, t);
+                                }
+                            });
                         }
-                    });
+                    } finally {
+                        exporter.closeSession();
+                    }
+                } catch (Throwable e) {
+                    logger.warn("Database export aborted due to an error.");
+                    throw new ExecutionException("A fatal error has occurred during export.", e);
+                } finally {
+                    statistics.merge(tileStatistics);
+                    if (tilingHelper.isUseTiling()) {
+                        logStatistics(tileStatistics, "Tile export summary:", Level.DEBUG);
+                    }
                 }
-            } finally {
-                exporter.closeSession();
             }
-        } catch (Throwable e) {
-            logger.warn("Database export aborted due to an error.");
-            throw new ExecutionException("A fatal error has occurred during export.", e);
         } finally {
             databaseManager.disconnect();
-            if (!statistics.isEmpty()) {
-                logger.info("Export summary:");
-                statistics.logFeatureSummary(Level.INFO);
-            } else {
-                logger.info("No features exported.");
-            }
+            logStatistics(statistics, "Export summary:", Level.INFO);
         }
 
         return shouldRun;
@@ -206,6 +241,12 @@ protected Query getQuery(ExportOptions exportOptions) throws ExecutionException
         }
     }
 
+    protected Tiling getTiling(ExportOptions exportOptions) throws ExecutionException {
+        return tilingOptions != null ?
+                tilingOptions.getTiling() :
+                exportOptions.getTiling().orElseGet(TilingHelper::noTiling);
+    }
+
     protected ExportOptions getExportOptions() throws ExecutionException {
         ExportOptions exportOptions;
         try {
@@ -270,6 +311,22 @@ protected WriteOptions getWriteOptions(ExportOptions exportOptions, DatabaseAdap
         return writeOptions;
     }
 
+    private String getTileCounter(TilingHelper helper, Tile tile) {
+        return helper.isUseTiling() ?
+                "[" + (tile.getRow() * helper.getTileMatrix().getColumns() + tile.getColumn() + 1) + "|" +
+                        helper.getTileMatrix().size() + "] " :
+                "";
+    }
+
+    private void logStatistics(FeatureStatistics statistics, String title, Level level) {
+        if (!statistics.isEmpty()) {
+            logger.log(level, title);
+            statistics.logFeatureSummary(level);
+        } else {
+            logger.log(level, "No features exported.");
+        }
+    }
+
     private void abort(Feature feature, long id, Throwable e) {
         synchronized (lock) {
             if (shouldRun) {

citydb-cli/src/main/java/org/citydb/cli/exporter/ExportOptions.java

@@ -23,12 +23,14 @@
 
 import org.citydb.config.SerializableConfig;
 import org.citydb.query.Query;
+import org.citydb.tiling.Tiling;
 
 import java.util.Optional;
 
 @SerializableConfig(name = "exportOptions")
 public class ExportOptions extends org.citydb.operation.exporter.ExportOptions {
     private Query query;
+    private Tiling tiling;
 
     public Optional<Query> getQuery() {
         return Optional.ofNullable(query);
@@ -38,4 +40,13 @@ public ExportOptions setQuery(Query query) {
         this.query = query;
         return this;
     }
+
+    public Optional<Tiling> getTiling() {
+        return Optional.ofNullable(tiling);
+    }
+
+    public ExportOptions setTiling(Tiling tiling) {
+        this.tiling = tiling;
+        return this;
+    }
 }