From 6c61b994c57ff021a581f3bb56e1da8e0b3b2629 Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Wed, 12 Jul 2023 10:22:39 +1200 Subject: [PATCH 1/2] fix(config): allow initializing config from URLs (#2830) --- packages/cogify/src/cogify/cli/cli.cog.ts | 7 ++- packages/cogify/src/cogify/cli/cli.cover.ts | 11 +++- .../src/cogify/{gdal.ts => gdal.command.ts} | 19 +++---- packages/cogify/src/cogify/gdal.runner.ts | 24 ++++++++- packages/cogify/src/cogify/stac.ts | 8 ++- packages/cogify/src/download.ts | 4 +- packages/cogify/src/preset.ts | 51 +++++++++++++++++++ packages/cogify/src/tile.cover.ts | 11 ++-- 8 files changed, 110 insertions(+), 25 deletions(-) rename packages/cogify/src/cogify/{gdal.ts => gdal.command.ts} (85%) create mode 100644 packages/cogify/src/preset.ts diff --git a/packages/cogify/src/cogify/cli/cli.cog.ts b/packages/cogify/src/cogify/cli/cli.cog.ts index 9345487a3..6ae421b8a 100644 --- a/packages/cogify/src/cogify/cli/cli.cog.ts +++ b/packages/cogify/src/cogify/cli/cli.cog.ts @@ -1,7 +1,7 @@ import { ProjectionLoader, TileId, TileMatrixSets } from '@basemaps/geo'; import { LogType, fsa } from '@basemaps/shared'; import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js'; -import { CogTiff } from '@cogeotiff/core'; +import { CogTiff, TiffTag } from '@cogeotiff/core'; import { Metrics } from '@linzjs/metrics'; import { command, flag, restPositionals } from 'cmd-ts'; import { mkdir, rm } from 'fs/promises'; @@ -11,11 +11,14 @@ import { CutlineOptimizer } from '../../cutline.js'; import { SourceDownloader, urlToString } from '../../download.js'; import { HashTransform } from '../../hash.stream.js'; import { getLogger, logArguments } from '../../log.js'; -import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp } from '../gdal.js'; +import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp } from '../gdal.command.js'; import { GdalRunner } from '../gdal.runner.js'; import { Url } from '../parsers.js'; import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js'; +// FIXME: HACK @cogeotiff/core to include the Lerc tiff tag +if (TiffTag[0xc5f2] == null) (TiffTag as any)[0xc5f2] = 'Lerc'; + function extractSourceFiles(item: CogifyStacItem, baseUrl: URL): URL[] { return item.links.filter((link) => link.rel === 'linz_basemaps:source').map((link) => new URL(link.href, baseUrl)); } diff --git a/packages/cogify/src/cogify/cli/cli.cover.ts b/packages/cogify/src/cogify/cli/cli.cover.ts index 4897554ba..7dd3704c4 100644 --- a/packages/cogify/src/cogify/cli/cli.cover.ts +++ b/packages/cogify/src/cogify/cli/cli.cover.ts @@ -4,13 +4,14 @@ import { GoogleTms, Nztm2000QuadTms, TileId } from '@basemaps/geo'; import { fsa } from '@basemaps/shared'; import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js'; import { Metrics } from '@linzjs/metrics'; -import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; +import { command, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts'; import { isArgo } from '../../argo.js'; import { CutlineOptimizer } from '../../cutline.js'; import { getLogger, logArguments } from '../../log.js'; import { TileCoverContext, createTileCover } from '../../tile.cover.js'; import { createFileStats } from '../stac.js'; import { Url } from '../parsers.js'; +import { Presets } from '../../preset.js'; const SupportedTileMatrix = [GoogleTms, Nztm2000QuadTms]; @@ -29,6 +30,13 @@ export const BasemapsCogifyCoverCommand = command({ defaultValue: () => 20, }), paths: restPositionals({ type: Url, displayName: 'path', description: 'Path to source imagery' }), + preset: option({ + type: oneOf(Object.keys(Presets)), + long: 'preset', + description: 'GDAL compression preset', + defaultValue: () => 'webp', + defaultValueIsSerializable: true, + }), tileMatrix: option({ type: string, long: 'tile-matrix', @@ -62,6 +70,7 @@ export const BasemapsCogifyCoverCommand = command({ logger, metrics, cutline, + preset: args.preset, }; const res = await createTileCover(ctx); diff --git a/packages/cogify/src/cogify/gdal.ts b/packages/cogify/src/cogify/gdal.command.ts similarity index 85% rename from packages/cogify/src/cogify/gdal.ts rename to packages/cogify/src/cogify/gdal.command.ts index 40a6e6ff4..8aa30e2bd 100644 --- a/packages/cogify/src/cogify/gdal.ts +++ b/packages/cogify/src/cogify/gdal.command.ts @@ -1,14 +1,7 @@ import { Epsg, EpsgCode, TileMatrixSets } from '@basemaps/geo'; import { GdalCommand } from './gdal.runner.js'; import { CogifyCreationOptions } from './stac.js'; - -export const CogifyDefaults = { - compression: 'webp', - blockSize: 512, - quality: 90, - warpResampling: 'bilinear', - overviewResampling: 'lanczos', -} as const; +import { Presets } from '../preset.js'; export function gdalBuildVrt(id: string, source: string[]): GdalCommand { if (source.length === 0) throw new Error('No source files given for :' + id); @@ -33,7 +26,7 @@ export function gdalBuildVrtWarp( ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp ['-s_srs', Epsg.get(sourceProjection).toEpsgString()], // Source EPSG ['-t_srs', tileMatrix.projection.toEpsgString()], // Target EPSG - ['-r', opt.warpResampling ?? CogifyDefaults.warpResampling], + opt.warpResampling ? ['-r', opt.warpResampling] : undefined, cutline.path ? ['-cutline', cutline.path, '-cblend', cutline.blend] : undefined, sourceVrt, id + '.' + tileMatrix.identifier + '.vrt', @@ -45,7 +38,7 @@ export function gdalBuildVrtWarp( } export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationOptions): GdalCommand { - const cfg = { ...CogifyDefaults, ...opt }; + const cfg = { ...Presets[opt.preset], ...opt }; const tileMatrix = TileMatrixSets.find(cfg.tileMatrix); if (tileMatrix == null) throw new Error('Unable to find tileMatrix: ' + cfg.tileMatrix); @@ -68,13 +61,14 @@ export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationO ['-of', 'COG'], ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS ['--config', 'GDAL_NUM_THREADS', 'all_cpus'], // Also required to NUM_THREADS till gdal 3.7.x - ['-co', 'BIGTIFF=YES'], // Default to BIG_TIFF + ['-co', 'BIGTIFF=IF_NEEDED'], // BigTiff is somewhat slower and most (All?) of the COGS should be well below 4GB ['-co', 'ADD_ALPHA=YES'], ['-co', 'BLOCKSIZE=512'], ['-co', `WARP_RESAMPLING=${cfg.warpResampling}`], ['-co', `OVERVIEW_RESAMPLING=${cfg.overviewResampling}`], ['-co', `COMPRESS=${cfg.compression}`], - ['-co', `QUALITY=${cfg.quality}`], + cfg.quality ? ['-co', `QUALITY=${cfg.quality}`] : undefined, + cfg.maxZError ? ['-co', `MAX_Z_ERROR=${cfg.maxZError}`] : undefined, ['-co', 'SPARSE_OK=YES'], ['-co', `TARGET_SRS=${tileMatrix.projection.toEpsgString()}`], ['-co', `EXTENT=${tileExtent.join(',')},`], @@ -82,6 +76,7 @@ export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationO sourceVrt, targetTiff, ] + .filter((f) => f != null) .flat() .map(String), }; diff --git a/packages/cogify/src/cogify/gdal.runner.ts b/packages/cogify/src/cogify/gdal.runner.ts index 3501907dd..7b235cd93 100644 --- a/packages/cogify/src/cogify/gdal.runner.ts +++ b/packages/cogify/src/cogify/gdal.runner.ts @@ -2,6 +2,7 @@ import { sha256base58 } from '@basemaps/config'; import { LogType } from '@basemaps/shared'; import { spawn } from 'child_process'; import { EventEmitter } from 'events'; +import { dirname } from 'path'; export interface GdalCommand { /** Output file location */ @@ -12,6 +13,22 @@ export interface GdalCommand { args: string[]; } +function getDockerContainer(): string { + const containerPath = process.env['GDAL_DOCKER_CONTAINER'] ?? 'ghcr.io/osgeo/gdal'; + const tag = process.env['GDAL_DOCKER_CONTAINER_TAG'] ?? 'ubuntu-small-3.7.0'; + return `${containerPath}:${tag}`; +} + +/** Convert a GDAL command to run using docker */ +function toDockerArgs(cmd: GdalCommand): string[] { + const dirName = dirname(cmd.output); + + const args = ['run']; + if (cmd.output) args.push(...['-v', `${dirName}:${dirName}`]); + args.push(...[getDockerContainer(), cmd.command, ...cmd.args]); + return args; +} + export class GdalRunner { parser: GdalProgressParser = new GdalProgressParser(); startTime: number; @@ -52,7 +69,12 @@ export class GdalRunner { }); this.startTime = performance.now(); - const child = spawn(this.cmd.command, this.cmd.args); + const useDocker = !!process.env['GDAL_DOCKER']; + if (useDocker) { + logger?.info({ command: this.cmd.command, commandHash, container: getDockerContainer() }, 'Gdal:Docker'); + } + + const child = useDocker ? spawn('docker', toDockerArgs(this.cmd)) : spawn(this.cmd.command, this.cmd.args); const outputBuff: Buffer[] = []; const errBuff: Buffer[] = []; diff --git a/packages/cogify/src/cogify/stac.ts b/packages/cogify/src/cogify/stac.ts index e03ec300c..4be6bf258 100644 --- a/packages/cogify/src/cogify/stac.ts +++ b/packages/cogify/src/cogify/stac.ts @@ -3,6 +3,9 @@ import { createHash } from 'node:crypto'; import { StacCollection, StacItem, StacLink } from 'stac-ts'; export interface CogifyCreationOptions { + /** Preset GDAL config to use */ + preset: string; + /** Tile to be created */ tile: Tile; @@ -16,7 +19,7 @@ export interface CogifyCreationOptions { * * @default 'webp' */ - compression?: 'webp' | 'jpeg'; + compression?: 'webp' | 'jpeg' | 'lerc'; /** * Output tile size @@ -35,6 +38,9 @@ export interface CogifyCreationOptions { */ quality?: number; + /** Max Z Error only used when compression is `lerc` */ + maxZError?: number; + /** * Resampling for warping * @default 'bilinear' diff --git a/packages/cogify/src/download.ts b/packages/cogify/src/download.ts index d8cdc899f..420d24461 100644 --- a/packages/cogify/src/download.ts +++ b/packages/cogify/src/download.ts @@ -54,7 +54,7 @@ export class SourceDownloader { if (asset.asset == null) return false; // No more items need this asset, clean it up const targetFile = await asset.asset; - logger.info({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup'); + logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup'); await fsa.delete(targetFile); return true; } @@ -100,7 +100,7 @@ export class SourceDownloader { const targetFile = fsa.joinAll(this.cachePath, 'source', newFileName); await this._checkHost(asset.url); - logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Download'); + logger.trace({ source: asset.url, target: targetFile }, 'Cog:Source:Download'); const hashStream = fsa.stream(urlToString(asset.url)).pipe(new HashTransform('sha256')); const startTime = performance.now(); await fsa.write(targetFile, hashStream); diff --git a/packages/cogify/src/preset.ts b/packages/cogify/src/preset.ts new file mode 100644 index 000000000..385020cd8 --- /dev/null +++ b/packages/cogify/src/preset.ts @@ -0,0 +1,51 @@ +import { CogifyCreationOptions } from './cogify/stac'; + +export const CogifyDefaults = { + compression: 'webp', + blockSize: 512, + quality: 90, + warpResampling: 'bilinear', + overviewResampling: 'lanczos', +} as const; + +export interface Preset { + name: string; + options: Partial; +} + +const webP: Preset = { + name: 'webp', + options: { + blockSize: CogifyDefaults.blockSize, + compression: CogifyDefaults.compression, + quality: CogifyDefaults.quality, + warpResampling: CogifyDefaults.warpResampling, + overviewResampling: CogifyDefaults.overviewResampling, + }, +}; + +const lerc1cm: Preset = { + name: 'lerc_0.01', + options: { + blockSize: CogifyDefaults.blockSize, + compression: 'lerc', + maxZError: 0.01, + // TODO should a different resampling be used for LERC? + warpResampling: CogifyDefaults.warpResampling, + overviewResampling: CogifyDefaults.overviewResampling, + }, +}; + +const lerc1mm: Preset = { + name: 'lerc_0.01', + options: { + blockSize: CogifyDefaults.blockSize, + compression: 'lerc', + maxZError: 0.001, + // TODO should a different resampling be used for LERC? + warpResampling: CogifyDefaults.warpResampling, + overviewResampling: CogifyDefaults.overviewResampling, + }, +}; + +export const Presets = { [webP.name]: webP, [lerc1cm.name]: lerc1cm, [lerc1mm.name]: lerc1mm }; diff --git a/packages/cogify/src/tile.cover.ts b/packages/cogify/src/tile.cover.ts index 21015330d..fee0cd34a 100644 --- a/packages/cogify/src/tile.cover.ts +++ b/packages/cogify/src/tile.cover.ts @@ -6,7 +6,6 @@ import { MultiPolygon, intersection, toFeatureCollection, union } from '@linzjs/ import { Metrics } from '@linzjs/metrics'; import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; import { createCovering } from './cogify/covering.js'; -import { CogifyDefaults } from './cogify/gdal.js'; import { CogifyLinkCutline, CogifyLinkSource, @@ -15,6 +14,7 @@ import { createFileStats, } from './cogify/stac.js'; import { CutlineOptimizer } from './cutline.js'; +import { Presets } from './preset.js'; export interface TileCoverContext { /** Unique id for the covering */ @@ -29,6 +29,8 @@ export interface TileCoverContext { metrics?: Metrics; /** Optional logger to trace covering creation */ logger?: LogType; + /** GDAL configuration preset */ + preset: string; } export interface TileCoverResult { /** Stac collection for the imagery */ @@ -136,15 +138,12 @@ export async function createTileCover(ctx: TileCoverContext): Promise Date: Wed, 12 Jul 2023 15:45:53 +1200 Subject: [PATCH 2/2] refactor: use mm for lerc resolution --- packages/cogify/src/preset.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cogify/src/preset.ts b/packages/cogify/src/preset.ts index 385020cd8..f65353d69 100644 --- a/packages/cogify/src/preset.ts +++ b/packages/cogify/src/preset.ts @@ -24,8 +24,8 @@ const webP: Preset = { }, }; -const lerc1cm: Preset = { - name: 'lerc_0.01', +const lerc10mm: Preset = { + name: 'lerc_10mm', options: { blockSize: CogifyDefaults.blockSize, compression: 'lerc', @@ -37,7 +37,7 @@ const lerc1cm: Preset = { }; const lerc1mm: Preset = { - name: 'lerc_0.01', + name: 'lerc_1mm', options: { blockSize: CogifyDefaults.blockSize, compression: 'lerc', @@ -48,4 +48,4 @@ const lerc1mm: Preset = { }, }; -export const Presets = { [webP.name]: webP, [lerc1cm.name]: lerc1cm, [lerc1mm.name]: lerc1mm }; +export const Presets = { [webP.name]: webP, [lerc10mm.name]: lerc10mm, [lerc1mm.name]: lerc1mm };