fix(config): allow initializing config from URLs (#2830)

packages/cogify/src/cogify/cli/cli.cog.ts

@@ -1,7 +1,7 @@
 import { ProjectionLoader, TileId, TileMatrixSets } from '@basemaps/geo';
 import { LogType, fsa } from '@basemaps/shared';
 import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
-import { CogTiff } from '@cogeotiff/core';
+import { CogTiff, TiffTag } from '@cogeotiff/core';
 import { Metrics } from '@linzjs/metrics';
 import { command, flag, restPositionals } from 'cmd-ts';
 import { mkdir, rm } from 'fs/promises';
@@ -11,11 +11,14 @@ import { CutlineOptimizer } from '../../cutline.js';
 import { SourceDownloader, urlToString } from '../../download.js';
 import { HashTransform } from '../../hash.stream.js';
 import { getLogger, logArguments } from '../../log.js';
-import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp } from '../gdal.js';
+import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp } from '../gdal.command.js';
 import { GdalRunner } from '../gdal.runner.js';
 import { Url } from '../parsers.js';
 import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js';
 
+// FIXME: HACK @cogeotiff/core to include the Lerc tiff tag
+if (TiffTag[0xc5f2] == null) (TiffTag as any)[0xc5f2] = 'Lerc';
+
 function extractSourceFiles(item: CogifyStacItem, baseUrl: URL): URL[] {
   return item.links.filter((link) => link.rel === 'linz_basemaps:source').map((link) => new URL(link.href, baseUrl));
 }

packages/cogify/src/cogify/cli/cli.cover.ts

@@ -4,13 +4,14 @@ import { GoogleTms, Nztm2000QuadTms, TileId } from '@basemaps/geo';
 import { fsa } from '@basemaps/shared';
 import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js';
 import { Metrics } from '@linzjs/metrics';
-import { command, number, option, optional, restPositionals, string } from 'cmd-ts';
+import { command, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts';
 import { isArgo } from '../../argo.js';
 import { CutlineOptimizer } from '../../cutline.js';
 import { getLogger, logArguments } from '../../log.js';
 import { TileCoverContext, createTileCover } from '../../tile.cover.js';
 import { createFileStats } from '../stac.js';
 import { Url } from '../parsers.js';
+import { Presets } from '../../preset.js';
 
 const SupportedTileMatrix = [GoogleTms, Nztm2000QuadTms];
 
@@ -29,6 +30,13 @@ export const BasemapsCogifyCoverCommand = command({
       defaultValue: () => 20,
     }),
     paths: restPositionals({ type: Url, displayName: 'path', description: 'Path to source imagery' }),
+    preset: option({
+      type: oneOf(Object.keys(Presets)),
+      long: 'preset',
+      description: 'GDAL compression preset',
+      defaultValue: () => 'webp',
+      defaultValueIsSerializable: true,
+    }),
     tileMatrix: option({
       type: string,
       long: 'tile-matrix',
@@ -62,6 +70,7 @@ export const BasemapsCogifyCoverCommand = command({
       logger,
       metrics,
       cutline,
+      preset: args.preset,
     };
 
     const res = await createTileCover(ctx);

packages/cogify/src/cogify/gdal.ts → packages/cogify/src/cogify/gdal.command.ts

@@ -1,14 +1,7 @@
 import { Epsg, EpsgCode, TileMatrixSets } from '@basemaps/geo';
 import { GdalCommand } from './gdal.runner.js';
 import { CogifyCreationOptions } from './stac.js';
-
-export const CogifyDefaults = {
-  compression: 'webp',
-  blockSize: 512,
-  quality: 90,
-  warpResampling: 'bilinear',
-  overviewResampling: 'lanczos',
-} as const;
+import { Presets } from '../preset.js';
 
 export function gdalBuildVrt(id: string, source: string[]): GdalCommand {
   if (source.length === 0) throw new Error('No source files given for :' + id);
@@ -33,7 +26,7 @@ export function gdalBuildVrtWarp(
       ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp
       ['-s_srs', Epsg.get(sourceProjection).toEpsgString()], // Source EPSG
       ['-t_srs', tileMatrix.projection.toEpsgString()], // Target EPSG
-      ['-r', opt.warpResampling ?? CogifyDefaults.warpResampling],
+      opt.warpResampling ? ['-r', opt.warpResampling] : undefined,
       cutline.path ? ['-cutline', cutline.path, '-cblend', cutline.blend] : undefined,
       sourceVrt,
       id + '.' + tileMatrix.identifier + '.vrt',
@@ -45,7 +38,7 @@ export function gdalBuildVrtWarp(
 }
 
 export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationOptions): GdalCommand {
-  const cfg = { ...CogifyDefaults, ...opt };
+  const cfg = { ...Presets[opt.preset], ...opt };
   const tileMatrix = TileMatrixSets.find(cfg.tileMatrix);
   if (tileMatrix == null) throw new Error('Unable to find tileMatrix: ' + cfg.tileMatrix);
 
@@ -68,20 +61,22 @@ export function gdalBuildCog(id: string, sourceVrt: string, opt: CogifyCreationO
       ['-of', 'COG'],
       ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS
       ['--config', 'GDAL_NUM_THREADS', 'all_cpus'], // Also required to NUM_THREADS till gdal 3.7.x
-      ['-co', 'BIGTIFF=YES'], // Default to BIG_TIFF
+      ['-co', 'BIGTIFF=IF_NEEDED'], // BigTiff is somewhat slower and most (All?) of the COGS should be well below 4GB
       ['-co', 'ADD_ALPHA=YES'],
       ['-co', 'BLOCKSIZE=512'],
       ['-co', `WARP_RESAMPLING=${cfg.warpResampling}`],
       ['-co', `OVERVIEW_RESAMPLING=${cfg.overviewResampling}`],
       ['-co', `COMPRESS=${cfg.compression}`],
-      ['-co', `QUALITY=${cfg.quality}`],
+      cfg.quality ? ['-co', `QUALITY=${cfg.quality}`] : undefined,
+      cfg.maxZError ? ['-co', `MAX_Z_ERROR=${cfg.maxZError}`] : undefined,
       ['-co', 'SPARSE_OK=YES'],
       ['-co', `TARGET_SRS=${tileMatrix.projection.toEpsgString()}`],
       ['-co', `EXTENT=${tileExtent.join(',')},`],
       ['-tr', targetResolution, targetResolution],
       sourceVrt,
       targetTiff,
     ]
+      .filter((f) => f != null)
       .flat()
       .map(String),
   };

packages/cogify/src/cogify/gdal.runner.ts

@@ -2,6 +2,7 @@ import { sha256base58 } from '@basemaps/config';
 import { LogType } from '@basemaps/shared';
 import { spawn } from 'child_process';
 import { EventEmitter } from 'events';
+import { dirname } from 'path';
 
 export interface GdalCommand {
   /** Output file location */
@@ -12,6 +13,22 @@ export interface GdalCommand {
   args: string[];
 }
 
+function getDockerContainer(): string {
+  const containerPath = process.env['GDAL_DOCKER_CONTAINER'] ?? 'ghcr.io/osgeo/gdal';
+  const tag = process.env['GDAL_DOCKER_CONTAINER_TAG'] ?? 'ubuntu-small-3.7.0';
+  return `${containerPath}:${tag}`;
+}
+
+/** Convert a GDAL command to run using docker */
+function toDockerArgs(cmd: GdalCommand): string[] {
+  const dirName = dirname(cmd.output);
+
+  const args = ['run'];
+  if (cmd.output) args.push(...['-v', `${dirName}:${dirName}`]);
+  args.push(...[getDockerContainer(), cmd.command, ...cmd.args]);
+  return args;
+}
+
 export class GdalRunner {
   parser: GdalProgressParser = new GdalProgressParser();
   startTime: number;
@@ -52,7 +69,12 @@ export class GdalRunner {
     });
     this.startTime = performance.now();
 
-    const child = spawn(this.cmd.command, this.cmd.args);
+    const useDocker = !!process.env['GDAL_DOCKER'];
+    if (useDocker) {
+      logger?.info({ command: this.cmd.command, commandHash, container: getDockerContainer() }, 'Gdal:Docker');
+    }
+
+    const child = useDocker ? spawn('docker', toDockerArgs(this.cmd)) : spawn(this.cmd.command, this.cmd.args);
 
     const outputBuff: Buffer[] = [];
     const errBuff: Buffer[] = [];

packages/cogify/src/cogify/stac.ts

@@ -3,6 +3,9 @@ import { createHash } from 'node:crypto';
 import { StacCollection, StacItem, StacLink } from 'stac-ts';
 
 export interface CogifyCreationOptions {
+  /** Preset GDAL config to use */
+  preset: string;
+
   /** Tile to be created */
   tile: Tile;
 
@@ -16,7 +19,7 @@ export interface CogifyCreationOptions {
    *
    * @default 'webp'
    */
-  compression?: 'webp' | 'jpeg';
+  compression?: 'webp' | 'jpeg' | 'lerc';
 
   /**
    * Output tile size
@@ -35,6 +38,9 @@ export interface CogifyCreationOptions {
    */
   quality?: number;
 
+  /** Max Z Error only used when compression is `lerc` */
+  maxZError?: number;
+
   /**
    * Resampling for warping
    * @default 'bilinear'

packages/cogify/src/download.ts

@@ -54,7 +54,7 @@ export class SourceDownloader {
     if (asset.asset == null) return false;
     // No more items need this asset, clean it up
     const targetFile = await asset.asset;
-    logger.info({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup');
+    logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Cleanup');
     await fsa.delete(targetFile);
     return true;
   }
@@ -100,7 +100,7 @@ export class SourceDownloader {
       const targetFile = fsa.joinAll(this.cachePath, 'source', newFileName);
 
       await this._checkHost(asset.url);
-      logger.debug({ source: asset.url, target: targetFile }, 'Cog:Source:Download');
+      logger.trace({ source: asset.url, target: targetFile }, 'Cog:Source:Download');
       const hashStream = fsa.stream(urlToString(asset.url)).pipe(new HashTransform('sha256'));
       const startTime = performance.now();
       await fsa.write(targetFile, hashStream);

packages/cogify/src/preset.ts

@@ -0,0 +1,51 @@
+import { CogifyCreationOptions } from './cogify/stac';
+
+export const CogifyDefaults = {
+  compression: 'webp',
+  blockSize: 512,
+  quality: 90,
+  warpResampling: 'bilinear',
+  overviewResampling: 'lanczos',
+} as const;
+
+export interface Preset {
+  name: string;
+  options: Partial<CogifyCreationOptions>;
+}
+
+const webP: Preset = {
+  name: 'webp',
+  options: {
+    blockSize: CogifyDefaults.blockSize,
+    compression: CogifyDefaults.compression,
+    quality: CogifyDefaults.quality,
+    warpResampling: CogifyDefaults.warpResampling,
+    overviewResampling: CogifyDefaults.overviewResampling,
+  },
+};
+
+const lerc1cm: Preset = {
+  name: 'lerc_0.01',
+  options: {
+    blockSize: CogifyDefaults.blockSize,
+    compression: 'lerc',
+    maxZError: 0.01,
+    // TODO should a different resampling be used for LERC?
+    warpResampling: CogifyDefaults.warpResampling,
+    overviewResampling: CogifyDefaults.overviewResampling,
+  },
+};
+
+const lerc1mm: Preset = {
+  name: 'lerc_0.01',
+  options: {
+    blockSize: CogifyDefaults.blockSize,
+    compression: 'lerc',
+    maxZError: 0.001,
+    // TODO should a different resampling be used for LERC?
+    warpResampling: CogifyDefaults.warpResampling,
+    overviewResampling: CogifyDefaults.overviewResampling,
+  },
+};
+
+export const Presets = { [webP.name]: webP, [lerc1cm.name]: lerc1cm, [lerc1mm.name]: lerc1mm };

packages/cogify/src/tile.cover.ts

@@ -6,7 +6,6 @@ import { MultiPolygon, intersection, toFeatureCollection, union } from '@linzjs/
 import { Metrics } from '@linzjs/metrics';
 import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js';
 import { createCovering } from './cogify/covering.js';
-import { CogifyDefaults } from './cogify/gdal.js';
 import {
   CogifyLinkCutline,
   CogifyLinkSource,
@@ -15,6 +14,7 @@ import {
   createFileStats,
 } from './cogify/stac.js';
 import { CutlineOptimizer } from './cutline.js';
+import { Presets } from './preset.js';
 
 export interface TileCoverContext {
   /** Unique id for the covering */
@@ -29,6 +29,8 @@ export interface TileCoverContext {
   metrics?: Metrics;
   /** Optional logger to trace covering creation */
   logger?: LogType;
+  /** GDAL configuration preset */
+  preset: string;
 }
 export interface TileCoverResult {
   /** Stac collection for the imagery */
@@ -136,15 +138,12 @@ export async function createTileCover(ctx: TileCoverContext): Promise<TileCoverR
         end_datetime: dateTime.end ?? undefined,
         'proj:epsg': ctx.tileMatrix.projection.code,
         'linz_basemaps:options': {
+          preset: ctx.preset,
+          ...Presets[ctx.preset].options,
           tile,
           tileMatrix: ctx.tileMatrix.identifier,
           sourceEpsg: ctx.imagery.projection,
-          blockSize: CogifyDefaults.blockSize,
-          compression: CogifyDefaults.compression,
-          quality: CogifyDefaults.quality,
           zoomLevel: targetBaseZoom,
-          warpResampling: CogifyDefaults.warpResampling,
-          overviewResampling: CogifyDefaults.overviewResampling,
         },
         'linz_basemaps:generated': {
           package: CliInfo.package,