From 401e31a902ba3a010d40e6bbac935d02bc010570 Mon Sep 17 00:00:00 2001 From: Marco Hutter Date: Fri, 12 Jul 2024 14:17:09 +0200 Subject: [PATCH 1/4] Added note about caveats of path.relative --- src/base/base/Paths.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/base/base/Paths.ts b/src/base/base/Paths.ts index f43b2e1e..11e587d8 100644 --- a/src/base/base/Paths.ts +++ b/src/base/base/Paths.ts @@ -102,6 +102,17 @@ export class Paths { * the result uses `/` forward slashes as the directory * separator. * + * Note: + * - The first argument is assumed to be a directory + * - The second argument is assumed to be a file name + * - The result is the relativized file name + * + * For example: In a call like + * `relativize("./example/directoryA", "./example/directoryB/file.txt")` + * the second argument has to be the file name, and the result + * will be "../directoryB/file.txt", which is the path of the + * file _relative to_ the directory that is given as the first argument. + * * @param directory - The directory * @param fileName - The file name * @returns The resulting path From c1af3e1777ca5de548c03f57eee32b75517cd1ef Mon Sep 17 00:00:00 2001 From: Marco Hutter Date: Fri, 12 Jul 2024 14:17:46 +0200 Subject: [PATCH 2/4] Add function to determine tileset directory name... Working around some ambiguities --- src/tilesets/tilesets/Tilesets.ts | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/tilesets/tilesets/Tilesets.ts b/src/tilesets/tilesets/Tilesets.ts index ce3659b9..d2071d1e 100644 --- a/src/tilesets/tilesets/Tilesets.ts +++ b/src/tilesets/tilesets/Tilesets.ts @@ -32,6 +32,40 @@ export class Tilesets { return "tileset.json"; } + /** + * Determine a directory name from the given tileset name. + * + * When the given name ends with `.json`, `.3tz`, or `.3dtiles` + * (case-insensitively), then the directory name of that file + * name is returned. Otherwise, the given name is assumed to + * be a directory name and returned directly. + * + * NOTE: This is working around the ambiguity that is related to + * https://github.com/CesiumGS/3d-tiles/issues/184 : When someone + * uses a path like "./data/target" as a target name, then it is + * not clear whether the result should be + * + * - a JSON file "./data/target" (without extension) + * - a file called "./data/target/tileset.json" + * + * The latter is the default assumption that is used for source + * data: When the source is a directory, then it assumes + * that there is a 'tileset.json' file in this directory. + * + * For the target, checking whether there is a file extension + * seems to be a reasonable workaround. + * + * @param tilesetName - The tileset name + * @returns The directory name + */ + static determineTilesetDirectoryName(tilesetName: string) { + const n = tilesetName.toLowerCase(); + if (n.endsWith(".json") || n.endsWith(".3tz") || n.endsWith(".3dtiles")) { + return path.dirname(tilesetName); + } + return tilesetName; + } + /** * Returns whether the given names likely refer to the same package. * From e839c5d530a837af0939ca60a0fcdc7c654b388d Mon Sep 17 00:00:00 2001 From: Marco Hutter Date: Fri, 12 Jul 2024 14:18:53 +0200 Subject: [PATCH 3/4] Make clear that "identifiers" are actually paths --- src/tools/tilesetProcessing/TilesetMerger.ts | 144 ++++++++++++------- 1 file changed, 89 insertions(+), 55 deletions(-) diff --git a/src/tools/tilesetProcessing/TilesetMerger.ts b/src/tools/tilesetProcessing/TilesetMerger.ts index 1cf7d9ef..79c1fd3e 100644 --- a/src/tools/tilesetProcessing/TilesetMerger.ts +++ b/src/tools/tilesetProcessing/TilesetMerger.ts @@ -37,12 +37,24 @@ export class TilesetMerger { private tilesetSourceJsonFileNames: string[]; /** - * Identifiers for the external tilesets. These will usually - * be just the last component of the directory name. For example, - * for a tileset like "./data/example/tileset.json", this will - * be "example". But the names are disambiguated, just in case... + * The directories that will contain the external tilesets. + * + * For the default `merge` operation, these will be the last component + * of the source directory name. For example, for a tileset like + * "./data/example/tileset.json", this will be "example". + * The merger will create subdirectories in the target directory, + * and copy the source tilesets into these directories. The merged + * tileset will then refer to these (copied) tilesets. Duplicate + * names will be disambiguated. + * + * For the `mergeJson` operation, these will be relative paths, + * pointing from the target directory to the original source + * directories. + * + * In both cases, these will be the directories that appear in + * the `content.uri` of the merged tileset. */ - private tilesetSourceIdentifiers: string[]; + private externalTilesetDirectories: string[]; /** * The target that the resulting tileset will be written to. @@ -61,7 +73,7 @@ export class TilesetMerger { constructor() { this.tilesetSources = []; this.tilesetSourceJsonFileNames = []; - this.tilesetSourceIdentifiers = []; + this.externalTilesetDirectories = []; } /** @@ -119,40 +131,61 @@ export class TilesetMerger { /** * Internal method to differentiate between `merge` and `mergeJson` + * + * @param tilesetSourceNames - The tileset source names + * @param tilesetTargetName - The tileset target name + * @param overwrite - Whether target files should be overwritten + * @returns A promise that resolves when the process is finished + * @throws TilesetError When the input could not be processed */ - async mergeOperation( + private async mergeOperation( tilesetSourceNames: string[], tilesetTargetName: string, overwrite: boolean, jsonOnly: boolean ): Promise { - // Create the sources and target + const tilesetTargetDirectory = + Tilesets.determineTilesetDirectoryName(tilesetTargetName); + for (const tilesetSourceName of tilesetSourceNames) { - // Determine the name of the file that contains the tileset JSON data + // Determine the name of the file that contains the tileset JSON data, + // and the source directory const tilesetSourceJsonFileName = Tilesets.determineTilesetJsonFileName(tilesetSourceName); - - // Determine an "identifier" for the tileset source - // (see `tilesetSourceIdentifiers` for details) - let tilesetSourceDirectoryName; - if (Paths.isDirectory(tilesetSourceName)) { - tilesetSourceDirectoryName = path.basename(tilesetSourceName); - } else { - tilesetSourceDirectoryName = path.basename( - path.dirname(tilesetSourceName) - ); - } - const tilesetSourceIdentifier = TilesetMerger.createIdentifier( - tilesetSourceDirectoryName, - this.tilesetSourceIdentifiers - ); + const externalTilesetDirectory = + Tilesets.determineTilesetDirectoryName(tilesetSourceName); const tilesetSource = TilesetSources.createAndOpen(tilesetSourceName); this.tilesetSources.push(tilesetSource); this.tilesetSourceJsonFileNames.push(tilesetSourceJsonFileName); - this.tilesetSourceIdentifiers.push( - !jsonOnly ? tilesetSourceIdentifier : tilesetSourceName - ); + + // Determine the directory name that will be used for referring + // to the source tileset (via the `content.uri` in the merged + // tileset), and store it in the 'externalTilesetDirectories'. + if (jsonOnly) { + // When only a merged JSON should be created, then the + // external tileset directory will be a relative path, + // pointing from the original source to the target + const relativeExternalTilesetDirectory = Paths.relativize( + tilesetTargetDirectory, + externalTilesetDirectory + ); + + this.externalTilesetDirectories.push(relativeExternalTilesetDirectory); + } else { + // When a full merge operation is performed, then the + // source tilesets are copied into the target directory, + // using a name that just consists of the base name (last + // path component) of the source directory, disambiguated + const externalTilesetDirectoryBaseName = path.basename( + externalTilesetDirectory + ); + const uniqueExternalTilesetDirectory = TilesetMerger.createUnique( + externalTilesetDirectoryBaseName, + this.externalTilesetDirectories + ); + this.externalTilesetDirectories.push(uniqueExternalTilesetDirectory); + } } this.tilesetTargetJsonFileName = @@ -162,8 +195,14 @@ export class TilesetMerger { overwrite ); - // Perform the actual merge - this.mergeInternal(jsonOnly); + // Perform the actual merge, creating the merged tileset JSON + // in the target + this.mergeInternal(); + + // Copy the resources from the sources to the target + if (!jsonOnly) { + this.copyResources(); + } // Clean up by closing the sources and the target for (const tilesetSource of this.tilesetSources) { @@ -172,15 +211,16 @@ export class TilesetMerger { await this.tilesetTarget.end(); this.tilesetSources.length = 0; - this.tilesetSourceIdentifiers.length = 0; + this.externalTilesetDirectories.length = 0; this.tilesetTarget = undefined; this.tilesetTargetJsonFileName = undefined; } /** - * Internal method for `merge` + * Internal method for `merge`, only creating the actual merged + * tileset JSON and putting it into the target. */ - private mergeInternal(jsonOnly: boolean) { + private mergeInternal() { if ( this.tilesetSources.length == 0 || !this.tilesetTarget || @@ -211,7 +251,7 @@ export class TilesetMerger { const box = TilesetMerger.getMergedBox(tilesets); const children = TilesetMerger.getChildren( tilesets, - this.tilesetSourceIdentifiers, + this.externalTilesetDirectories, this.tilesetSourceJsonFileNames ); const mergedTileset = { @@ -236,17 +276,14 @@ export class TilesetMerger { this.tilesetTargetJsonFileName, mergedTilesetBuffer ); - - // Copy the resources from the sources to the target - if (!jsonOnly) this.copyResources(); } /** * Copy the resources from the source tilesets into the target. * * This will obtain the entries of all sources, and add them - * to the target, adding the `tilesetSourceIdentifier` to the - * path for disambiguation. + * to the target, adding the `externalTilesetDirectory` to the + * path. */ private copyResources(): void { if (this.tilesetSources.length == 0 || !this.tilesetTarget) { @@ -256,11 +293,11 @@ export class TilesetMerger { const length = this.tilesetSources.length; for (let i = 0; i < length; ++i) { const tilesetSource = this.tilesetSources[i]; - const tilesetSourceIdentifier = this.tilesetSourceIdentifiers[i]; + const externalTilesetDirectory = this.externalTilesetDirectories[i]; const sourceKeys = tilesetSource.getKeys(); for (const sourceKey of sourceKeys) { const value = tilesetSource.getValue(sourceKey); - const targetKey = tilesetSourceIdentifier + "/" + sourceKey; + const targetKey = Paths.join(externalTilesetDirectory, sourceKey); if (value) { this.tilesetTarget.addEntry(targetKey, value); } @@ -269,29 +306,26 @@ export class TilesetMerger { } /** - * Creates an identifier that does not exist yet. + * Creates a string that does not exist yet. * * If the given prefix is not yet contained in the given list, * then it is returned. Otherwise, it is made "unique" in an * unspecified way, and then returned. * - * This does NOT add the new identifier to the given list! + * This does NOT add the new string to the given list! * * @param prefix - The prefix - * @param existingIdentifiers - The existing identifiers - * @returns The new identifier + * @param existing - The existing strings + * @returns The unique string */ - private static createIdentifier( - prefix: string, - existingIdentifiers: string[] - ): string { - let identifier = prefix; + private static createUnique(prefix: string, existing: string[]): string { + let result = prefix; let counter = 0; for (;;) { - if (!existingIdentifiers.includes(identifier)) { - return identifier; + if (!existing.includes(result)) { + return result; } - identifier = `${prefix}-${counter}`; + result = `${prefix}-${counter}`; counter++; } } @@ -304,16 +338,16 @@ export class TilesetMerger { private static getChildren( tilesets: Tileset[], - tilesetSourceIdentifiers: string[], + externalTilesetDirectories: string[], tilesetJsonFileNames: string[] ) { const length = tilesets.length; const children = Array(length); for (let i = 0; i < length; ++i) { const tilesetJsonFileName = tilesetJsonFileNames[i]; - const tilesetSourceIdentifier = tilesetSourceIdentifiers[i]; + const externalTilesetDirectory = externalTilesetDirectories[i]; const tilesetUrl = Paths.join( - tilesetSourceIdentifier, + externalTilesetDirectory, tilesetJsonFileName ); From 4080cff31640a11d82a78b09b6f06ce8486a1261 Mon Sep 17 00:00:00 2001 From: Marco Hutter Date: Fri, 12 Jul 2024 14:19:03 +0200 Subject: [PATCH 4/4] Update specs for TilesetMerger --- .../tilesetProcessing/TilesetMergerSpec.ts | 165 +++++++++++++++--- 1 file changed, 145 insertions(+), 20 deletions(-) diff --git a/specs/tools/tilesetProcessing/TilesetMergerSpec.ts b/specs/tools/tilesetProcessing/TilesetMergerSpec.ts index bd7467e8..fb1f459b 100644 --- a/specs/tools/tilesetProcessing/TilesetMergerSpec.ts +++ b/specs/tools/tilesetProcessing/TilesetMergerSpec.ts @@ -7,31 +7,39 @@ import { TilesetOperations } from "../../../src/tools"; import { SpecHelpers } from "../../SpecHelpers"; const SPECS_DATA_BASE_DIRECTORY = SpecHelpers.getSpecsDataBaseDirectory(); - -const basicInputs = [ - SPECS_DATA_BASE_DIRECTORY + "/mergeTilesets/basicMerge/TilesetA", - SPECS_DATA_BASE_DIRECTORY + "/mergeTilesets/basicMerge/sub/TilesetA", -]; -const basicOutput = - SPECS_DATA_BASE_DIRECTORY + "/output/mergeTilesets/basicMerge"; const overwrite = true; describe("TilesetMerger", function () { afterEach(function () { SpecHelpers.forceDeleteDirectory( - SPECS_DATA_BASE_DIRECTORY + "/output/mergeTilesets" + Paths.join(SPECS_DATA_BASE_DIRECTORY, "output/mergeTilesets") ); }); + it("merges tilesets from directories into a single tileset directory", async function () { + const inputDirectories = [ + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/TilesetA" + ), + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/sub/TilesetA" + ), + ]; + const outputDirectory = Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "output/mergeTilesets/basicMerge" + ); + const outputFile = Paths.join(outputDirectory, "tileset.json"); - it("merges tilesets into a single tileset", async function () { - await TilesetOperations.merge(basicInputs, basicOutput, overwrite); + await TilesetOperations.merge(inputDirectories, outputDirectory, overwrite); // Ensure that the output directory contains the expected files: // All files of the input, disambiguated for the same base name // (i.e. "TilesetA" and "TilesetA-0" - this is not specified, // but has to be assumed here) const actualRelativeFiles = - SpecHelpers.collectRelativeFileNames(basicOutput); + SpecHelpers.collectRelativeFileNames(outputDirectory); actualRelativeFiles.sort(); const expectedRelativeFiles = [ "TilesetA-0/ll.b3dm", @@ -52,9 +60,66 @@ describe("TilesetMerger", function () { // Ensure that the single 'tileset.json' contains the // proper content URIs for the external tilesets: - const tilesetJsonBuffer = fs.readFileSync( - Paths.join(basicOutput, "tileset.json") + const tilesetJsonBuffer = fs.readFileSync(outputFile); + const tileset = JSON.parse(tilesetJsonBuffer.toString()); + const actualContentUris = await SpecHelpers.collectExplicitContentUris( + tileset.root + ); + actualContentUris.sort(); + + const expectedContentUris = [ + "TilesetA-0/tileset.json", + "TilesetA/tileset.json", + ]; + expect(actualContentUris).toEqual(expectedContentUris); + }); + + it("merges tilesets from files into a single tileset file", async function () { + const inputFiles = [ + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/TilesetA/tileset.json" + ), + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/sub/TilesetA/tileset.json" + ), + ]; + const outputDirectory = Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "output/mergeTilesets/basicMerge" ); + const outputFile = Paths.join(outputDirectory, "tileset.json"); + + await TilesetOperations.merge(inputFiles, outputFile, overwrite); + + // Ensure that the output directory contains the expected files: + // All files of the input, disambiguated for the same base name + // (i.e. "TilesetA" and "TilesetA-0" - this is not specified, + // but has to be assumed here) + const actualRelativeFiles = + SpecHelpers.collectRelativeFileNames(outputDirectory); + actualRelativeFiles.sort(); + const expectedRelativeFiles = [ + "TilesetA-0/ll.b3dm", + "TilesetA-0/lr.b3dm", + "TilesetA-0/parent.b3dm", + "TilesetA-0/tileset.json", + "TilesetA-0/ul.b3dm", + "TilesetA-0/ur.b3dm", + "TilesetA/ll.b3dm", + "TilesetA/lr.b3dm", + "TilesetA/parent.b3dm", + "TilesetA/tileset.json", + "TilesetA/ul.b3dm", + "TilesetA/ur.b3dm", + "tileset.json", + ]; + expect(actualRelativeFiles).toEqual(expectedRelativeFiles); + + // Ensure that the single 'tileset.json' contains the + // proper content URIs for the external tilesets: + const tilesetJsonBuffer = fs.readFileSync(outputFile); const tileset = JSON.parse(tilesetJsonBuffer.toString()); const actualContentUris = await SpecHelpers.collectExplicitContentUris( tileset.root @@ -68,21 +133,81 @@ describe("TilesetMerger", function () { expect(actualContentUris).toEqual(expectedContentUris); }); - it("merges tilesets into a single tileset for mergeJson", async function () { - await TilesetOperations.mergeJson(basicInputs, basicOutput, overwrite); + it("merges tilesets from directories into a single tileset in a directory for mergeJson", async function () { + const inputDirectories = [ + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/TilesetA" + ), + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/sub/TilesetA" + ), + ]; + const outputDirectory = Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "output/mergeTilesets/basicMerge" + ); + const outputFile = Paths.join(outputDirectory, "tileset.json"); + + await TilesetOperations.mergeJson( + inputDirectories, + outputDirectory, + overwrite + ); // Ensure that the output directory contains the expected files: const actualRelativeFiles = - SpecHelpers.collectRelativeFileNames(basicOutput); + SpecHelpers.collectRelativeFileNames(outputDirectory); actualRelativeFiles.sort(); const expectedRelativeFiles = ["tileset.json"]; expect(actualRelativeFiles).toEqual(expectedRelativeFiles); // Ensure that the single 'tileset.json' contains the // proper content URIs for the external tilesets: - const tilesetJsonBuffer = fs.readFileSync( - Paths.join(basicOutput, "tileset.json") + const tilesetJsonBuffer = fs.readFileSync(outputFile); + const tileset = JSON.parse(tilesetJsonBuffer.toString()); + const actualContentUris = await SpecHelpers.collectExplicitContentUris( + tileset.root ); + actualContentUris.sort(); + + const expectedContentUris = [ + "../../../mergeTilesets/basicMerge/TilesetA/tileset.json", + "../../../mergeTilesets/basicMerge/sub/TilesetA/tileset.json", + ]; + expect(actualContentUris).toEqual(expectedContentUris); + }); + + it("merges tilesets from files into a single tileset file for mergeJson", async function () { + const inputFiles = [ + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/TilesetA/tileset.json" + ), + Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "mergeTilesets/basicMerge/sub/TilesetA/tileset.json" + ), + ]; + const outputDirectory = Paths.join( + SPECS_DATA_BASE_DIRECTORY, + "output/mergeTilesets/basicMerge" + ); + const outputFile = Paths.join(outputDirectory, "tileset.json"); + + await TilesetOperations.mergeJson(inputFiles, outputFile, overwrite); + + // Ensure that the output directory contains the expected files: + const actualRelativeFiles = + SpecHelpers.collectRelativeFileNames(outputDirectory); + actualRelativeFiles.sort(); + const expectedRelativeFiles = ["tileset.json"]; + expect(actualRelativeFiles).toEqual(expectedRelativeFiles); + + // Ensure that the single 'tileset.json' contains the + // proper content URIs for the external tilesets: + const tilesetJsonBuffer = fs.readFileSync(outputFile); const tileset = JSON.parse(tilesetJsonBuffer.toString()); const actualContentUris = await SpecHelpers.collectExplicitContentUris( tileset.root @@ -90,8 +215,8 @@ describe("TilesetMerger", function () { actualContentUris.sort(); const expectedContentUris = [ - "specs/data/mergeTilesets/basicMerge/TilesetA/tileset.json", - "specs/data/mergeTilesets/basicMerge/sub/TilesetA/tileset.json", + "../../../mergeTilesets/basicMerge/TilesetA/tileset.json", + "../../../mergeTilesets/basicMerge/sub/TilesetA/tileset.json", ]; expect(actualContentUris).toEqual(expectedContentUris); });