From 5c61219cfad07ad61e93aee1b67001f892df68f5 Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Fri, 1 Dec 2023 14:25:03 +0000 Subject: [PATCH 01/12] ISSUE #654 Increased supermesh vertex threshold to improve download performance --- .../modeloptimizer/repo_optimizer_multipart.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bouncer/src/repo/manipulator/modeloptimizer/repo_optimizer_multipart.cpp b/bouncer/src/repo/manipulator/modeloptimizer/repo_optimizer_multipart.cpp index 49d5e286a..a35c0d385 100644 --- a/bouncer/src/repo/manipulator/modeloptimizer/repo_optimizer_multipart.cpp +++ b/bouncer/src/repo/manipulator/modeloptimizer/repo_optimizer_multipart.cpp @@ -39,8 +39,14 @@ using Scalar = float; using Bvh = bvh::Bvh; using BvhVector3 = bvh::Vector3; -static const size_t REPO_MP_MAX_VERTEX_COUNT = 65536; +// The vertex count is used as a rough approximation of the total geometry size. +// This figure is empirically set to end up with an average bundle size of 24 Mb. +static const size_t REPO_MP_MAX_VERTEX_COUNT = 1200000; + +// This limit is used to prevent metadata files becoming unwieldly, and the +// supermesh UV resolution falling below the quantisation noise floor. static const size_t REPO_MP_MAX_MESHES_IN_SUPERMESH = 5000; + static const size_t REPO_BVH_MAX_LEAF_SIZE = 16; static const size_t REPO_MODEL_LOW_CLUSTERING_RATIO = 0.2f; From ffb790c50d953ad0e70fe4bc5a138e95d423808a Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Fri, 1 Dec 2023 14:25:18 +0000 Subject: [PATCH 02/12] ISSUE #654 Gzip all asset bundles by default --- wrapper/src/c_sharp_wrapper.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/wrapper/src/c_sharp_wrapper.cpp b/wrapper/src/c_sharp_wrapper.cpp index 88340f628..b96932639 100644 --- a/wrapper/src/c_sharp_wrapper.cpp +++ b/wrapper/src/c_sharp_wrapper.cpp @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include using namespace repo::lib; @@ -386,12 +389,18 @@ bool CSharpWrapper::saveAssetBundles( if (inputStream.is_open()) { - std::ostringstream ss; - ss << inputStream.rdbuf(); - auto s = ss.str(); boost::filesystem::path path(assetFiles[i]); std::string fileName = path.filename().string(); fileName = "/" + databaseName + "/" + projectName + "/" + fileName; + + std::ostringstream ss; // memory stream containing the binary data to write to the database + + boost::iostreams::filtering_streambuf in; + in.push(boost::iostreams::gzip_compressor()); + in.push(inputStream); + boost::iostreams::copy(in, ss); + + auto s = ss.str(); webBuffers.geoFiles[fileName] = std::vector(s.begin(), s.end()); } else From a9cdbb97917f492ed891ce3c99dab46e02c1b0cf Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Wed, 6 Dec 2023 16:20:40 +0000 Subject: [PATCH 03/12] ISSUE #654 Updated web buffers type so that the metadata bson follows the vector around. Updated the wrapper to store the encoding in the bson so it gets written to the db. --- .../src/repo/lib/datastructure/repo_structs.h | 7 +++++-- .../export/repo_model_export_asset.cpp | 2 +- .../export/repo_model_export_gltf.cpp | 18 ++++++++++-------- .../export/repo_model_export_gltf.h | 2 +- .../export/repo_model_export_src.cpp | 6 +++--- .../export/repo_model_export_src.h | 2 +- .../export/repo_model_export_web.cpp | 3 ++- .../modelutility/repo_scene_manager.cpp | 2 +- wrapper/src/c_sharp_wrapper.cpp | 8 +++++++- 9 files changed, 31 insertions(+), 19 deletions(-) diff --git a/bouncer/src/repo/lib/datastructure/repo_structs.h b/bouncer/src/repo/lib/datastructure/repo_structs.h index cc3d8efd4..d2454547a 100644 --- a/bouncer/src/repo/lib/datastructure/repo_structs.h +++ b/bouncer/src/repo/lib/datastructure/repo_structs.h @@ -25,9 +25,12 @@ #include "repo_vector.h" #include +typedef std::unordered_map, repo::core::model::RepoBSON>> repo_web_geo_files_t; +typedef std::unordered_map> repo_web_json_files_t; + typedef struct { - std::unordered_map> geoFiles; //files where geometery are stored - std::unordered_map> jsonFiles; //JSON mapping files + repo_web_geo_files_t geoFiles; //files where geometery are stored + repo_web_json_files_t jsonFiles; //JSON mapping files repo::core::model::RepoUnityAssets unityAssets; //Unity assets list }repo_web_buffers_t; diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_asset.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_asset.cpp index c02b61840..5cde3b05d 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_asset.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_asset.cpp @@ -85,7 +85,7 @@ AssetModelExport::~AssetModelExport() repo_web_buffers_t AssetModelExport::getAllFilesExportedAsBuffer() const { return { - std::unordered_map>(), + repo_web_geo_files_t(), getJSONFilesAsBuffer(), getUnityAssets() }; diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp index c8df7ddc7..f3b3c3388 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp @@ -564,9 +564,9 @@ repo_web_buffers_t GLTFModelExport::getAllFilesExportedAsBuffer() const return{ getGLTFFilesAsBuffer(), getJSONFilesAsBuffer() }; } -std::unordered_map> GLTFModelExport::getGLTFFilesAsBuffer() const +repo_web_geo_files_t GLTFModelExport::getGLTFFilesAsBuffer() const { - std::unordered_map> files; + repo_web_geo_files_t files; //GLTF files for (const auto &pair : trees) { @@ -575,10 +575,11 @@ std::unordered_map> GLTFModelExport::getGLTFFi std::string jsonString = ss.str(); if (!jsonString.empty()) { - files[pair.first] = std::vector(); + files[pair.first] = { std::vector(), repo::core::model::RepoBSON() }; size_t byteLength = jsonString.size() * sizeof(*jsonString.data()); - files[pair.first].resize(byteLength); - memcpy(files[pair.first].data(), jsonString.data(), byteLength); + auto& buffer = std::get<0>(files[pair.first]); + buffer.resize(byteLength); + memcpy(buffer.data(), jsonString.data(), byteLength); } else { @@ -597,10 +598,11 @@ std::unordered_map> GLTFModelExport::getGLTFFi //None of the gltf should ever share the same name, this is a sanity check if (it == files.end()) { - files[fileName] = std::vector(); + files[fileName] = { std::vector(), repo::core::model::RepoBSON() }; + auto& buffer = std::get<0>(files[fileName]); size_t byteLength = pair.second.size() * sizeof(*pair.second.data()); - files[fileName].resize(byteLength); - memcpy(files[fileName].data(), pair.second.data(), byteLength); + buffer.resize(byteLength); + memcpy(buffer.data(), pair.second.data(), byteLength); } else { diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.h b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.h index 8ee677106..1640a296d 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.h +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.h @@ -236,7 +236,7 @@ namespace repo{ * Return the GLTF file as raw bytes buffer * returns an empty vector if the export has failed */ - std::unordered_map> getGLTFFilesAsBuffer() const; + repo_web_geo_files_t getGLTFFilesAsBuffer() const; /** * Reindex the given faces base on the given information diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp index 6d82f5ed5..a5dededb7 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp @@ -136,9 +136,9 @@ SRCModelExport::~SRCModelExport() { } -std::unordered_map> SRCModelExport::getSRCFilesAsBuffer() const +repo_web_geo_files_t SRCModelExport::getSRCFilesAsBuffer() const { - std::unordered_map < std::string, std::vector > fileBuffers; + repo_web_geo_files_t fileBuffers; for (const auto &treePair : trees) { @@ -198,7 +198,7 @@ std::unordered_map> SRCModelExport::getSRCFile //Add data buffer to the full buffer buffer.insert(buffer.end(), fullDataArray.begin(), fullDataArray.end()); - fileBuffers[fName] = buffer; + fileBuffers[fName] = { buffer, {} }; } else { diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.h b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.h index 9338e746a..8892b40c6 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.h +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.h @@ -94,7 +94,7 @@ namespace repo{ * Return the SRC file as raw bytes buffer * returns an empty vector if the export has failed */ - std::unordered_map> getSRCFilesAsBuffer() const; + repo_web_geo_files_t getSRCFilesAsBuffer() const; }; } //namespace modelconvertor } //namespace manipulator diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp index 6ae921989..a9bbed115 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp @@ -72,7 +72,8 @@ bool WebModelExport::exportToFile( FILE* fp = fopen(boostPath.string().c_str(), "wb"); if (fp) { - fwrite(buff.second.data(), sizeof(*buff.second.data()), buff.second.size(), fp); + auto content = std::get<0>(buff.second); + fwrite(content.data(), sizeof(*content.data()), content.size(), fp); fclose(fp); } else diff --git a/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp b/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp index b36260d4a..34fb679d2 100644 --- a/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp +++ b/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp @@ -43,7 +43,7 @@ bool SceneManager::commitWebBuffers( //Upload the files for (const auto &bufferPair : resultBuffers.geoFiles) { - if (success &= fileManager->uploadFileAndCommit(databaseName, projectName + "." + geoStashExt, bufferPair.first, bufferPair.second)) + if (success &= fileManager->uploadFileAndCommit(databaseName, projectName + "." + geoStashExt, bufferPair.first, std::get<0>(bufferPair.second), std::get<1>(bufferPair.second))) { repoInfo << "File (" << bufferPair.first << ") added successfully to file storage."; } diff --git a/wrapper/src/c_sharp_wrapper.cpp b/wrapper/src/c_sharp_wrapper.cpp index b96932639..d3751b3c9 100644 --- a/wrapper/src/c_sharp_wrapper.cpp +++ b/wrapper/src/c_sharp_wrapper.cpp @@ -17,6 +17,7 @@ #include "c_sharp_wrapper.h" +#include #include #include #include @@ -395,13 +396,18 @@ bool CSharpWrapper::saveAssetBundles( std::ostringstream ss; // memory stream containing the binary data to write to the database + // All unity asset bundles are gzipped by default + boost::iostreams::filtering_streambuf in; in.push(boost::iostreams::gzip_compressor()); in.push(inputStream); boost::iostreams::copy(in, ss); + repo::core::model::RepoBSONBuilder builder; + builder.append("encoding", "gzip"); + auto s = ss.str(); - webBuffers.geoFiles[fileName] = std::vector(s.begin(), s.end()); + webBuffers.geoFiles[fileName] = { std::vector(s.begin(), s.end()), builder.obj() }; } else { From 2ad91e9e759bda07038094f3f95f5a163b66d0ad Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 14 Dec 2023 11:06:08 +0000 Subject: [PATCH 04/12] ISSSUE #654 Moved compression into file manager, with a new parameter to control it. Set all the webbuffers to be gzipped by default, because .io can decompress as required. --- .../handler/fileservice/repo_file_manager.cpp | 57 +++++++++++++++++-- .../handler/fileservice/repo_file_manager.h | 11 +++- .../modelutility/repo_scene_manager.cpp | 2 +- wrapper/src/c_sharp_wrapper.cpp | 23 ++------ 4 files changed, 68 insertions(+), 25 deletions(-) diff --git a/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp b/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp index e362b7bcc..8bae46272 100644 --- a/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp +++ b/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp @@ -18,9 +18,15 @@ #include "repo_file_manager.h" #include "../../../lib/repo_exception.h" #include "../../model/repo_model_global.h" +#include "../../model/bson/repo_bson_builder.h" #include "../../model/bson/repo_bson_factory.h" #include "repo_file_handler_fs.h" #include "repo_file_handler_gridfs.h" +#include +#include +#include +#include +#include using namespace repo::core::handler::fileservice; @@ -46,11 +52,45 @@ bool FileManager::uploadFileAndCommit( const std::string &collectionNamePrefix, const std::string &fileName, const std::vector &bin, - const repo::core::model::RepoBSON &metadata) + const repo::core::model::RepoBSON &metadata, + const Encoding &encoding) { bool success = true; auto fileUUID = repo::lib::RepoUUID::createUUID(); - auto linkName = defaultHandler->uploadFile(databaseName, collectionNamePrefix, fileUUID.toString(), bin); + + // Create local references that can be reassigned, if we need to do any + // further processing... + + const std::vector* fileContents = &bin; + repo::core::model::RepoBSON fileMetadata = metadata; + + switch (encoding) + { + case Encoding::Gzip: + { + // Use stringstream as a binary container to hold the compressed data + std::ostringstream compressedstream; + + // Bufferstream operates directly over the user provided array + boost::interprocess::bufferstream uncompressed((char*)bin.data(), bin.size()); + + boost::iostreams::filtering_istream in; + in.push(boost::iostreams::gzip_compressor()); + in.push(std::istream(uncompressed.rdbuf())); // For some reason bufferstream is ambigous between stream and streambuf, so wrap it unambiguously + + boost::iostreams::copy(in, compressedstream); + + auto compresseddata = compressedstream.str(); + fileContents = new std::vector(compresseddata.begin(), compresseddata.end()); + + repo::core::model::RepoBSONBuilder builder; + builder.append("encoding", "gzip"); + fileMetadata = metadata.cloneAndAddFields(&builder.obj()); + } + break; + } + + auto linkName = defaultHandler->uploadFile(databaseName, collectionNamePrefix, fileUUID.toString(), *fileContents); if (success = !linkName.empty()) { success = upsertFileRef( databaseName, @@ -58,8 +98,17 @@ bool FileManager::uploadFileAndCommit( cleanFileName(fileName), linkName, defaultHandler->getType(), - bin.size(), - metadata); + fileContents->size(), + fileMetadata); + } + + // If we've created a new vector, clean it up. metadata doesn't need to be + // deleted because Mongo BSONs have built in smart-pointers allowing them to + // be passed by value. + + if (fileContents != &bin) + { + delete fileContents; } return success; diff --git a/bouncer/src/repo/core/handler/fileservice/repo_file_manager.h b/bouncer/src/repo/core/handler/fileservice/repo_file_manager.h index a407624cc..ab1f23077 100644 --- a/bouncer/src/repo/core/handler/fileservice/repo_file_manager.h +++ b/bouncer/src/repo/core/handler/fileservice/repo_file_manager.h @@ -54,6 +54,14 @@ namespace repo { repo::core::handler::AbstractDatabaseHandler *dbHandler ); + /* + * Possible options for static compression of stored files + */ + enum Encoding { + None = 0, + Gzip = 1 + }; + /** * Upload file and commit ref entry to database. */ @@ -62,7 +70,8 @@ namespace repo { const std::string &collectionNamePrefix, const std::string &fileName, const std::vector &bin, - const repo::core::model::RepoBSON &metadata = repo::core::model::RepoBSON() + const repo::core::model::RepoBSON &metadata = repo::core::model::RepoBSON(), + const Encoding &encoding = Encoding::None ); /** diff --git a/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp b/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp index 34fb679d2..566ca2b98 100644 --- a/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp +++ b/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp @@ -43,7 +43,7 @@ bool SceneManager::commitWebBuffers( //Upload the files for (const auto &bufferPair : resultBuffers.geoFiles) { - if (success &= fileManager->uploadFileAndCommit(databaseName, projectName + "." + geoStashExt, bufferPair.first, std::get<0>(bufferPair.second), std::get<1>(bufferPair.second))) + if (success &= fileManager->uploadFileAndCommit(databaseName, projectName + "." + geoStashExt, bufferPair.first, std::get<0>(bufferPair.second), std::get<1>(bufferPair.second), repo::core::handler::fileservice::FileManager::Encoding::Gzip)) // Web geometry files are gzipped by default { repoInfo << "File (" << bufferPair.first << ") added successfully to file storage."; } diff --git a/wrapper/src/c_sharp_wrapper.cpp b/wrapper/src/c_sharp_wrapper.cpp index d3751b3c9..27edc4801 100644 --- a/wrapper/src/c_sharp_wrapper.cpp +++ b/wrapper/src/c_sharp_wrapper.cpp @@ -17,14 +17,10 @@ #include "c_sharp_wrapper.h" -#include #include #include #include #include -#include -#include -#include #include using namespace repo::lib; @@ -390,24 +386,13 @@ bool CSharpWrapper::saveAssetBundles( if (inputStream.is_open()) { + std::ostringstream ss; + ss << inputStream.rdbuf(); + auto s = ss.str(); boost::filesystem::path path(assetFiles[i]); std::string fileName = path.filename().string(); fileName = "/" + databaseName + "/" + projectName + "/" + fileName; - - std::ostringstream ss; // memory stream containing the binary data to write to the database - - // All unity asset bundles are gzipped by default - - boost::iostreams::filtering_streambuf in; - in.push(boost::iostreams::gzip_compressor()); - in.push(inputStream); - boost::iostreams::copy(in, ss); - - repo::core::model::RepoBSONBuilder builder; - builder.append("encoding", "gzip"); - - auto s = ss.str(); - webBuffers.geoFiles[fileName] = { std::vector(s.begin(), s.end()), builder.obj() }; + webBuffers.geoFiles[fileName] = { std::vector(s.begin(), s.end()), { } }; } else { From de0d455484566912219333bccdccbd7a0956f822 Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 14 Dec 2023 11:55:01 +0000 Subject: [PATCH 05/12] ISSUE #654 Removed the bson from repo_web_geo_files since its not currently used --- bouncer/src/repo/lib/datastructure/repo_structs.h | 8 ++++---- .../modelconvertor/export/repo_model_export_gltf.cpp | 8 ++++---- .../modelconvertor/export/repo_model_export_web.cpp | 2 +- .../repo/manipulator/modelutility/repo_scene_manager.cpp | 2 +- wrapper/src/c_sharp_wrapper.cpp | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bouncer/src/repo/lib/datastructure/repo_structs.h b/bouncer/src/repo/lib/datastructure/repo_structs.h index d2454547a..484b7de97 100644 --- a/bouncer/src/repo/lib/datastructure/repo_structs.h +++ b/bouncer/src/repo/lib/datastructure/repo_structs.h @@ -25,14 +25,14 @@ #include "repo_vector.h" #include -typedef std::unordered_map, repo::core::model::RepoBSON>> repo_web_geo_files_t; -typedef std::unordered_map> repo_web_json_files_t; +using repo_web_geo_files_t = std::unordered_map>; +using repo_web_json_files_t = std::unordered_map>; -typedef struct { +using repo_web_buffers_t = struct { repo_web_geo_files_t geoFiles; //files where geometery are stored repo_web_json_files_t jsonFiles; //JSON mapping files repo::core::model::RepoUnityAssets unityAssets; //Unity assets list -}repo_web_buffers_t; +}; //This is used to map info for multipart optimization typedef struct { diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp index f3b3c3388..1f577d16b 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp @@ -575,9 +575,9 @@ repo_web_geo_files_t GLTFModelExport::getGLTFFilesAsBuffer() const std::string jsonString = ss.str(); if (!jsonString.empty()) { - files[pair.first] = { std::vector(), repo::core::model::RepoBSON() }; + files[pair.first] = std::vector(); size_t byteLength = jsonString.size() * sizeof(*jsonString.data()); - auto& buffer = std::get<0>(files[pair.first]); + auto& buffer = files[pair.first]; buffer.resize(byteLength); memcpy(buffer.data(), jsonString.data(), byteLength); } @@ -598,8 +598,8 @@ repo_web_geo_files_t GLTFModelExport::getGLTFFilesAsBuffer() const //None of the gltf should ever share the same name, this is a sanity check if (it == files.end()) { - files[fileName] = { std::vector(), repo::core::model::RepoBSON() }; - auto& buffer = std::get<0>(files[fileName]); + files[fileName] = std::vector(); + auto& buffer = files[fileName]; size_t byteLength = pair.second.size() * sizeof(*pair.second.data()); buffer.resize(byteLength); memcpy(buffer.data(), pair.second.data(), byteLength); diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp index a9bbed115..24de31db3 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp @@ -72,7 +72,7 @@ bool WebModelExport::exportToFile( FILE* fp = fopen(boostPath.string().c_str(), "wb"); if (fp) { - auto content = std::get<0>(buff.second); + auto content = buff.second; fwrite(content.data(), sizeof(*content.data()), content.size(), fp); fclose(fp); } diff --git a/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp b/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp index 566ca2b98..2672b084a 100644 --- a/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp +++ b/bouncer/src/repo/manipulator/modelutility/repo_scene_manager.cpp @@ -43,7 +43,7 @@ bool SceneManager::commitWebBuffers( //Upload the files for (const auto &bufferPair : resultBuffers.geoFiles) { - if (success &= fileManager->uploadFileAndCommit(databaseName, projectName + "." + geoStashExt, bufferPair.first, std::get<0>(bufferPair.second), std::get<1>(bufferPair.second), repo::core::handler::fileservice::FileManager::Encoding::Gzip)) // Web geometry files are gzipped by default + if (success &= fileManager->uploadFileAndCommit(databaseName, projectName + "." + geoStashExt, bufferPair.first, bufferPair.second, {}, repo::core::handler::fileservice::FileManager::Encoding::Gzip)) // Web geometry files are gzipped by default { repoInfo << "File (" << bufferPair.first << ") added successfully to file storage."; } diff --git a/wrapper/src/c_sharp_wrapper.cpp b/wrapper/src/c_sharp_wrapper.cpp index 27edc4801..88340f628 100644 --- a/wrapper/src/c_sharp_wrapper.cpp +++ b/wrapper/src/c_sharp_wrapper.cpp @@ -392,7 +392,7 @@ bool CSharpWrapper::saveAssetBundles( boost::filesystem::path path(assetFiles[i]); std::string fileName = path.filename().string(); fileName = "/" + databaseName + "/" + projectName + "/" + fileName; - webBuffers.geoFiles[fileName] = { std::vector(s.begin(), s.end()), { } }; + webBuffers.geoFiles[fileName] = std::vector(s.begin(), s.end()); } else { From f532b5a77e40c51eb8a4ae95d1893c7cc87c879d Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 14 Dec 2023 11:58:05 +0000 Subject: [PATCH 06/12] ISSUE #654 Updated src exporter for revised type --- .../manipulator/modelconvertor/export/repo_model_export_src.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp index a5dededb7..b53337c1b 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_src.cpp @@ -198,7 +198,7 @@ repo_web_geo_files_t SRCModelExport::getSRCFilesAsBuffer() const //Add data buffer to the full buffer buffer.insert(buffer.end(), fullDataArray.begin(), fullDataArray.end()); - fileBuffers[fName] = { buffer, {} }; + fileBuffers[fName] = buffer; } else { From 4b64aed7371f41be58e41682ee5f5c2d6d07cefa Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 14 Dec 2023 11:59:39 +0000 Subject: [PATCH 07/12] ISSUE #654 Removed variable as was not a reference --- .../modelconvertor/export/repo_model_export_web.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp index 24de31db3..6ae921989 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_web.cpp @@ -72,8 +72,7 @@ bool WebModelExport::exportToFile( FILE* fp = fopen(boostPath.string().c_str(), "wb"); if (fp) { - auto content = buff.second; - fwrite(content.data(), sizeof(*content.data()), content.size(), fp); + fwrite(buff.second.data(), sizeof(*buff.second.data()), buff.second.size(), fp); fclose(fp); } else From ce6315f26b7ad3f302fd2beadbc4b7e17467ace3 Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 14 Dec 2023 12:01:18 +0000 Subject: [PATCH 08/12] ISSUE #654 Removed unneeded variable interdiction --- .../modelconvertor/export/repo_model_export_gltf.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp index 1f577d16b..d1bef9779 100644 --- a/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp +++ b/bouncer/src/repo/manipulator/modelconvertor/export/repo_model_export_gltf.cpp @@ -577,9 +577,8 @@ repo_web_geo_files_t GLTFModelExport::getGLTFFilesAsBuffer() const { files[pair.first] = std::vector(); size_t byteLength = jsonString.size() * sizeof(*jsonString.data()); - auto& buffer = files[pair.first]; - buffer.resize(byteLength); - memcpy(buffer.data(), jsonString.data(), byteLength); + files[pair.first].resize(byteLength); + memcpy(files[pair.first].data(), jsonString.data(), byteLength); } else { @@ -599,10 +598,9 @@ repo_web_geo_files_t GLTFModelExport::getGLTFFilesAsBuffer() const if (it == files.end()) { files[fileName] = std::vector(); - auto& buffer = files[fileName]; size_t byteLength = pair.second.size() * sizeof(*pair.second.data()); - buffer.resize(byteLength); - memcpy(buffer.data(), pair.second.data(), byteLength); + files[fileName].resize(byteLength); + memcpy(files[fileName].data(), pair.second.data(), byteLength); } else { From aa66fa81c4e578ad1d06e40a0299633e9ba34cfb Mon Sep 17 00:00:00 2001 From: sebjf Date: Tue, 9 Jan 2024 10:13:36 +0000 Subject: [PATCH 09/12] ISSUE #654 Minor tweaks to compile on GCC --- .../repo/core/handler/fileservice/repo_file_manager.cpp | 8 ++++++-- bouncer/src/repo/lib/datastructure/repo_structs.h | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp b/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp index 8bae46272..f783b8b03 100644 --- a/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp +++ b/bouncer/src/repo/core/handler/fileservice/repo_file_manager.cpp @@ -74,9 +74,12 @@ bool FileManager::uploadFileAndCommit( // Bufferstream operates directly over the user provided array boost::interprocess::bufferstream uncompressed((char*)bin.data(), bin.size()); + // In stream form for filtering_istream + std::istream uncompressedStream(uncompressed.rdbuf()); + boost::iostreams::filtering_istream in; in.push(boost::iostreams::gzip_compressor()); - in.push(std::istream(uncompressed.rdbuf())); // For some reason bufferstream is ambigous between stream and streambuf, so wrap it unambiguously + in.push(uncompressedStream); // For some reason bufferstream is ambigous between stream and streambuf, so wrap it unambiguously boost::iostreams::copy(in, compressedstream); @@ -85,7 +88,8 @@ bool FileManager::uploadFileAndCommit( repo::core::model::RepoBSONBuilder builder; builder.append("encoding", "gzip"); - fileMetadata = metadata.cloneAndAddFields(&builder.obj()); + auto bson = builder.obj(); + fileMetadata = metadata.cloneAndAddFields(&bson); } break; } diff --git a/bouncer/src/repo/lib/datastructure/repo_structs.h b/bouncer/src/repo/lib/datastructure/repo_structs.h index 484b7de97..afaad0ecc 100644 --- a/bouncer/src/repo/lib/datastructure/repo_structs.h +++ b/bouncer/src/repo/lib/datastructure/repo_structs.h @@ -28,7 +28,7 @@ using repo_web_geo_files_t = std::unordered_map>; using repo_web_json_files_t = std::unordered_map>; -using repo_web_buffers_t = struct { +struct repo_web_buffers_t{ repo_web_geo_files_t geoFiles; //files where geometery are stored repo_web_json_files_t jsonFiles; //JSON mapping files repo::core::model::RepoUnityAssets unityAssets; //Unity assets list From 638737dda57b2def5ecef89bfd7eb4c360b2e54e Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 11 Jan 2024 09:40:44 +0000 Subject: [PATCH 10/12] ISSUE #654 Fixed typo in Readme (this is really to make Travis run...) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 364966377..d3f905d82 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ 3drepobouncer [![Build Status](https://travis-ci.org/3drepo/3drepobouncer.svg?branch=master)](https://travis-ci.org/3drepo/3drepobouncer) [![Coverage Status](https://coveralls.io/repos/github/3drepo/3drepobouncer/badge.svg?branch=master)](https://coveralls.io/github/3drepo/3drepobouncer?branch=master) ========= -3DRepoBouncer is essentially the refactored 3DRepoCore and (parts of) 3DRepoGUI. It is a C++ library providing 3D Repo Scene Graph definition, repository management and manipulation logic as well as direct MongoDB databse access. +3DRepoBouncer is essentially the refactored 3DRepoCore and (parts of) 3DRepoGUI. It is a C++ library providing 3D Repo Scene Graph definition, repository management and manipulation logic as well as direct MongoDB database access. ### Latest Releases We always recommend using the [Latest stable release](https://github.com/3drepo/3drepobouncer/releases). However, to access cutting-edge development versions, check out the [tags](https://github.com/3drepo/3drepobouncer/tags). From ad07f71644dfaab0b381d88f2951a2aeb28da804 Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 11 Jan 2024 10:42:28 +0000 Subject: [PATCH 11/12] ISSUE #654 Updated travis build script to use git instead of svn based on this: https://stackoverflow.com/a/73254328/470687 --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 250ca13cb..ab50e27e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,9 @@ matrix: - sudo apt-get install -y gnupg mongodb-org=5.0.8 mongodb-org-database=5.0.8 mongodb-org-server=5.0.8 mongodb-org-shell=5.0.8 mongodb-org-tools=5.0.8 - sudo systemctl daemon-reload && sudo systemctl start mongod && echo $(mongod --version) - mkdir testData && cd testData - - svn --no-auth-cache export --username $TESTUSER --password $TESTPW https://github.com/3drepo/tests/trunk/cplusplus/bouncer + - git clone --filter=blob:none --sparse https://$TESTUSER:$TESTPW@github.com/3drepo/tests.git . + - git sparse-checkout add cplusplus/bouncer + - mv cplusplus/bouncer/ . - cd ../ - until nc -z localhost 27017; do echo Waiting for MongoDB; sleep 1; done - mongo admin testData/bouncer/createUser.js @@ -114,7 +116,9 @@ matrix: - export CXX="g++" - export CC="gcc" - mkdir testData && cd testData - - svn --no-auth-cache export --username $TESTUSER --password $TESTPW https://github.com/3drepo/tests/trunk/cplusplus/bouncer/ext_libs/focal + - git clone --filter=blob:none --sparse https://$TESTUSER:$TESTPW@github.com/3drepo/tests.git . + - git sparse-checkout add cplusplus/bouncer/ext_libs/focal + - mv cplusplus/bouncer/ext_libs/focal/ . - cd ../ - echo ============ BOOST INSTALL ============= - sudo apt-get install libboost-all-dev From f3faf93331b575fa2728d21cceb4ca8656e56c6b Mon Sep 17 00:00:00 2001 From: Sebastian Friston Date: Thu, 11 Jan 2024 11:33:20 +0000 Subject: [PATCH 12/12] ISSUE #654 updated test parameters given new supermesh limits --- .../manipulator/modeloptimizer/ut_repo_optimizer_multipart.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/unit/repo/manipulator/modeloptimizer/ut_repo_optimizer_multipart.cpp b/test/src/unit/repo/manipulator/modeloptimizer/ut_repo_optimizer_multipart.cpp index c9455b441..83da89a4f 100644 --- a/test/src/unit/repo/manipulator/modeloptimizer/ut_repo_optimizer_multipart.cpp +++ b/test/src/unit/repo/manipulator/modeloptimizer/ut_repo_optimizer_multipart.cpp @@ -389,7 +389,7 @@ TEST(MultipartOptimizer, TestSingleOversizedMesh) auto nMesh = 3; repo::core::model::RepoNodeSet meshes, trans, dummy; trans.insert(root); - meshes.insert(createRandomMesh(65537, false, 3, { rootID })); + meshes.insert(createRandomMesh(1200000 + 1, false, 3, { rootID })); // 1200000 comes from the const in repo_optimizer_multipart.cpp repo::core::model::RepoScene* scene = new repo::core::model::RepoScene({}, dummy, meshes, dummy, dummy, dummy, trans); ASSERT_TRUE(scene->hasRoot(DEFAULT_GRAPH));