Issue 5223 Release candidate 5.12.0

.azure/build-and-deploy.yaml

@@ -120,7 +120,7 @@ stages:
           command: 'upgrade'
           chartType: 'Name'
           chartName: '3drepo/io'
-          chartVersion: '5.7.0'
+          chartVersion: '5.11.2'
           waitForExecution: false
           releaseName: '$(branchName)'
           overrideValues: 'image.tag=$(Build.SourceVersion),branchName=$(branchName),$(customHelmOverride)'

.travis.yml

@@ -63,4 +63,4 @@ matrix:
 notifications:
     email:
         recipients:
-            - devOps@3drepo.org
+            - 3drepo[email protected]

backend/.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+	// Use IntelliSense to learn about possible attributes.
+	// Hover to view descriptions of existing attributes.
+	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"type": "node",
+			"request": "attach",
+			"name": "Attach Program",
+			"skipFiles": [
+				"<node_internals>/**"
+			],
+			"port": 9229,
+			"address": "localhost"
+		}
+	]
+}

backend/VERSION.json

@@ -1,4 +1,4 @@
-{ "VERSION" : "5.10.6",
+{ "VERSION" : "5.11.1",
   "unity" : {
 	"current" : "2.20.0",
 	"supported": []

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "3drepo.io",
-  "version": "5.10.6",
+  "version": "5.11.1",
   "engines": {
     "node": "18.x.x"
   },
@@ -25,93 +25,92 @@
     "run-scheduled-tasks": "cross-env NODE_CONFIG_DIR=../config node ./src/scripts/utility/scheduler"
   },
   "dependencies": {
-    "@azure/msal-node": "1.16.0",
-    "@elastic/elasticsearch": "7.17.0",
+    "@azure/msal-node": "1.18.4",
+    "@elastic/elasticsearch": "7.17.14",
     "amqplib": "0.8.0",
     "apidoc": "0.29.0",
-    "app-config": "1.0.0",
-    "archiver": "5.3.1",
-    "async": "2.6.4",
-    "axios": "0.28.0",
-    "body-parser": "1.20.2",
+    "app-config": "1.0.1",
+    "archiver": "6.0.2",
+    "async": "3.2.5",
+    "axios": "1.7.2",
+    "body-parser": "1.20.3",
     "combined-stream": "1.0.8",
-    "compression": "1.7.3",
+    "compression": "1.7.4",
     "connect-mongo": "4.6.0",
-    "cookie-parser": "1.4.6",
+    "cookie-parser": "1.4.7",
     "cors": "2.8.5",
-    "countries-and-timezones": "3.3.0",
+    "countries-and-timezones": "3.6.0",
     "crypto-js": "4.2.0",
     "cryptolens": "1.0.1-4.2",
     "csv-parse": "4.8.5",
-    "dayjs": "1.10.7",
+    "dayjs": "1.11.11",
     "device": "0.3.12",
     "ejs": "3.1.10",
-    "elastic-apm-node": "3.41.1",
-    "express": "4.17.3",
-    "express-body-parser-error-handler": "1.0.4",
-    "express-session": "1.17.3",
-    "express-socket.io-session": "1.3.5",
+    "elastic-apm-node": "4.8.0",
+    "express": "4.21.1",
+    "express-body-parser-error-handler": "1.0.7",
+    "express-session": "1.18.1",
     "file-type": "16.5.4",
-    "geoip-lite": "1.4.2",
+    "geoip-lite": "1.4.10",
     "is-html-content": "1.0.0",
     "json2csv": "4.3.3",
     "lodash": "4.17.21",
-    "mime-types": "2.1.24",
-    "moment": "2.29.4",
-    "moment-timezone": "0.5.41",
+    "mime-types": "2.1.35",
+    "moment": "2.30.1",
+    "moment-timezone": "0.5.45",
     "mongodb": "3.7.3",
-    "multer": "1.4.4",
+    "multer": "1.4.5-lts.1",
     "nanoid": "3.3.1",
-    "nodemailer": "6.9.9",
-    "openapi-schema-validator": "12.0.0",
+    "nodemailer": "6.9.14",
+    "openapi-schema-validator": "12.1.3",
     "paypal-rest-sdk": "1.8.1",
-    "pug": "3.0.2",
+    "pug": "3.0.3",
     "serialize-javascript": "4.0.0",
     "serve-favicon": "2.5.0",
     "sharp": "0.32.6",
     "slash": "3.0.0",
-    "socket.io": "4.6.1",
+    "socket.io": "4.8.0",
     "string-to-stream": "3.0.1",
     "supertest-session": "4.1.0",
-    "swagger-jsdoc": "6.1.0",
+    "swagger-jsdoc": "6.2.8",
     "swagger-ui-express": "4.3.0",
     "ua-parser-js": "0.7.33",
     "useragent": "2.3.0",
     "uuid": "8.3.2",
     "uuid-parse": "1.1.0",
     "vhost": "3.0.2",
-    "winston": "3.8.2",
+    "winston": "3.13.0",
     "winston-daily-rotate-file": "3.10.0",
     "xml2js": "0.5.0",
     "xmlbuilder": "10.1.1",
-    "yargs": "17.4.0",
+    "yargs": "17.7.2",
     "yauzl-promise": "2.1.3",
     "yup": "0.32.9",
     "zxcvbn": "4.4.2"
   },
   "devDependencies": {
-    "@jest/test-sequencer": "29.4.1",
-    "@shelf/jest-mongodb": "4.1.6",
+    "@jest/test-sequencer": "29.7.0",
+    "@shelf/jest-mongodb": "4.3.2",
     "chai": "4.2.0",
     "chai-shallow-deep-equal": "1.4.6",
-    "cross-env": "7.0.2",
+    "cross-env": "7.0.3",
     "deep-equal-in-any-order": "1.1.15",
     "eslint": "8.35.0",
     "eslint-config-airbnb-base": "15.0.0",
-    "eslint-plugin-import": "2.27.5",
+    "eslint-plugin-import": "2.29.1",
     "eslint-plugin-node": "11.1.0",
     "eslint-plugin-require-sort": "1.3.0",
     "eslint-plugin-security": "1.7.1",
     "form-data": "4.0.0",
-    "jest": "29.4.1",
-    "jest-environment-node": "29.4.1",
+    "jest": "29.7.0",
+    "jest-environment-node": "29.7.0",
     "mocha": "9.2.2",
     "mock-express-request": "0.2.2",
     "nodemon": "2.0.21",
-    "openapi-types": "12.1.0",
-    "proxyquire": "2.1.0",
+    "openapi-types": "12.1.3",
+    "proxyquire": "2.1.3",
     "sinon": "7.2.3",
-    "socket.io-client": "4.6.1",
+    "socket.io-client": "4.8.0",
     "supertest": "3.4.2"
   },
   "description": "The backend for 3drepo.io",

backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js

@@ -0,0 +1,100 @@
+/**
+ *  Copyright (C) 2024 3D Repo Ltd
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU Affero General Public License as
+ *  published by the Free Software Foundation, either version 3 of the
+ *  License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU Affero General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Affero General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/**
+ * This script is used to check the processing status of models/drawings.
+ * Processing status should be 'ok' or 'failed'.
+ * The utility script `resetProcessingFlag` can be used to reset zombie statuses for models.
+ */
+
+const { v5Path } = require('../../../interop');
+
+const { logger } = require(`${v5Path}/utils/logger`);
+const { getTeamspaceList } = require('../../utils');
+
+const { find } = require(`${v5Path}/handler/db`);
+const { SETTINGS_COL, processStatuses } = require(`${v5Path}/models/modelSettings.constants`);
+const { DRAWINGS_HISTORY_COL } = require(`${v5Path}/models/revisions.constants`);
+const { sendSystemEmail } = require(`${v5Path}/services/mailer`);
+const { templates: emailTemplates } = require(`${v5Path}/services/mailer/mailer.constants`);
+const { UUIDToString } = require(`${v5Path}/utils/helper/uuids`);
+const Path = require('path');
+
+let TIME_LIMIT = 24 * 60 * 60 * 1000; // hours * 1 hour in ms
+
+const processTeamspace = async (teamspace) => {
+	const expiredTimestamp = new Date(new Date() - TIME_LIMIT);
+	const zombieQuery = {
+		status: { $exists: true, $not: { $regex: `(${processStatuses.OK})|(${processStatuses.FAILED})` } },
+		timestamp: { $lt: expiredTimestamp },
+	};
+
+	logger.logInfo(`\t-${teamspace}`);
+
+	const zombieModels = await find(teamspace, SETTINGS_COL, zombieQuery, { status: 1, timestamp: 1 });
+	const zombieDrawings = await find(teamspace, DRAWINGS_HISTORY_COL, zombieQuery, { status: 1, timestamp: 1 });
+
+	return [
+		...zombieModels.map(({ _id, status, timestamp }) => `${teamspace}, model, ${_id}, ${status}, ${timestamp}`),
+		...zombieDrawings.map(({ _id, status, timestamp }) => `${teamspace}, drawing, ${UUIDToString(_id)}, ${status}, ${timestamp}`),
+	];
+};
+
+const run = async (teamspace, limit, notify) => {
+	logger.logInfo(`Check processing flag(s) in ${teamspace ?? 'all teamspaces'}`);
+
+	if (limit) {
+		TIME_LIMIT = limit * 60 * 60 * 1000;
+	}
+
+	const teamspaces = teamspace ? [teamspace] : await getTeamspaceList();
+	const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts)))).flat();
+
+	if (notify && results.length > 0) {
+		logger.logInfo(`Zombie processing statuses found: ${results.length}`);
+		const data = {
+			script: Path.basename(__filename, Path.extname(__filename)),
+			title: 'Zombie processing statuses found',
+			message: `${results.length} zombie processing statuses found`,
+			logExcerpt: JSON.stringify(results),
+		};
+		await sendSystemEmail(emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, data);
+	}
+};
+
+const genYargs = /* istanbul ignore next */(yargs) => {
+	const commandName = Path.basename(__filename, Path.extname(__filename));
+	const argsSpec = (subYargs) => subYargs.option('teamspace', {
+		describe: 'Target a specific teamspace (if unspecified, all teamspaces will be targetted)',
+		type: 'string',
+	}).option('limit', {
+		describe: 'Time limit (hours, default: 24) where models/drawings may still be processing',
+		type: 'number',
+	}).option('notify', {
+		describe: 'Send e-mail notification if results are found (default: false)',
+		type: 'boolean',
+	});
+	return yargs.command(commandName,
+		'Checks the processing status of models/drawings.',
+		argsSpec,
+		(argv) => run(argv.teamspace, argv.limit, argv.notify));
+};
+
+module.exports = {
+	run,
+	genYargs,
+};

backend/src/scripts/utility/scene/removeIncompleteRevisions.js

@@ -31,6 +31,7 @@ const readline = require('readline');
 const { deleteMany, find } = require(`${v5Path}/handler/db`);
 const { removeFilesWithMeta } = require(`${v5Path}/services/filesManager`);
 const { UUIDToString } = require(`${v5Path}/utils/helper/uuids`);
+const { modelTypes } = require(`${v5Path}/models/modelSettings.constants`);
 
 const entriesLimit = 500;
 
@@ -123,11 +124,27 @@ const removeRevisions = async (teamspace, model, revNodes) => {
 	await removeRecords(teamspace, `${model}.scene.ref`, { rev_id: { $in: revIds } });
 };
 
+const removeDrawingRevisions = async (teamspace, revNodes) => {
+	const revIds = revNodes.map(({ _id }) => _id);
+	const rFiles = revNodes.flatMap(({ rFile }) => rFile ?? []);
+
+	logger.logInfo(`\t\t-Drawings - removing ${revIds.length} zombie revisions`);
+	logger.logInfo('\t\t\tRemoving model files');
+	await removeFilesHelper(teamspace, `${modelTypes.DRAWING}s.history.ref`, { _id: { $in: rFiles } });
+};
+
 const cleanUpRevisions = async (teamspace, colName, filter) => {
+	const colSubstring = colName.slice(0, -('.history'.length));
+	const isDrawing = colSubstring === `${modelTypes.DRAWING}s`;
+
 	const badRevisions = await find(teamspace, colName, filter, { rFile: 1 });
+
 	if (badRevisions.length) {
-		const model = colName.slice(0, -('.history'.length));
-		await removeRevisions(teamspace, model, badRevisions);
+		if (isDrawing) {
+			await removeDrawingRevisions(teamspace, badRevisions);
+		} else {
+			await removeRevisions(teamspace, colSubstring, badRevisions);
+		}
 	}
 
 	await removeRecords(teamspace, colName, filter);

backend/src/v4/config.js

@@ -273,10 +273,12 @@ config.cn_queue = { maxRetries: 3, ...config.cn_queue};
 // upload size limit
 config.uploadSizeLimit = coalesce(config.uploadSizeLimit, 209715200);
 config.resourceUploadSizeLimit =  config.resourceUploadSizeLimit || 104857600;
+config.drawingSizeLimit = coalesce(config.drawingSizeLimit, 524288000);
 
 // upload configurations (v5)
 config.fileUploads = {
 	modelSizeLimit: config.uploadSizeLimit,
+	drawingSizeLimit: config.drawingSizeLimit,
 	resourceSizeLimit:config.resourceUploadSizeLimit,
 	imageSizeLimit: 1048576,
 	imageExtensions: ["png", "jpg", "jpeg", "gif"],