From a33afe8d86ddeaa65419c54d57e2c49d3e8177e2 Mon Sep 17 00:00:00 2001 From: Charence Date: Wed, 14 Aug 2024 12:59:36 +0100 Subject: [PATCH 01/15] ISSUE #5106 - check flag and output to console ISSUE #5106 - send e-mail notification for script ISSUE #5106 - check processing flag test ISSUE #5106 - check processing ISSUE #5106 - handle sendMail ISSUE #5106 - mock sendSystemEmail ISSUE #4761 - standardise date formats across app ISSUE #4761 - stop passing formatted message to intl function that takes specific string inputs ISSUE #5115 - bump chart to 5.11.2 ISSUE #5106 - rename script --- .../modelProcessing/detectZombieProcessing.js | 100 +++++++++++++ .../detectZombieProcessing.test.js | 135 ++++++++++++++++++ .../revisionsListItem.component.tsx | 14 +- 3 files changed, 245 insertions(+), 4 deletions(-) create mode 100644 backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js create mode 100644 backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js diff --git a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js new file mode 100644 index 00000000000..c2db90c690d --- /dev/null +++ b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js @@ -0,0 +1,100 @@ +/** + * Copyright (C) 2024 3D Repo Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * This script is used to check the processing state of model(s). + * Model status should be 'ok' or 'failed' unless it is either 'queued' or 'processing'. + * Models that have a 'queued' or 'processing' status while not present in the queued + * have in a bad state and resetProcessingFlag can be used to reset it. + */ + +const { v5Path } = require('../../../interop'); + +const { logger } = require(`${v5Path}/utils/logger`); +const { getTeamspaceList } = require('../../utils'); + +const { find } = require(`${v5Path}/handler/db`); +const { SETTINGS_COL } = require(`${v5Path}/models/modelSettings.constants`); +const { sendSystemEmail } = require(`${v5Path}/services/mailer`); +const { templates: emailTemplates } = require(`${v5Path}/services/mailer/mailer.constants`); +const Path = require('path'); + +let TIME_LIMIT = 24 * 60 * 60 * 1000; // hours * 1 hour in ms + +const processTeamspace = async (teamspace, model) => { + const expiredTimestamp = new Date((new Date()) - TIME_LIMIT); + const incompleteStatusQuery = { $or: [{ status: 'processing' }, { status: 'queued' }], timestamp: { $lt: expiredTimestamp } }; + const query = model ? { ...incompleteStatusQuery, _id: model } : incompleteStatusQuery; + + const incompleteModels = await find(teamspace, SETTINGS_COL, query, { status: 1, timestamp: 1 }); + logger.logInfo(`\t-${teamspace}`); + + return incompleteModels.map(({ _id, status, timestamp }) => { + logger.logInfo(`\t\t${_id} - status: ${status}, timestamp: ${timestamp}`); + return { teamspace, model: _id, status, timestamp }; + }); +}; + +const run = async (teamspace, model, limit, notify) => { + if (model && !teamspace) { + throw new Error('Teamspace must be provided if model is defined'); + } + logger.logInfo(`Check processing flag(s) in ${teamspace ?? 'all teamspaces'}${model ? `.${model}` : ''}`); + + if (limit) { + TIME_LIMIT = limit * 60 * 60 * 1000; + } + + const teamspaces = teamspace ? [teamspace] : await getTeamspaceList(); + const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts, model)))).flat(); + + if (notify && results.length > 0) { + const data = { + err: JSON.stringify(results), + scope: Path.basename(__filename, Path.extname(__filename)), + title: 'Unexpected model status found', + message: `${results.length} unexpected status found`, + }; + await sendSystemEmail(emailTemplates.ERROR_NOTIFICATION.name, data); + } +}; + +const genYargs = /* istanbul ignore next */(yargs) => { + const commandName = Path.basename(__filename, Path.extname(__filename)); + const argsSpec = (subYargs) => subYargs.option('teamspace', { + describe: 'Target a specific teamspace (if unspecified, all teamspaces will be targetted)', + type: 'string', + }).option('model', { + describe: 'Target a specific model (if unspecified, all models will ba targetted)', + type: 'string', + }).option('limit', { + describe: 'Time limit (hours, default: 24) where models may still be processing', + type: 'number', + }).option('notify', { + describe: 'Send e-mail notification if results are found (default: false)', + type: 'boolean', + }); + return yargs.command(commandName, + 'Checks the processing state of a model.', + argsSpec, + (argv) => run(argv.teamspace, argv.model, argv.limit, argv.notify)); +}; + +module.exports = { + run, + genYargs, +}; diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js new file mode 100644 index 00000000000..9eb92c9c633 --- /dev/null +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -0,0 +1,135 @@ +/** + * Copyright (C) 2024 3D Repo Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const { + determineTestGroup, + db: { reset: resetDB, createModel }, + generateRandomString, + generateRandomModel, + generateRandomNumber, +} = require('../../helper/services'); +const { times } = require('lodash'); +const { utilScripts, src } = require('../../helper/path'); + +const { findModels } = require(`${src}/models/modelSettings`); +const { deleteIfUndefined } = require(`${src}/utils/helper/objects`); +const { disconnect } = require(`${src}/handler/db`); + +jest.mock('../../../../src/v5/services/mailer'); +const Mailer = require(`${src}/services/mailer`); + +const CheckProcessingFlags = require(`${utilScripts}/modelProcessing/checkProcessingFlags`); + +const modelStates = ['processing', 'queued']; + +const recentDate = new Date(); + +const setupData = () => { + const modelProms = times(2, async () => { + const teamspace = generateRandomString(); + const models = await Promise.all(times(modelStates.length, async (n) => { + const { _id, name, properties } = generateRandomModel({ + properties: deleteIfUndefined({ status: modelStates[n], timestamp: recentDate }), + }); + await createModel(teamspace, _id, name, properties); + return _id; + })); + + return { teamspace, models }; + }); + return Promise.all(modelProms); +}; + +const checkModelsStatus = async (teamspace, models) => { + const data = await findModels(teamspace, { _id: { $in: models } }, { _id: 1, status: 1, timestamp: 1 }); + + const expectedData = models.map((_id, ind) => ({ _id, timestamp: recentDate, status: modelStates[ind] })); + + expect(data.length).toBe(expectedData.length); + expect(data).toEqual(expect.arrayContaining(expectedData)); +}; + +const runTest = () => { + describe('Check Processing flags', () => { + let data; + beforeEach(async () => { + await resetDB(); + data = await setupData(); + }); + test('Should throw an error if model is provided but not teamspace', async () => { + const error = new Error('Teamspace must be provided if model is defined'); + await expect(CheckProcessingFlags.run(undefined, generateRandomString())).rejects.toEqual(error); + await Promise.all(data.map(({ teamspace, models }) => checkModelsStatus(teamspace, models))); + }); + + test('Should only process the predefined teamspace if it is defined', async () => { + await CheckProcessingFlags.run(data[0].teamspace); + await Promise.all(data.map( + ({ teamspace, models }) => checkModelsStatus(teamspace, models), + )); + }); + + test('Should only process the predefined model if it is defined', async () => { + await CheckProcessingFlags.run(data[0].teamspace, data[0].models[0]); + await Promise.all([ + ...data.map(({ teamspace, models }) => checkModelsStatus(teamspace, models)), + checkModelsStatus(data[0].teamspace, [data[0].models[0]]), + ]); + }); + + test('Should process all models if no parameters are provided', async () => { + await CheckProcessingFlags.run(); + await Promise.all(data.map( + ({ teamspace, models }) => checkModelsStatus(teamspace, models), + )); + }); + + test('Should do nothing if teamspace is not found', async () => { + await CheckProcessingFlags.run(generateRandomString()); + await Promise.all(data.map( + ({ teamspace, models }) => checkModelsStatus(teamspace, models), + )); + }); + + test('Should do nothing if model is not found', async () => { + await CheckProcessingFlags.run(data[0].teamspace, generateRandomString()); + await Promise.all(data.map( + ({ teamspace, models }) => checkModelsStatus(teamspace, models), + )); + }); + + test('Should set time limit if given', async () => { + await CheckProcessingFlags.run(undefined, undefined, generateRandomNumber()); + await Promise.all(data.map( + ({ teamspace, models }) => checkModelsStatus(teamspace, models), + )); + }); + + test('Should send notification if set and there are results', async () => { + await CheckProcessingFlags.run(undefined, undefined, undefined, true); + expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); + await Promise.all(data.map( + ({ teamspace, models }) => checkModelsStatus(teamspace, models), + )); + }); + }); +}; + +describe(determineTestGroup(__filename), () => { + runTest(); + afterAll(disconnect); +}); diff --git a/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx b/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx index c90f4606336..0cbc3079c20 100644 --- a/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx +++ b/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx @@ -20,6 +20,8 @@ import { FormattedMessage } from 'react-intl'; import { Tooltip } from '@mui/material'; import { Container, DownloadButton, DownloadIcon } from './revisionsListItem.styles'; import { RevisionsListItemButton } from './revisionsListItemButton/revisionsListItemButton.component'; +import { downloadAuthUrl } from '@components/authenticatedResource/authenticatedResource.hooks'; +import { formatDateTime } from '@/v5/helpers/intl.helper'; type IRevisionsListItem = { onSetVoidStatus: (voidStatus: boolean) => void; @@ -52,10 +54,14 @@ export const RevisionsListItem = ({ }; return ( - - {children} - - {hasPermission && ( + + {formatDateTime(timestamp)} + + {tag} + {desc} + {(format || '').toLowerCase()} + + { hasCollaboratorAccess && ( Date: Tue, 3 Sep 2024 16:18:01 +0100 Subject: [PATCH 02/15] ISSUE #5106 - revert merge error --- .../revisionsListItem.component.tsx | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx b/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx index 0cbc3079c20..c90f4606336 100644 --- a/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx +++ b/frontend/src/v5/ui/components/shared/revisionDetails/components/revisionsListItem/revisionsListItem.component.tsx @@ -20,8 +20,6 @@ import { FormattedMessage } from 'react-intl'; import { Tooltip } from '@mui/material'; import { Container, DownloadButton, DownloadIcon } from './revisionsListItem.styles'; import { RevisionsListItemButton } from './revisionsListItemButton/revisionsListItemButton.component'; -import { downloadAuthUrl } from '@components/authenticatedResource/authenticatedResource.hooks'; -import { formatDateTime } from '@/v5/helpers/intl.helper'; type IRevisionsListItem = { onSetVoidStatus: (voidStatus: boolean) => void; @@ -54,14 +52,10 @@ export const RevisionsListItem = ({ }; return ( - - {formatDateTime(timestamp)} - - {tag} - {desc} - {(format || '').toLowerCase()} - - { hasCollaboratorAccess && ( + + {children} + + {hasPermission && ( Date: Mon, 9 Sep 2024 18:40:19 +0100 Subject: [PATCH 03/15] ISSUE #5106 - detect zombie processing drawings, zombie mailer tmpl --- .../modelProcessing/detectZombieProcessing.js | 60 +++++++++---------- .../v5/services/mailer/mailer.constants.js | 2 + .../html/zombieProcessingStatuses.html | 6 ++ .../templates/zombieProcessingStatuses.js | 41 +++++++++++++ 4 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 backend/src/v5/services/mailer/templates/html/zombieProcessingStatuses.html create mode 100644 backend/src/v5/services/mailer/templates/zombieProcessingStatuses.js diff --git a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js index c2db90c690d..53353ebe887 100644 --- a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js +++ b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js @@ -16,10 +16,9 @@ */ /** - * This script is used to check the processing state of model(s). - * Model status should be 'ok' or 'failed' unless it is either 'queued' or 'processing'. - * Models that have a 'queued' or 'processing' status while not present in the queued - * have in a bad state and resetProcessingFlag can be used to reset it. + * This script is used to check the processing status of models/drawings. + * Processing status should be 'ok' or 'failed'. + * The utility script `resetProcessingFlag` can be used to reset zombie statuses for models. */ const { v5Path } = require('../../../interop'); @@ -28,48 +27,52 @@ const { logger } = require(`${v5Path}/utils/logger`); const { getTeamspaceList } = require('../../utils'); const { find } = require(`${v5Path}/handler/db`); -const { SETTINGS_COL } = require(`${v5Path}/models/modelSettings.constants`); +const { SETTINGS_COL, processStatuses } = require(`${v5Path}/models/modelSettings.constants`); +const { DRAWINGS_HISTORY_COL } = require(`${v5Path}/models/revisions.constants`); const { sendSystemEmail } = require(`${v5Path}/services/mailer`); const { templates: emailTemplates } = require(`${v5Path}/services/mailer/mailer.constants`); +const { UUIDToString } = require(`${v5Path}/utils/helper/uuids`); const Path = require('path'); let TIME_LIMIT = 24 * 60 * 60 * 1000; // hours * 1 hour in ms -const processTeamspace = async (teamspace, model) => { +const processTeamspace = async (teamspace) => { const expiredTimestamp = new Date((new Date()) - TIME_LIMIT); - const incompleteStatusQuery = { $or: [{ status: 'processing' }, { status: 'queued' }], timestamp: { $lt: expiredTimestamp } }; - const query = model ? { ...incompleteStatusQuery, _id: model } : incompleteStatusQuery; + const zombieQuery = { + status: { $exists: true, $not: { $regex: `(${processStatuses.OK})|(${processStatuses.FAILED})` } }, + timestamp: { $lt: expiredTimestamp }, + }; - const incompleteModels = await find(teamspace, SETTINGS_COL, query, { status: 1, timestamp: 1 }); logger.logInfo(`\t-${teamspace}`); - return incompleteModels.map(({ _id, status, timestamp }) => { - logger.logInfo(`\t\t${_id} - status: ${status}, timestamp: ${timestamp}`); - return { teamspace, model: _id, status, timestamp }; - }); + const zombieModels = await find(teamspace, SETTINGS_COL, zombieQuery, { status: 1, timestamp: 1 }); + const zombieDrawings = await find(teamspace, DRAWINGS_HISTORY_COL, zombieQuery, { status: 1, timestamp: 1 }); + + return [ + ...await Promise.all(zombieModels.map(({ _id, status, timestamp }) => `${teamspace}, model, ${_id}, ${status}, ${timestamp}`)), + ...await Promise.all(zombieDrawings.map(({ _id, status, timestamp }) => `${teamspace}, drawing, ${UUIDToString(_id)}, ${status}, ${timestamp}`)), + ]; }; -const run = async (teamspace, model, limit, notify) => { - if (model && !teamspace) { - throw new Error('Teamspace must be provided if model is defined'); - } - logger.logInfo(`Check processing flag(s) in ${teamspace ?? 'all teamspaces'}${model ? `.${model}` : ''}`); +const run = async (teamspace, limit, notify) => { + logger.logInfo(`Check processing flag(s) in ${teamspace ?? 'all teamspaces'}`); if (limit) { TIME_LIMIT = limit * 60 * 60 * 1000; } const teamspaces = teamspace ? [teamspace] : await getTeamspaceList(); - const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts, model)))).flat(); + const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts)))).flat(); if (notify && results.length > 0) { + logger.logInfo(`Zombie processing statuses found: ${results.length}`); const data = { - err: JSON.stringify(results), - scope: Path.basename(__filename, Path.extname(__filename)), - title: 'Unexpected model status found', - message: `${results.length} unexpected status found`, + script: Path.basename(__filename, Path.extname(__filename)), + title: 'Zombie processing statuses found', + message: `${results.length} zombie processing statuses found`, + logExcerpt: JSON.stringify(results), }; - await sendSystemEmail(emailTemplates.ERROR_NOTIFICATION.name, data); + await sendSystemEmail(emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, data); } }; @@ -78,20 +81,17 @@ const genYargs = /* istanbul ignore next */(yargs) => { const argsSpec = (subYargs) => subYargs.option('teamspace', { describe: 'Target a specific teamspace (if unspecified, all teamspaces will be targetted)', type: 'string', - }).option('model', { - describe: 'Target a specific model (if unspecified, all models will ba targetted)', - type: 'string', }).option('limit', { - describe: 'Time limit (hours, default: 24) where models may still be processing', + describe: 'Time limit (hours, default: 24) where models/drawings may still be processing', type: 'number', }).option('notify', { describe: 'Send e-mail notification if results are found (default: false)', type: 'boolean', }); return yargs.command(commandName, - 'Checks the processing state of a model.', + 'Checks the processing status of models/drawings.', argsSpec, - (argv) => run(argv.teamspace, argv.model, argv.limit, argv.notify)); + (argv) => run(argv.teamspace, argv.limit, argv.notify)); }; module.exports = { diff --git a/backend/src/v5/services/mailer/mailer.constants.js b/backend/src/v5/services/mailer/mailer.constants.js index 910b3ed6d74..aae01705397 100644 --- a/backend/src/v5/services/mailer/mailer.constants.js +++ b/backend/src/v5/services/mailer/mailer.constants.js @@ -21,6 +21,7 @@ const forgotPasswordSso = require('./templates/forgotPasswordSSO'); const modelImportError = require('./templates/modelImportError'); const { toConstantCase } = require('../../utils/helper/strings'); const verifyUser = require('./templates/verifyUser'); +const zombieProcessingStatuses = require('./templates/zombieProcessingStatuses'); const MailerConstants = {}; @@ -30,6 +31,7 @@ const templates = { forgotPasswordSso, errorNotification, modelImportError, + zombieProcessingStatuses, }; MailerConstants.templates = {}; diff --git a/backend/src/v5/services/mailer/templates/html/zombieProcessingStatuses.html b/backend/src/v5/services/mailer/templates/html/zombieProcessingStatuses.html new file mode 100644 index 00000000000..b7f74cb677e --- /dev/null +++ b/backend/src/v5/services/mailer/templates/html/zombieProcessingStatuses.html @@ -0,0 +1,6 @@ +

<%= message %> on <%= domain %>

+

Log:

+

+ <%-logExcerpt-%> +

+ diff --git a/backend/src/v5/services/mailer/templates/zombieProcessingStatuses.js b/backend/src/v5/services/mailer/templates/zombieProcessingStatuses.js new file mode 100644 index 00000000000..00f413a49e0 --- /dev/null +++ b/backend/src/v5/services/mailer/templates/zombieProcessingStatuses.js @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2024 3D Repo Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const Yup = require('yup'); +const config = require('../../../utils/config'); +const { generateTemplateFn } = require('./common'); + +const TEMPLATE_PATH = `${__dirname}/html/zombieProcessingStatuses.html`; + +const dataSchema = Yup.object({ + script: Yup.string().default(''), + title: Yup.string().default('Zombie Processing Statuses'), + domain: Yup.string().default(() => config.getBaseURL()), + logExcerpt: Yup.string().default('No logs found.').transform( + (val) => val.replaceAll('<', '<').replaceAll('>', '>').replace(/(\r\n|\n|\r)/gm, '
')), + +}).required(true); + +const ZombieProcessingStatusesTemplate = {}; +ZombieProcessingStatusesTemplate.subject = (data) => { + const { domain, title, script } = dataSchema.cast(data); + return `[${domain}][${script}] ${title}`; +}; + +ZombieProcessingStatusesTemplate.html = generateTemplateFn(dataSchema, TEMPLATE_PATH); + +module.exports = ZombieProcessingStatusesTemplate; From 2ee3855b0dcab471c1ac94234a96f774b2958aed Mon Sep 17 00:00:00 2001 From: Charence Date: Tue, 17 Sep 2024 15:35:19 +0100 Subject: [PATCH 04/15] #5106 - drawing zombie tests --- backend/tests/v5/helper/services.js | 6 +- .../detectZombieProcessing.test.js | 107 +++++++++--------- 2 files changed, 58 insertions(+), 55 deletions(-) diff --git a/backend/tests/v5/helper/services.js b/backend/tests/v5/helper/services.js index d05b34c0cc0..6721d21cae5 100644 --- a/backend/tests/v5/helper/services.js +++ b/backend/tests/v5/helper/services.js @@ -457,7 +457,7 @@ ServiceHelper.generateRandomModel = ({ modelType = modelTypes.CONTAINER, viewers }; }; -ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType) => { +ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType, status) => { const _id = ServiceHelper.generateUUIDString(); const entry = deleteIfUndefined({ _id, @@ -471,6 +471,10 @@ ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType void: !!isVoid, }); + if (status) { + entry.status = status; + } + if (hasFile) { entry.rFile = modelType === modelTypes.DRAWING ? [ServiceHelper.generateUUIDString()] : [`${_id}_${ServiceHelper.generateRandomString()}_ifc`]; entry.refData = ServiceHelper.generateRandomString(); diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js index 9eb92c9c633..42fdf24233c 100644 --- a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -17,28 +17,34 @@ const { determineTestGroup, - db: { reset: resetDB, createModel }, + db: { reset: resetDB, createModel, createRevision }, generateRandomString, generateRandomModel, - generateRandomNumber, + generateRevisionEntry, } = require('../../helper/services'); const { times } = require('lodash'); const { utilScripts, src } = require('../../helper/path'); -const { findModels } = require(`${src}/models/modelSettings`); +// const { findModels } = require(`${src}/models/modelSettings`); +const { modelTypes, processStatuses } = require(`${src}/models/modelSettings.constants`); const { deleteIfUndefined } = require(`${src}/utils/helper/objects`); const { disconnect } = require(`${src}/handler/db`); +// const { templates: emailTemplates } = require(`${src}/services/mailer/mailer.constants`); jest.mock('../../../../src/v5/services/mailer'); const Mailer = require(`${src}/services/mailer`); -const CheckProcessingFlags = require(`${utilScripts}/modelProcessing/checkProcessingFlags`); +// const Drawings = require(`${src}/processors/teamspaces/projects/models/drawings`); -const modelStates = ['processing', 'queued']; +const DetectZombieProcessing = require(`${utilScripts}/modelProcessing/detectZombieProcessing`); +// const Path = require('path'); -const recentDate = new Date(); +const modelStates = Object.values(processStatuses); // ['processing', 'queued']; + +const recentDate = new Date((new Date()) - 36 * 60 * 60 * 1000); const setupData = () => { + // console.log(processStatuses); const modelProms = times(2, async () => { const teamspace = generateRandomString(); const models = await Promise.all(times(modelStates.length, async (n) => { @@ -48,83 +54,76 @@ const setupData = () => { await createModel(teamspace, _id, name, properties); return _id; })); + const drawings = await Promise.all(times(modelStates.length, async (n) => { + const project = generateRandomString(); + const revision = generateRevisionEntry(false, false, modelTypes.DRAWING, modelStates[n]); + await createRevision(teamspace, project, revision._id, revision, modelTypes.DRAWING); + return revision._id; + })); - return { teamspace, models }; + return { teamspace, models, drawings }; }); return Promise.all(modelProms); }; +/* const checkModelsStatus = async (teamspace, models) => { const data = await findModels(teamspace, { _id: { $in: models } }, { _id: 1, status: 1, timestamp: 1 }); - const expectedData = models.map((_id, ind) => ({ _id, timestamp: recentDate, status: modelStates[ind] })); + // const expectedData = models.map((_id, ind) => ({ _id, timestamp: recentDate, status: modelStates[ind] })); + // console.log(expectedData); expect(data.length).toBe(expectedData.length); expect(data).toEqual(expect.arrayContaining(expectedData)); }; +*/ const runTest = () => { - describe('Check Processing flags', () => { + describe('Detect zombie processing', () => { let data; beforeEach(async () => { await resetDB(); data = await setupData(); - }); - test('Should throw an error if model is provided but not teamspace', async () => { - const error = new Error('Teamspace must be provided if model is defined'); - await expect(CheckProcessingFlags.run(undefined, generateRandomString())).rejects.toEqual(error); - await Promise.all(data.map(({ teamspace, models }) => checkModelsStatus(teamspace, models))); + // console.log(data); + // "ZOMBIE_PROCESSING_STATUSES", {"logExcerpt": "[\"73d9b3813003ec9a1298, model, 4e0a04f7-a51d-4b75-9f4e-18889503a7e2, uploading, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 6fc908ae-4efa-4d45-b037-b12c21c36b64, uploaded, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 97c834ac-539b-42d6-b9be-7c353e342398, queued, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 1f0ec23c-8ba1-40e1-9853-4413264c225a, processing, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, c5d5b146-ec98-4fb9-9be8-26c7018dc99f, Generating Bundles, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 946d5de8-1f0b-416d-ad35-da78a419eb9a, Queued for Unity, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 98769cea-a9d4-4a0d-98c3-440b6a3bf4f9, uploading, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, e17af882-f661-4e7c-b6a5-f99e70fe0f68, uploaded, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 8ac80ea9-b616-43c6-aad6-259043b4dda9, queued, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 68843449-065e-4f21-af33-5b2508924752, processing, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 55c6e54d-0407-4488-8dd5-04e9ded63a68, Generating Bundles, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 661dbaf4-fa88-46d6-bfbd-c97e609e1ab9, Queued for Unity, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\"]", "message": "12 zombie processing statuses found", "script": "detectZombieProcessing", "title": "Zombie processing statuses found"} }); - test('Should only process the predefined teamspace if it is defined', async () => { - await CheckProcessingFlags.run(data[0].teamspace); - await Promise.all(data.map( - ({ teamspace, models }) => checkModelsStatus(teamspace, models), - )); + test('Should do nothing if notify is not set', async () => { + await DetectZombieProcessing.run(); + expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(0); }); - test('Should only process the predefined model if it is defined', async () => { - await CheckProcessingFlags.run(data[0].teamspace, data[0].models[0]); - await Promise.all([ - ...data.map(({ teamspace, models }) => checkModelsStatus(teamspace, models)), - checkModelsStatus(data[0].teamspace, [data[0].models[0]]), - ]); + test('Should send system mail if notify is set', async () => { + await DetectZombieProcessing.run(undefined, undefined, true); + expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); + /* + const expectedResults = []; + const expectedData = { + script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), + title: 'Zombie processing statuses found', + message: `${expectedResults.length} zombie processing statuses found`, + logExcerpt: JSON.stringify(expectedResults), + }; + expect(Mailer.sendSystemEmail).toHaveBeenCalledWith( + emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, + expectedData, + ); + */ }); - test('Should process all models if no parameters are provided', async () => { - await CheckProcessingFlags.run(); - await Promise.all(data.map( - ({ teamspace, models }) => checkModelsStatus(teamspace, models), - )); + test('Should send system mail if the predefined teamspace exists', async () => { + await DetectZombieProcessing.run(data[0].teamspace, undefined, true); + expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); }); test('Should do nothing if teamspace is not found', async () => { - await CheckProcessingFlags.run(generateRandomString()); - await Promise.all(data.map( - ({ teamspace, models }) => checkModelsStatus(teamspace, models), - )); + await DetectZombieProcessing.run(generateRandomString(), undefined, true); + expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(0); }); - test('Should do nothing if model is not found', async () => { - await CheckProcessingFlags.run(data[0].teamspace, generateRandomString()); - await Promise.all(data.map( - ({ teamspace, models }) => checkModelsStatus(teamspace, models), - )); - }); - - test('Should set time limit if given', async () => { - await CheckProcessingFlags.run(undefined, undefined, generateRandomNumber()); - await Promise.all(data.map( - ({ teamspace, models }) => checkModelsStatus(teamspace, models), - )); - }); - - test('Should send notification if set and there are results', async () => { - await CheckProcessingFlags.run(undefined, undefined, undefined, true); - expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); - await Promise.all(data.map( - ({ teamspace, models }) => checkModelsStatus(teamspace, models), - )); + test('Should do nothing if time limit is extended', async () => { + await DetectZombieProcessing.run(undefined, 48 * 60 * 60 * 1000, true); + expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(0); }); }); }; From 82a52e41045c8ac39fa513ec7a79f835596fe182 Mon Sep 17 00:00:00 2001 From: Charence Date: Wed, 18 Sep 2024 09:15:23 +0100 Subject: [PATCH 05/15] ISSUE #5106 - update zombie test --- backend/tests/v5/helper/services.js | 4 +-- .../detectZombieProcessing.test.js | 32 ++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/backend/tests/v5/helper/services.js b/backend/tests/v5/helper/services.js index 6721d21cae5..5ed9cbbb911 100644 --- a/backend/tests/v5/helper/services.js +++ b/backend/tests/v5/helper/services.js @@ -457,7 +457,7 @@ ServiceHelper.generateRandomModel = ({ modelType = modelTypes.CONTAINER, viewers }; }; -ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType, status) => { +ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType, timestamp, status) => { const _id = ServiceHelper.generateUUIDString(); const entry = deleteIfUndefined({ _id, @@ -466,7 +466,7 @@ ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType revCode: modelType === modelTypes.DRAWING ? ServiceHelper.generateRandomString(10) : undefined, format: modelType === modelTypes.DRAWING ? '.pdf' : undefined, author: ServiceHelper.generateRandomString(), - timestamp: ServiceHelper.generateRandomDate(), + timestamp: timestamp || ServiceHelper.generateRandomDate(), desc: ServiceHelper.generateRandomString(), void: !!isVoid, }); diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js index 42fdf24233c..ede66d42d5f 100644 --- a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -29,7 +29,7 @@ const { utilScripts, src } = require('../../helper/path'); const { modelTypes, processStatuses } = require(`${src}/models/modelSettings.constants`); const { deleteIfUndefined } = require(`${src}/utils/helper/objects`); const { disconnect } = require(`${src}/handler/db`); -// const { templates: emailTemplates } = require(`${src}/services/mailer/mailer.constants`); +const { templates: emailTemplates } = require(`${src}/services/mailer/mailer.constants`); jest.mock('../../../../src/v5/services/mailer'); const Mailer = require(`${src}/services/mailer`); @@ -37,14 +37,13 @@ const Mailer = require(`${src}/services/mailer`); // const Drawings = require(`${src}/processors/teamspaces/projects/models/drawings`); const DetectZombieProcessing = require(`${utilScripts}/modelProcessing/detectZombieProcessing`); -// const Path = require('path'); +const Path = require('path'); const modelStates = Object.values(processStatuses); // ['processing', 'queued']; const recentDate = new Date((new Date()) - 36 * 60 * 60 * 1000); const setupData = () => { - // console.log(processStatuses); const modelProms = times(2, async () => { const teamspace = generateRandomString(); const models = await Promise.all(times(modelStates.length, async (n) => { @@ -52,13 +51,13 @@ const setupData = () => { properties: deleteIfUndefined({ status: modelStates[n], timestamp: recentDate }), }); await createModel(teamspace, _id, name, properties); - return _id; + return { model: _id, status: modelStates[n] }; })); const drawings = await Promise.all(times(modelStates.length, async (n) => { const project = generateRandomString(); - const revision = generateRevisionEntry(false, false, modelTypes.DRAWING, modelStates[n]); + const revision = generateRevisionEntry(false, false, modelTypes.DRAWING, recentDate, modelStates[n]); await createRevision(teamspace, project, revision._id, revision, modelTypes.DRAWING); - return revision._id; + return { drawing: revision._id, status: modelStates[n] }; })); return { teamspace, models, drawings }; @@ -72,7 +71,6 @@ const checkModelsStatus = async (teamspace, models) => { // const expectedData = models.map((_id, ind) => ({ _id, timestamp: recentDate, status: modelStates[ind] })); - // console.log(expectedData); expect(data.length).toBe(expectedData.length); expect(data).toEqual(expect.arrayContaining(expectedData)); }; @@ -84,8 +82,6 @@ const runTest = () => { beforeEach(async () => { await resetDB(); data = await setupData(); - // console.log(data); - // "ZOMBIE_PROCESSING_STATUSES", {"logExcerpt": "[\"73d9b3813003ec9a1298, model, 4e0a04f7-a51d-4b75-9f4e-18889503a7e2, uploading, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 6fc908ae-4efa-4d45-b037-b12c21c36b64, uploaded, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 97c834ac-539b-42d6-b9be-7c353e342398, queued, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 1f0ec23c-8ba1-40e1-9853-4413264c225a, processing, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, c5d5b146-ec98-4fb9-9be8-26c7018dc99f, Generating Bundles, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"73d9b3813003ec9a1298, model, 946d5de8-1f0b-416d-ad35-da78a419eb9a, Queued for Unity, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 98769cea-a9d4-4a0d-98c3-440b6a3bf4f9, uploading, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, e17af882-f661-4e7c-b6a5-f99e70fe0f68, uploaded, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 8ac80ea9-b616-43c6-aad6-259043b4dda9, queued, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 68843449-065e-4f21-af33-5b2508924752, processing, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 55c6e54d-0407-4488-8dd5-04e9ded63a68, Generating Bundles, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\",\"fcba2b9c606c4d47139d, model, 661dbaf4-fa88-46d6-bfbd-c97e609e1ab9, Queued for Unity, Tue Sep 10 2024 06:47:46 GMT+0100 (British Summer Time)\"]", "message": "12 zombie processing statuses found", "script": "detectZombieProcessing", "title": "Zombie processing statuses found"} }); test('Should do nothing if notify is not set', async () => { @@ -96,8 +92,23 @@ const runTest = () => { test('Should send system mail if notify is set', async () => { await DetectZombieProcessing.run(undefined, undefined, true); expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); - /* const expectedResults = []; + data.forEach(({ teamspace, models, drawings }) => { + models.forEach(({ model, status }) => { + if (status !== processStatuses.OK && status !== processStatuses.FAILED) { + expectedResults.push( + `${teamspace}, model, ${model}, ${status}, ${recentDate}`, + ); + } + }); + drawings.forEach(({ drawing, status }) => { + if (status !== processStatuses.OK && status !== processStatuses.FAILED) { + expectedResults.push( + `${teamspace}, drawing, ${drawing}, ${status}, ${recentDate}`, + ); + } + }); + }); const expectedData = { script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), title: 'Zombie processing statuses found', @@ -108,7 +119,6 @@ const runTest = () => { emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, expectedData, ); - */ }); test('Should send system mail if the predefined teamspace exists', async () => { From 5edcab92c7250110fa1b65b726ac1ed75b6a6a45 Mon Sep 17 00:00:00 2001 From: Charence Date: Wed, 18 Sep 2024 09:24:09 +0100 Subject: [PATCH 06/15] ISSUE #5106 - check mail contents --- .../detectZombieProcessing.test.js | 65 +++++++++---------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js index ede66d42d5f..1ff55baac5a 100644 --- a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -65,16 +65,34 @@ const setupData = () => { return Promise.all(modelProms); }; -/* -const checkModelsStatus = async (teamspace, models) => { - const data = await findModels(teamspace, { _id: { $in: models } }, { _id: 1, status: 1, timestamp: 1 }); - - // const expectedData = models.map((_id, ind) => ({ _id, timestamp: recentDate, status: modelStates[ind] })); - - expect(data.length).toBe(expectedData.length); - expect(data).toEqual(expect.arrayContaining(expectedData)); +const checkMail = (data, filteredTeamspace) => { + const expectedResults = data.map(({ teamspace, models, drawings }) => { + if (!filteredTeamspace || teamspace === filteredTeamspace) { + const expectedModels = models.map(({ model, status }) => { + if (status !== processStatuses.OK && status !== processStatuses.FAILED) { + return `${teamspace}, model, ${model}, ${status}, ${recentDate}`; + } + }); + const expectedDrawings = drawings.map(({ drawing, status }) => { + if (status !== processStatuses.OK && status !== processStatuses.FAILED) { + return `${teamspace}, drawing, ${drawing}, ${status}, ${recentDate}`; + } + }); + return [ ...expectedModels, ...expectedDrawings ]; + } + }).flat().filter(Boolean); + console.log(expectedResults); + const expectedData = { + script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), + title: 'Zombie processing statuses found', + message: `${expectedResults.length} zombie processing statuses found`, + logExcerpt: JSON.stringify(expectedResults), + }; + expect(Mailer.sendSystemEmail).toHaveBeenCalledWith( + emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, + expectedData, + ); }; -*/ const runTest = () => { describe('Detect zombie processing', () => { @@ -92,38 +110,13 @@ const runTest = () => { test('Should send system mail if notify is set', async () => { await DetectZombieProcessing.run(undefined, undefined, true); expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); - const expectedResults = []; - data.forEach(({ teamspace, models, drawings }) => { - models.forEach(({ model, status }) => { - if (status !== processStatuses.OK && status !== processStatuses.FAILED) { - expectedResults.push( - `${teamspace}, model, ${model}, ${status}, ${recentDate}`, - ); - } - }); - drawings.forEach(({ drawing, status }) => { - if (status !== processStatuses.OK && status !== processStatuses.FAILED) { - expectedResults.push( - `${teamspace}, drawing, ${drawing}, ${status}, ${recentDate}`, - ); - } - }); - }); - const expectedData = { - script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), - title: 'Zombie processing statuses found', - message: `${expectedResults.length} zombie processing statuses found`, - logExcerpt: JSON.stringify(expectedResults), - }; - expect(Mailer.sendSystemEmail).toHaveBeenCalledWith( - emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, - expectedData, - ); + checkMail(data); }); test('Should send system mail if the predefined teamspace exists', async () => { await DetectZombieProcessing.run(data[0].teamspace, undefined, true); expect(Mailer.sendSystemEmail).toHaveBeenCalledTimes(1); + checkMail(data, data[0].teamspace); }); test('Should do nothing if teamspace is not found', async () => { From c28c406e99f5e4d627a21df05a29df8c4f263c25 Mon Sep 17 00:00:00 2001 From: Charence Date: Wed, 18 Sep 2024 09:33:11 +0100 Subject: [PATCH 07/15] ISSUE #5106 - satisfy lint --- .../detectZombieProcessing.test.js | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js index 1ff55baac5a..c266bb9851b 100644 --- a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -25,7 +25,6 @@ const { const { times } = require('lodash'); const { utilScripts, src } = require('../../helper/path'); -// const { findModels } = require(`${src}/models/modelSettings`); const { modelTypes, processStatuses } = require(`${src}/models/modelSettings.constants`); const { deleteIfUndefined } = require(`${src}/utils/helper/objects`); const { disconnect } = require(`${src}/handler/db`); @@ -34,12 +33,10 @@ const { templates: emailTemplates } = require(`${src}/services/mailer/mailer.con jest.mock('../../../../src/v5/services/mailer'); const Mailer = require(`${src}/services/mailer`); -// const Drawings = require(`${src}/processors/teamspaces/projects/models/drawings`); - const DetectZombieProcessing = require(`${utilScripts}/modelProcessing/detectZombieProcessing`); const Path = require('path'); -const modelStates = Object.values(processStatuses); // ['processing', 'queued']; +const modelStates = Object.values(processStatuses); const recentDate = new Date((new Date()) - 36 * 60 * 60 * 1000); @@ -68,20 +65,16 @@ const setupData = () => { const checkMail = (data, filteredTeamspace) => { const expectedResults = data.map(({ teamspace, models, drawings }) => { if (!filteredTeamspace || teamspace === filteredTeamspace) { - const expectedModels = models.map(({ model, status }) => { - if (status !== processStatuses.OK && status !== processStatuses.FAILED) { - return `${teamspace}, model, ${model}, ${status}, ${recentDate}`; - } - }); - const expectedDrawings = drawings.map(({ drawing, status }) => { - if (status !== processStatuses.OK && status !== processStatuses.FAILED) { - return `${teamspace}, drawing, ${drawing}, ${status}, ${recentDate}`; - } - }); - return [ ...expectedModels, ...expectedDrawings ]; + const expectedModels = models.map(({ model, status }) => ( + (status !== processStatuses.OK && status !== processStatuses.FAILED) + ? `${teamspace}, model, ${model}, ${status}, ${recentDate}` : '')); + const expectedDrawings = drawings.map(({ drawing, status }) => ( + (status !== processStatuses.OK && status !== processStatuses.FAILED) + ? `${teamspace}, drawing, ${drawing}, ${status}, ${recentDate}` : '')); + return [...expectedModels, ...expectedDrawings]; } + return undefined; }).flat().filter(Boolean); - console.log(expectedResults); const expectedData = { script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), title: 'Zombie processing statuses found', From 5eb2d3e18bcc393a6c38e6bab14521774750d937 Mon Sep 17 00:00:00 2001 From: Charence Date: Thu, 19 Sep 2024 14:02:31 +0100 Subject: [PATCH 08/15] ISSUE #5106 - add mailer tmpl test --- .../zombieProcessingStatuses.test.js | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 backend/tests/v5/unit/services/mailer/templates/zombieProcessingStatuses.test.js diff --git a/backend/tests/v5/unit/services/mailer/templates/zombieProcessingStatuses.test.js b/backend/tests/v5/unit/services/mailer/templates/zombieProcessingStatuses.test.js new file mode 100644 index 00000000000..9abc3524b8e --- /dev/null +++ b/backend/tests/v5/unit/services/mailer/templates/zombieProcessingStatuses.test.js @@ -0,0 +1,51 @@ +/** + * Copyright (C) 2024 3D Repo Ltd + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const { src } = require('../../../../helper/path'); +const { determineTestGroup, generateRandomString } = require('../../../../helper/services'); +const isHtml = require('is-html-content'); + +const ZombieProcessingStatuses = require(`${src}/services/mailer/templates/zombieProcessingStatuses`); + +const testHtml = () => { + describe('get template html', () => { + test('should get zombieProcessingStatuses template html', async () => { + const res = await ZombieProcessingStatuses.html({ + message: generateRandomString(), + domain: generateRandomString(), + logExcerpt: generateRandomString(), + }); + expect(isHtml(res)).toEqual(true); + }); + }); +}; + +const testSubject = () => { + describe.each([ + ['data object is empty', {}], + ['data object is not empty', { domain: generateRandomString(), title: generateRandomString(), script: generateRandomString() }], + ])('get subject', (desc, data) => { + test(`should succeed if ${desc}`, () => { + expect(ZombieProcessingStatuses.subject(data).length).not.toEqual(0); + }); + }); +}; + +describe(determineTestGroup(__filename), () => { + testHtml(); + testSubject(); +}); From a15e79678014a6b8b12400ed7479d0c145b36540 Mon Sep 17 00:00:00 2001 From: Charence Date: Fri, 20 Sep 2024 17:02:10 +0100 Subject: [PATCH 09/15] ISSUE #5106 - address feedback --- .../utility/modelProcessing/detectZombieProcessing.js | 6 +++--- backend/tests/v5/helper/services.js | 5 +---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js index 53353ebe887..fa085cd5945 100644 --- a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js +++ b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js @@ -37,7 +37,7 @@ const Path = require('path'); let TIME_LIMIT = 24 * 60 * 60 * 1000; // hours * 1 hour in ms const processTeamspace = async (teamspace) => { - const expiredTimestamp = new Date((new Date()) - TIME_LIMIT); + const expiredTimestamp = new Date(new Date() - TIME_LIMIT); const zombieQuery = { status: { $exists: true, $not: { $regex: `(${processStatuses.OK})|(${processStatuses.FAILED})` } }, timestamp: { $lt: expiredTimestamp }, @@ -49,8 +49,8 @@ const processTeamspace = async (teamspace) => { const zombieDrawings = await find(teamspace, DRAWINGS_HISTORY_COL, zombieQuery, { status: 1, timestamp: 1 }); return [ - ...await Promise.all(zombieModels.map(({ _id, status, timestamp }) => `${teamspace}, model, ${_id}, ${status}, ${timestamp}`)), - ...await Promise.all(zombieDrawings.map(({ _id, status, timestamp }) => `${teamspace}, drawing, ${UUIDToString(_id)}, ${status}, ${timestamp}`)), + ...zombieModels.map(({ _id, status, timestamp }) => `${teamspace}, model, ${_id}, ${status}, ${timestamp}`), + ...zombieDrawings.map(({ _id, status, timestamp }) => `${teamspace}, drawing, ${UUIDToString(_id)}, ${status}, ${timestamp}`), ]; }; diff --git a/backend/tests/v5/helper/services.js b/backend/tests/v5/helper/services.js index 5ed9cbbb911..a4167018b99 100644 --- a/backend/tests/v5/helper/services.js +++ b/backend/tests/v5/helper/services.js @@ -462,6 +462,7 @@ ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType const entry = deleteIfUndefined({ _id, tag: modelType === modelTypes.DRAWING ? undefined : ServiceHelper.generateRandomString(), + status, statusCode: modelType === modelTypes.DRAWING ? statusCodes[0].code : undefined, revCode: modelType === modelTypes.DRAWING ? ServiceHelper.generateRandomString(10) : undefined, format: modelType === modelTypes.DRAWING ? '.pdf' : undefined, @@ -471,10 +472,6 @@ ServiceHelper.generateRevisionEntry = (isVoid = false, hasFile = true, modelType void: !!isVoid, }); - if (status) { - entry.status = status; - } - if (hasFile) { entry.rFile = modelType === modelTypes.DRAWING ? [ServiceHelper.generateUUIDString()] : [`${_id}_${ServiceHelper.generateRandomString()}_ifc`]; entry.refData = ServiceHelper.generateRandomString(); From b01b6ab5f48e34430c24cf2f6b7395db4f3adc6d Mon Sep 17 00:00:00 2001 From: Charence Date: Mon, 23 Sep 2024 14:25:29 +0100 Subject: [PATCH 10/15] ISSUE #5106 - ensure ordering of results sorted for tests --- .../scripts/utility/modelProcessing/detectZombieProcessing.js | 2 +- .../v5/scripts/modelProcessing/detectZombieProcessing.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js index fa085cd5945..c2f38fbe23d 100644 --- a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js +++ b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js @@ -62,7 +62,7 @@ const run = async (teamspace, limit, notify) => { } const teamspaces = teamspace ? [teamspace] : await getTeamspaceList(); - const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts)))).flat(); + const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts)))).flat().sort(); if (notify && results.length > 0) { logger.logInfo(`Zombie processing statuses found: ${results.length}`); diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js index c266bb9851b..525dabfd5b8 100644 --- a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -74,7 +74,7 @@ const checkMail = (data, filteredTeamspace) => { return [...expectedModels, ...expectedDrawings]; } return undefined; - }).flat().filter(Boolean); + }).flat().filter(Boolean).sort(); const expectedData = { script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), title: 'Zombie processing statuses found', From 3a4227e1e0572ca4c54049e9dd301e21be376f98 Mon Sep 17 00:00:00 2001 From: Charence Date: Tue, 24 Sep 2024 12:14:47 +0100 Subject: [PATCH 11/15] ISSUE #5106 - rm sort for tests --- .../utility/modelProcessing/detectZombieProcessing.js | 2 +- .../modelProcessing/detectZombieProcessing.test.js | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js index c2f38fbe23d..fa085cd5945 100644 --- a/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js +++ b/backend/src/scripts/utility/modelProcessing/detectZombieProcessing.js @@ -62,7 +62,7 @@ const run = async (teamspace, limit, notify) => { } const teamspaces = teamspace ? [teamspace] : await getTeamspaceList(); - const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts)))).flat().sort(); + const results = (await Promise.all(teamspaces.map((ts) => processTeamspace(ts)))).flat(); if (notify && results.length > 0) { logger.logInfo(`Zombie processing statuses found: ${results.length}`); diff --git a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js index 525dabfd5b8..3aae102e81d 100644 --- a/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js +++ b/backend/tests/v5/scripts/modelProcessing/detectZombieProcessing.test.js @@ -63,7 +63,7 @@ const setupData = () => { }; const checkMail = (data, filteredTeamspace) => { - const expectedResults = data.map(({ teamspace, models, drawings }) => { + const expectedLogExcerpt = data.map(({ teamspace, models, drawings }) => { if (!filteredTeamspace || teamspace === filteredTeamspace) { const expectedModels = models.map(({ model, status }) => ( (status !== processStatuses.OK && status !== processStatuses.FAILED) @@ -74,17 +74,18 @@ const checkMail = (data, filteredTeamspace) => { return [...expectedModels, ...expectedDrawings]; } return undefined; - }).flat().filter(Boolean).sort(); + }).flat().filter(Boolean); const expectedData = { script: Path.basename(__filename, Path.extname(__filename)).replace(/\.test/, ''), title: 'Zombie processing statuses found', - message: `${expectedResults.length} zombie processing statuses found`, - logExcerpt: JSON.stringify(expectedResults), + message: `${expectedLogExcerpt.length} zombie processing statuses found`, }; expect(Mailer.sendSystemEmail).toHaveBeenCalledWith( emailTemplates.ZOMBIE_PROCESSING_STATUSES.name, - expectedData, + expect.objectContaining(expectedData), ); + const actualLogExcerpt = JSON.parse(Mailer.sendSystemEmail.mock.calls[0][1].logExcerpt); + expect(actualLogExcerpt).toEqual(expect.arrayContaining(expectedLogExcerpt)); }; const runTest = () => { From a0735d38264ffb570e2aa240c1ed0f41b7569b38 Mon Sep 17 00:00:00 2001 From: Carmen Fan Date: Wed, 25 Sep 2024 13:00:40 +0100 Subject: [PATCH 12/15] ISSUE #5177 write the file path with placeholder instead of the full path --- backend/src/v5/services/modelProcessing.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/v5/services/modelProcessing.js b/backend/src/v5/services/modelProcessing.js index 8ef211a004c..78dbe21a929 100644 --- a/backend/src/v5/services/modelProcessing.js +++ b/backend/src/v5/services/modelProcessing.js @@ -79,7 +79,7 @@ const queueDrawingUpload = async (teamspace, project, model, revId, data, fileBu database: teamspace, project: model, revId, - file, + file: `${SHARED_SPACE_TAG}/${Path.join(revId, `${revId}${data.format}`)}`, }; await mkdir(pathToRevFolder); @@ -98,6 +98,7 @@ const queueDrawingUpload = async (teamspace, project, model, revId, data, fileBu publish(events.QUEUED_TASK_UPDATE, { teamspace, model, corId: revId, status: processStatuses.QUEUED }); } catch (err) { + console.log(err); logger.logError('Failed to queue drawing task', err.message); publish(events.QUEUED_TASK_COMPLETED, { teamspace, model, corId: revId, value: 4 }); } From 41623ff0e57e5c4d5006762d747842a90d5fb393 Mon Sep 17 00:00:00 2001 From: Carmen Fan Date: Wed, 25 Sep 2024 13:12:22 +0100 Subject: [PATCH 13/15] ISSUE #5177 - remove console log --- backend/src/v5/services/modelProcessing.js | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/v5/services/modelProcessing.js b/backend/src/v5/services/modelProcessing.js index 78dbe21a929..c790645c7f3 100644 --- a/backend/src/v5/services/modelProcessing.js +++ b/backend/src/v5/services/modelProcessing.js @@ -98,7 +98,6 @@ const queueDrawingUpload = async (teamspace, project, model, revId, data, fileBu publish(events.QUEUED_TASK_UPDATE, { teamspace, model, corId: revId, status: processStatuses.QUEUED }); } catch (err) { - console.log(err); logger.logError('Failed to queue drawing task', err.message); publish(events.QUEUED_TASK_COMPLETED, { teamspace, model, corId: revId, value: 4 }); } From 19643d4703ed7b6c4295794753bd1545d63dada1 Mon Sep 17 00:00:00 2001 From: Carmen Fan Date: Wed, 25 Sep 2024 14:30:59 +0100 Subject: [PATCH 14/15] ISSUE #5177 do not use Path.join as it is generating backward slashes on windows --- backend/src/v5/services/modelProcessing.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/v5/services/modelProcessing.js b/backend/src/v5/services/modelProcessing.js index c790645c7f3..807cccbaf6f 100644 --- a/backend/src/v5/services/modelProcessing.js +++ b/backend/src/v5/services/modelProcessing.js @@ -79,7 +79,7 @@ const queueDrawingUpload = async (teamspace, project, model, revId, data, fileBu database: teamspace, project: model, revId, - file: `${SHARED_SPACE_TAG}/${Path.join(revId, `${revId}${data.format}`)}`, + file: `${SHARED_SPACE_TAG}/${revId}/${revId}${data.format}`, }; await mkdir(pathToRevFolder); From 76f35eb7aab82e668e573326bffbb97f73815486 Mon Sep 17 00:00:00 2001 From: ChristosTsiotsias <71635617+ChristosTsiotsias@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:23:13 +0100 Subject: [PATCH 15/15] ISSUE #5163 address feedback --- backend/src/v5/utils/helper/units.js | 5 +- .../tests/v5/unit/utils/helper/units.test.js | 67 ++++++++++++------- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/backend/src/v5/utils/helper/units.js b/backend/src/v5/utils/helper/units.js index 277303f626c..a9e284205aa 100644 --- a/backend/src/v5/utils/helper/units.js +++ b/backend/src/v5/utils/helper/units.js @@ -30,12 +30,13 @@ const UNITS_CONVERSION_FACTORS_TO_METRES = { UnitsHelper.convertArrayUnits = (array, fromUnit, toUnit) => { const fromFactor = UNITS_CONVERSION_FACTORS_TO_METRES[fromUnit]; const toFactor = UNITS_CONVERSION_FACTORS_TO_METRES[toUnit]; + const scale = toFactor / fromFactor; if (!array.every(isNumber) || !fromFactor || !toFactor) { - return null; + return array; } - return array.map((n) => (n / fromFactor) * toFactor); + return array.map((n) => n * scale); }; module.exports = UnitsHelper; diff --git a/backend/tests/v5/unit/utils/helper/units.test.js b/backend/tests/v5/unit/utils/helper/units.test.js index a05c09457ef..a050c6cdd95 100644 --- a/backend/tests/v5/unit/utils/helper/units.test.js +++ b/backend/tests/v5/unit/utils/helper/units.test.js @@ -20,36 +20,51 @@ const { generateRandomString } = require('../../../helper/services'); const UnitsHelper = require(`${src}/utils/helper/units`); +const UNITS_CONVERSION_FACTORS_TO_METRES = { + m: 1, + dm: 10, + cm: 100, + mm: 1000, + ft: 3.28084, +}; + +const getScaleFactor = (fromUnit, toUnit) => { + const fromFactor = UNITS_CONVERSION_FACTORS_TO_METRES[fromUnit]; + const toFactor = UNITS_CONVERSION_FACTORS_TO_METRES[toUnit]; + return toFactor / fromFactor; +}; + const testConvertArrayUnits = () => { describe.each([ - ['invalid fromUnit', [1, 5], generateRandomString(), 'm'], - ['invalid toUnit', [1, 5], 'm', generateRandomString()], - ['array with non numbers', [generateRandomString(), 5], 'm', 'mm'], - ['m to dm', [1, 5], 'm', 'dm', [10, 50]], - ['m to cm', [1, 5], 'm', 'cm', [100, 500]], - ['m to mm', [1, 5], 'm', 'mm', [1000, 5000]], - ['m to ft', [1, 5], 'm', 'ft', [3.281, 16.404]], - ['dm to m', [1, 5], 'dm', 'm', [0.1, 0.5]], - ['dm to cm', [1, 5], 'dm', 'cm', [10, 50]], - ['dm to mm', [1, 5], 'dm', 'mm', [100, 500]], - ['dm to ft', [1, 5], 'dm', 'ft', [0.328, 1.640]], - ['mm to dm', [1, 5], 'mm', 'dm', [0.01, 0.05]], - ['mm to cm', [1, 5], 'mm', 'cm', [0.1, 0.5]], - ['mm to m', [1, 5], 'mm', 'm', [0.001, 0.005]], - ['mm to ft', [1, 5], 'mm', 'ft', [0.003, 0.016]], - ['ft to dm', [1, 5], 'ft', 'dm', [3.048, 15.24]], - ['ft to cm', [1, 5], 'ft', 'cm', [30.48, 152.4]], - ['ft to m', [1, 5], 'ft', 'm', [0.305, 1.524]], - ['ft to mm', [1, 5], 'ft', 'mm', [304.8, 1524]], - ])('Convert array units', (description, array, fromUnit, toUnit, result = null) => { - test(`with ${description} should return ${result}`, () => { - let res = UnitsHelper.convertArrayUnits(array, fromUnit, toUnit); + ['invalid fromUnit', [1, 5], generateRandomString(), 'm', true], + ['invalid toUnit', [1, 5], 'm', generateRandomString(), true], + ['array with non numbers', [generateRandomString(), 5], 'm', 'mm', true], + ['m to dm', [1, 5], 'm', 'dm'], + ['m to cm', [1, 5], 'm', 'cm'], + ['m to mm', [1, 5], 'm', 'mm'], + ['m to ft', [1, 5], 'm', 'ft'], + ['dm to m', [1, 5], 'dm', 'm'], + ['dm to cm', [1, 5], 'dm', 'cm'], + ['dm to mm', [1, 5], 'dm', 'mm'], + ['dm to ft', [1, 5], 'dm', 'ft'], + ['mm to dm', [1, 5], 'mm', 'dm'], + ['mm to cm', [1, 5], 'mm', 'cm'], + ['mm to m', [1, 5], 'mm', 'm'], + ['mm to ft', [1, 5], 'mm', 'ft'], + ['ft to dm', [1, 5], 'ft', 'dm'], + ['ft to cm', [1, 5], 'ft', 'cm'], + ['ft to m', [1, 5], 'ft', 'm'], + ['ft to mm', [1, 5], 'ft', 'mm'], + ])('Convert array units', (description, array, fromUnit, toUnit, invalidInput) => { + test(`with ${description} should return ${invalidInput ? 'the same array' : 'the converted array'}`, () => { + const res = UnitsHelper.convertArrayUnits(array, fromUnit, toUnit); - if (res) { - res = res.map((r) => Math.round(r * 1000) / 1000); + if (invalidInput) { + expect(res).toEqual(array); + } else { + const scaleFactor = getScaleFactor(fromUnit, toUnit); + expect(res).toEqual(array.map((n) => n * scaleFactor)); } - - expect(res).toEqual(result); }); }); };