diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dfaf3c86..61032e39 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,8 +1,67 @@ +# This file is managed by the repo-content-updater project. Manual changes here will result in a PR to bring back +# inline with the upstream template, unless you remove the dependabot managed file property from the repo + version: 2 updates: + - package-ecosystem: "gomod" + directory: / + schedule: + interval: "weekly" + day: "tuesday" + open-pull-requests-limit: 10 + labels: + - dependencies + - go + - "Changed" + reviewers: ["cmmarslender", "starttoaster"] + groups: + global: + patterns: + - "*" + + - package-ecosystem: "pip" + directory: / + schedule: + interval: "weekly" + day: "tuesday" + open-pull-requests-limit: 10 + labels: + - dependencies + - python + - "Changed" + reviewers: ["emlowe", "altendky"] + + - package-ecosystem: "github-actions" + directory: / + schedule: + interval: "weekly" + day: "tuesday" + open-pull-requests-limit: 10 + labels: + - dependencies + - github_actions + - "Changed" + reviewers: ["cmmarslender", "Starttoaster", "pmaslana"] + - package-ecosystem: "npm" - directory: "/" + directory: / + schedule: + interval: "weekly" + day: "tuesday" + open-pull-requests-limit: 10 + labels: + - dependencies + - javascript + - "Changed" + reviewers: ["cmmarslender", "emlowe"] + + - package-ecosystem: cargo + directory: / schedule: interval: "weekly" - day: "sunday" - target-branch: "develop" + day: "tuesday" + open-pull-requests-limit: 10 + labels: + - dependencies + - rust + - "Changed" diff --git a/package-lock.json b/package-lock.json index 8d742efc..5c4ce084 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "cadt", - "version": "1.7.15", + "version": "1.7.16", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cadt", - "version": "1.7.15", + "version": "1.7.16", "dependencies": { "@babel/eslint-parser": "^7.23.10", "async-mutex": "^0.4.1", @@ -39,26 +39,26 @@ "cadt": "build/server.js" }, "devDependencies": { - "@babel/cli": "^7.24.1", - "@babel/core": "^7.24.4", + "@babel/cli": "^7.24.5", + "@babel/core": "^7.24.5", "@babel/plugin-syntax-import-attributes": "^7.24.1", - "@babel/preset-env": "^7.24.4", + "@babel/preset-env": "^7.24.5", "@babel/register": "^7.23.7", - "@commitlint/cli": "^19.2.1", - "@commitlint/config-conventional": "^19.1.0", - "babel-plugin-module-resolver": "^5.0.0", - "chai": "^5.1.0", + "@commitlint/cli": "^19.3.0", + "@commitlint/config-conventional": "^19.2.2", + "babel-plugin-module-resolver": "^5.0.2", + "chai": "^5.1.1", "chai-http": "^4.4.0", - "eslint": "^8.57.0", + "eslint": "^8.0.0", "eslint-plugin-es": "^4.1.0", - "eslint-plugin-mocha": "^10.4.1", + "eslint-plugin-mocha": "^10.4.3", "husky": "^9.0.11", "mocha": "^10.4.0", - "semver": "^7.6.0", - "sinon": "^17.0.1", + "semver": "^7.6.2", + "sinon": "^18.0.0", "socket.io-client": "^4.7.5", "standard-version": "^9.5.0", - "supertest": "^6.3.4" + "supertest": "^7.0.0" }, "engines": { "node": ">=16.13" @@ -2565,9 +2565,9 @@ "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" }, "node_modules/@types/node": { - "version": "20.12.11", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.11.tgz", - "integrity": "sha512-vDg9PZ/zi+Nqp6boSOT7plNuthRugEKixDv5sFTIpkE89MmNtEArAShI4mxuX2+UrLEe9pxC1vm2cjm9YlWbJw==", + "version": "20.12.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.12.12.tgz", + "integrity": "sha512-eWLDGF/FOSPtAvEqeRAQ4C8LSA7M1I7i0ky1I8U7kD1J5ITyW3AsRhQrKVoWf5pFKZ2kILsEGJhsI9r93PYnOw==", "dependencies": { "undici-types": "~5.26.4" } @@ -2774,6 +2774,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", + "deprecated": "This package is no longer supported.", "optional": true, "dependencies": { "delegates": "^1.0.0", @@ -3100,12 +3101,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -3319,9 +3320,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001617", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001617.tgz", - "integrity": "sha512-mLyjzNI9I+Pix8zwcrpxEbGlfqOkF9kM3ptzmKNw5tizSyYwMe+nGLTqMK9cO+0E+Bh6TsBxNAaHWEM8xwSsmA==", + "version": "1.0.30001620", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001620.tgz", + "integrity": "sha512-WJvYsOjd1/BYUY6SNGUosK9DUidBPDTnOARHp3fSmFO1ekdxaY6nKRttEVrfMmYi80ctS0kz1wiWmm14fVc3ew==", "funding": [ { "type": "opencollective", @@ -4698,9 +4699,9 @@ "integrity": "sha512-LDx6oHrK+PhzLKJU9j5S7/Y3jM/mUHvD/DeI1WQmJn652iPC5Y4TBzC9l+5OMOXlyTTA+SmVUPm0HQUwpD5Jqw==" }, "node_modules/core-js-compat": { - "version": "3.37.0", - "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.37.0.tgz", - "integrity": "sha512-vYq4L+T8aS5UuFg4UwDhc7YNRWVeVZwltad9C/jV3R2LgVOpS9BDr7l/WL6BN0dbV3k1XejPTHqqEzJgsa0frA==", + "version": "3.37.1", + "resolved": "https://registry.npmjs.org/core-js-compat/-/core-js-compat-3.37.1.tgz", + "integrity": "sha512-9TNiImhKvQqSUkOvk/mMRZzOANTiEVC7WaBNhHcKM7x+/5E1l5NvsysR19zuDQScE8k+kfQXWRN3AtS/eOSHpg==", "dev": true, "dependencies": { "browserslist": "^4.23.0" @@ -5144,9 +5145,9 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, "node_modules/electron-to-chromium": { - "version": "1.4.764", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.764.tgz", - "integrity": "sha512-ZXbPV46Y4dNCA+k7YHB+BYlzcoMtZ1yH6V0tQ1ul0wmA7RiwJfS29LSdRlE1myWBXRzEgm/Lz6tryj5WVQiLmg==" + "version": "1.4.777", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.777.tgz", + "integrity": "sha512-n02NCwLJ3wexLfK/yQeqfywCblZqLcXphzmid5e8yVPdtEcida7li0A5WQKghHNG0FeOMCzeFOzEbtAh5riXFw==" }, "node_modules/emoji-regex": { "version": "8.0.0", @@ -5978,9 +5979,9 @@ "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -6197,6 +6198,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", + "deprecated": "This package is no longer supported.", "optional": true, "dependencies": { "aproba": "^1.0.3 || ^2.0.0", @@ -8484,9 +8486,9 @@ "dev": true }, "node_modules/nise": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.9.tgz", - "integrity": "sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/nise/-/nise-6.0.0.tgz", + "integrity": "sha512-K8ePqo9BFvN31HXwEtTNGzgrPpmvgciDsFz8aztFjt4LqKO/JeFD8tBOeuDiCMXrIl/m1YvfH8auSpxfaD09wg==", "dev": true, "dependencies": { "@sinonjs/commons": "^3.0.0", @@ -8634,6 +8636,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", + "deprecated": "This package is no longer supported.", "optional": true, "dependencies": { "are-we-there-yet": "^3.0.0", @@ -8923,9 +8926,9 @@ "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -10033,17 +10036,17 @@ "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" }, "node_modules/sinon": { - "version": "17.0.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", - "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/sinon/-/sinon-18.0.0.tgz", + "integrity": "sha512-+dXDXzD1sBO6HlmZDd7mXZCR/y5ECiEiGCBSGuFD/kZ0bDTofPYc6JaeGmPSF+1j1MejGUWkORbYOLDyvqCWpA==", "dev": true, "dependencies": { - "@sinonjs/commons": "^3.0.0", + "@sinonjs/commons": "^3.0.1", "@sinonjs/fake-timers": "^11.2.2", "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.5", - "supports-color": "^7.2.0" + "diff": "^5.2.0", + "nise": "^6.0.0", + "supports-color": "^7" }, "funding": { "type": "opencollective", @@ -10688,16 +10691,62 @@ } }, "node_modules/supertest": { - "version": "6.3.4", - "resolved": "https://registry.npmjs.org/supertest/-/supertest-6.3.4.tgz", - "integrity": "sha512-erY3HFDG0dPnhw4U+udPfrzXa4xhSG+n4rxfRuZWCUvjFWwKl+OxWf/7zk50s84/fAAs7vf5QAb9uRa0cCykxw==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/supertest/-/supertest-7.0.0.tgz", + "integrity": "sha512-qlsr7fIC0lSddmA3tzojvzubYxvlGtzumcdHgPwbFWMISQwL22MhM2Y3LNt+6w9Yyx7559VW5ab70dgphm8qQA==", "dev": true, "dependencies": { "methods": "^1.1.2", - "superagent": "^8.1.2" + "superagent": "^9.0.1" }, "engines": { - "node": ">=6.4.0" + "node": ">=14.18.0" + } + }, + "node_modules/supertest/node_modules/formidable": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-3.5.1.tgz", + "integrity": "sha512-WJWKelbRHN41m5dumb0/k8TeAx7Id/y3a+Z7QfhxP/htI9Js5zYaEDtG8uMgG0vM0lOlqnmjE99/kfpOYi/0Og==", + "dev": true, + "dependencies": { + "dezalgo": "^1.0.4", + "hexoid": "^1.0.0", + "once": "^1.4.0" + }, + "funding": { + "url": "https://ko-fi.com/tunnckoCore/commissions" + } + }, + "node_modules/supertest/node_modules/mime": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz", + "integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/supertest/node_modules/superagent": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/superagent/-/superagent-9.0.2.tgz", + "integrity": "sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==", + "dev": true, + "dependencies": { + "component-emitter": "^1.3.0", + "cookiejar": "^2.1.4", + "debug": "^4.3.4", + "fast-safe-stringify": "^2.1.1", + "form-data": "^4.0.0", + "formidable": "^3.5.1", + "methods": "^1.1.2", + "mime": "2.6.0", + "qs": "^6.11.0" + }, + "engines": { + "node": ">=14.18.0" } }, "node_modules/supports-color": { @@ -11091,9 +11140,9 @@ } }, "node_modules/update-browserslist-db": { - "version": "1.0.15", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.15.tgz", - "integrity": "sha512-K9HWH62x3/EalU1U6sjSZiylm9C8tgq2mSvshZpqc7QE69RaA2qjhkW2HlNA0tFpEbtyFz7HTqbSdN4MSwUodA==", + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz", + "integrity": "sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ==", "funding": [ { "type": "opencollective", @@ -11110,7 +11159,7 @@ ], "dependencies": { "escalade": "^3.1.2", - "picocolors": "^1.0.0" + "picocolors": "^1.0.1" }, "bin": { "update-browserslist-db": "cli.js" @@ -11394,6 +11443,7 @@ "version": "0.19.3", "resolved": "https://cdn.sheetjs.com/xlsx-0.19.3/xlsx-0.19.3.tgz", "integrity": "sha512-8IfgFctB7fkvqkTGF2MnrDrC6vzE28Wcc1aSbdDQ+4/WFtzfS73YuapbuaPZwGqpR2e0EeDMIrFOJubQVLWFNA==", + "license": "Apache-2.0", "bin": { "xlsx": "bin/xlsx.njs" }, diff --git a/package.json b/package.json index d069468d..e797dba5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cadt", - "version": "1.7.15", + "version": "1.7.16", "_comment": "DONT CHANGE MAJOR UNLESS DATAMODEL CHANGES: The major version corresponds to the datamodel version your using, so 2.0.0 means it'll use datamodel v2", "private": true, "bin": "build/server.js", @@ -59,26 +59,26 @@ "winston-daily-rotate-file": "^4.7.1" }, "devDependencies": { - "@babel/cli": "^7.24.1", - "@babel/core": "^7.24.4", + "@babel/cli": "^7.24.5", + "@babel/core": "^7.24.5", "@babel/plugin-syntax-import-attributes": "^7.24.1", - "@babel/preset-env": "^7.24.4", + "@babel/preset-env": "^7.24.5", "@babel/register": "^7.23.7", - "@commitlint/cli": "^19.2.1", - "@commitlint/config-conventional": "^19.1.0", - "babel-plugin-module-resolver": "^5.0.0", - "chai": "^5.1.0", + "@commitlint/cli": "^19.3.0", + "@commitlint/config-conventional": "^19.2.2", + "babel-plugin-module-resolver": "^5.0.2", + "chai": "^5.1.1", "chai-http": "^4.4.0", - "eslint": "^8.57.0", + "eslint": "^8.0.0", "eslint-plugin-es": "^4.1.0", - "eslint-plugin-mocha": "^10.4.1", + "eslint-plugin-mocha": "^10.4.3", "husky": "^9.0.11", "mocha": "^10.4.0", - "semver": "^7.6.0", - "sinon": "^17.0.1", + "semver": "^7.6.2", + "sinon": "^18.0.0", "socket.io-client": "^4.7.5", "standard-version": "^9.5.0", - "supertest": "^6.3.4" + "supertest": "^7.0.0" }, "standard-version": { "skip": { diff --git a/src/controllers/audit.controller.js b/src/controllers/audit.controller.js index 2716df9c..eae5735e 100644 --- a/src/controllers/audit.controller.js +++ b/src/controllers/audit.controller.js @@ -1,9 +1,10 @@ import { Audit } from '../models'; - +import _ from 'lodash'; import { paginationParams, optionallyPaginatedResponse, } from '../utils/helpers'; +import { assertIfReadOnlyMode } from '../utils/data-assertions.js'; export const findAll = async (req, res) => { try { @@ -20,7 +21,7 @@ export const findAll = async (req, res) => { return res.json(optionallyPaginatedResponse(auditResults, page, limit)); } catch (error) { res.status(400).json({ - message: 'Can not retreive audit data', + message: 'Can not retrieve audit data', error: error.message, success: false, }); @@ -32,9 +33,73 @@ export const findConflicts = async (req, res) => { return res.json(await Audit.findConflicts()); } catch (error) { res.status(400).json({ - message: 'Can not retreive audit data', + message: 'Can not retrieve audit data', error: error.message, success: false, }); } }; + +export const resetToGeneration = async (req, res) => { + try { + await assertIfReadOnlyMode(); + const { generation, orgUid } = req.body; + + const result = await Audit.resetToGeneration(generation, orgUid); + if (_.isNil(result)) { + throw new Error('query failed'); + } + return res.json({ + message: result + ? 'reset to generation ' + String(generation) + : 'no matching records', + success: true, + }); + } catch (error) { + if (error.message === 'SQLITE_BUSY: database is locked') { + res.status(400).json({ + message: 'failed to change generation', + error: 'cadt is currently syncing, please try again later', + success: false, + }); + } else { + res.status(400).json({ + message: 'failed to change generation', + error: error.message, + success: false, + }); + } + } +}; + +export const resetToDate = async (req, res) => { + try { + await assertIfReadOnlyMode(); + const { date, orgUid, includeHomeOrg } = req.body; + + const result = orgUid + ? await Audit.resetOrgToDate(date, orgUid) + : await Audit.resetToDate(date, includeHomeOrg); + if (_.isNil(result)) { + throw new Error('query failed'); + } + return res.json({ + message: result ? 'reset to date ' + String(date) : 'no matching records', + success: true, + }); + } catch (error) { + if (error.message === 'SQLITE_BUSY: database is locked') { + res.status(400).json({ + message: 'failed to reset to date', + error: 'cadt is currently syncing, please try again later', + success: false, + }); + } else { + res.status(400).json({ + message: 'failed to reset to date', + error: error.message, + success: false, + }); + } + } +}; diff --git a/src/models/audit/audit.model.js b/src/models/audit/audit.model.js index 4851504a..e10057df 100644 --- a/src/models/audit/audit.model.js +++ b/src/models/audit/audit.model.js @@ -6,6 +6,7 @@ import { sequelize, safeMirrorDbHandler } from '../../database'; import { AuditMirror } from './audit.model.mirror'; import ModelTypes from './audit.modeltypes.cjs'; import findDuplicateIssuancesSql from './sql/find-duplicate-issuances.sql.js'; +import { Organization } from '../organizations/index.js'; class Audit extends Model { static async create(values, options) { @@ -45,6 +46,56 @@ class Audit extends Model { const [results] = await sequelize.query(findDuplicateIssuancesSql); return results; } + + static async resetToGeneration(generation, orgUid) { + const where = { + generation: { [Sequelize.Op.gt]: generation }, + }; + + if (orgUid) { + where.orgUid = orgUid; + } + + return await Audit.destroy({ where }); + } + + static async resetToDate(date, includeHomeOrg) { + const timestampInSeconds = Math.round(new Date(date).valueOf() / 1000); + const homeOrgUid = Organization.getHomeOrg()?.uid; + + const conditions = [ + sequelize.where( + sequelize.cast( + sequelize.col('onChainConfirmationTimeStamp'), + 'UNSIGNED', + ), + { [Sequelize.Op.gt]: timestampInSeconds }, + ), + ]; + + if (!includeHomeOrg && homeOrgUid) { + conditions.push({ orguid: { [Sequelize.Op.ne]: homeOrgUid } }); + } + + return await Audit.destroy({ where: { [Sequelize.Op.and]: conditions } }); + } + + static async resetOrgToDate(date, orgUid) { + const timestampInSeconds = Math.round(new Date(date).valueOf() / 1000); + + return await Audit.destroy({ + where: { + orgUid: orgUid, + [Sequelize.Op.and]: sequelize.where( + sequelize.cast( + sequelize.col('onchainConfirmationTimeStamp'), + 'UNSIGNED', + ), + { [Sequelize.Op.gt]: timestampInSeconds }, + ), + }, + }); + } } Audit.init(ModelTypes, { diff --git a/src/routes/v1/resources/audit.js b/src/routes/v1/resources/audit.js index 0a485eed..60bed495 100644 --- a/src/routes/v1/resources/audit.js +++ b/src/routes/v1/resources/audit.js @@ -4,7 +4,11 @@ import express from 'express'; import joiExpress from 'express-joi-validation'; import { AuditController } from '../../../controllers'; -import { auditGetSchema } from '../../../validations'; +import { + auditGetSchema, + auditResetToDateSchema, + auditResetToGenerationSchema, +} from '../../../validations'; const validator = joiExpress.createValidator({ passError: true }); const AuditRouter = express.Router(); @@ -17,4 +21,20 @@ AuditRouter.get('/findConflicts', (req, res) => { return AuditController.findConflicts(req, res); }); +AuditRouter.post( + '/resetToGeneration', + validator.body(auditResetToGenerationSchema), + (req, res) => { + return AuditController.resetToGeneration(req, res); + }, +); + +AuditRouter.post( + '/resetToDate', + validator.body(auditResetToDateSchema), + (req, res) => { + return AuditController.resetToDate(req, res); + }, +); + export { AuditRouter }; diff --git a/src/tasks/index.js b/src/tasks/index.js index 4ce68209..ac6c0efe 100644 --- a/src/tasks/index.js +++ b/src/tasks/index.js @@ -6,6 +6,7 @@ import syncRegistries from './sync-registries'; import syncOrganizationMeta from './sync-organization-meta'; import syncGovernanceBody from './sync-governance-body'; import mirrorCheck from './mirror-check'; +import resetAuditTable from './reset-audit-table'; const scheduler = new ToadScheduler(); @@ -25,6 +26,7 @@ const start = () => { syncRegistries, syncOrganizationMeta, mirrorCheck, + resetAuditTable, ]; defaultJobs.forEach((defaultJob) => { jobRegistry[defaultJob.id] = defaultJob; diff --git a/src/tasks/reset-audit-table.js b/src/tasks/reset-audit-table.js new file mode 100644 index 00000000..bf715819 --- /dev/null +++ b/src/tasks/reset-audit-table.js @@ -0,0 +1,66 @@ +import { SimpleIntervalJob, Task } from 'toad-scheduler'; +import { Audit, Meta } from '../models'; +import { logger } from '../config/logger.cjs'; +import dotenv from 'dotenv'; +import _ from 'lodash'; +dotenv.config(); + +const task = new Task('reset-audit-table', async () => { + try { + const metaResult = await Meta.findOne({ + where: { + metaKey: 'may2024AuditResetTaskHasRun', + }, + attributes: ['metaValue'], + raw: true, + }); + + const taskHasRun = metaResult?.metaValue; + + if (taskHasRun === 'true') { + return; + } + + logger.info('performing audit table reset'); + + const where = { type: 'NO CHANGE' }; + const noChangeEntries = await Audit.findAll({ where }); + + if (noChangeEntries.length) { + const result = await Audit.resetToDate('2024-05-11'); + logger.info( + 'audit table has been reset, records modified: ' + String(result), + ); + } + + if (_.isNil(taskHasRun)) { + await Meta.create({ + metaKey: 'may2024AuditResetTaskHasRun', + metaValue: 'true', + }); + } else { + await Meta.update( + { metavalue: 'true' }, + { + where: { + metakey: 'may2024AuditResetTaskHasRun', + }, + returning: true, + }, + ); + } + } catch (error) { + logger.error('Retrying in 600 seconds', error); + } +}); + +const job = new SimpleIntervalJob( + { + seconds: 600, + runImmediately: true, + }, + task, + { id: 'reset-audit-table', preventOverrun: true }, +); + +export default job; diff --git a/src/tasks/sync-registries.js b/src/tasks/sync-registries.js index 11d251d4..d2a33873 100644 --- a/src/tasks/sync-registries.js +++ b/src/tasks/sync-registries.js @@ -173,7 +173,9 @@ const syncOrganizationAudit = async (organization) => { const rootHistory = await datalayer.getRootHistory(organization.registryId); if (!rootHistory.length) { - logger.info(`No root history found for ${organization.name}`); + logger.info( + `No root history found for ${organization.name} (store ${organization.orgUid})`, + ); return; } @@ -204,7 +206,9 @@ const syncOrganizationAudit = async (organization) => { let currentGeneration = _.get(rootHistory, '[0]'); if (!lastRootSaved) { - logger.info(`Syncing new registry ${organization.name}`); + logger.info( + `Syncing new registry ${organization.name} (store ${organization.orgUid})`, + ); await Audit.create({ orgUid: organization.orgUid, @@ -241,7 +245,7 @@ const syncOrganizationAudit = async (organization) => { if (lastProcessedIndex > rootHistory.length) { logger.error( - `Could not find root history for ${organization.name} with timestamp ${currentGeneration.timestamp}, something is wrong and the sync for this organization will be paused until this is resolved.`, + `Could not find root history for ${organization.name} (store ${organization.orgUid}) with timestamp ${currentGeneration.timestamp}, something is wrong and the sync for this organization will be paused until this is resolved.`, ); } @@ -269,10 +273,10 @@ const syncOrganizationAudit = async (organization) => { // Organization not synced, sync it logger.info(' '); logger.info( - `Syncing ${organization.name} generation ${toBeProcessedIndex}`, + `Syncing ${organization.name} generation ${toBeProcessedIndex} (store ${organization.orgUid})`, ); logger.info( - `${organization.name} is ${syncRemaining} DataLayer generations away from being fully synced.`, + `${organization.name} is ${syncRemaining} DataLayer generations away from being fully synced (store ${organization.orgUid}).`, ); if (!CONFIG.USE_SIMULATOR) { @@ -282,9 +286,9 @@ const syncOrganizationAudit = async (organization) => { organization.registryId, ); - if (lastProcessedIndex > sync_status.generation) { + if (toBeProcessedIndex > sync_status.generation) { const warningMsg = [ - `No data found for ${organization.name} in the current datalayer generation.`, + `No data found for ${organization.name} (store ${organization.orgUid}) in the current datalayer generation.`, `DataLayer not yet caught up to generation ${lastProcessedIndex}. The current processed generation is ${sync_status.generation}.`, `This issue is often temporary and could be due to a lag in data propagation.`, 'Syncing for this organization will be paused until this is resolved.', @@ -306,7 +310,7 @@ const syncOrganizationAudit = async (organization) => { if (!_.get(root2, 'confirmed')) { logger.info( - `Waiting for the latest root for ${organization.name} to confirm`, + `Waiting for the latest root for ${organization.name} to confirm (store ${organization.orgUid})`, ); return; } @@ -341,7 +345,7 @@ const syncOrganizationAudit = async (organization) => { const updateTransaction = async (transaction, mirrorTransaction) => { logger.info( - `Syncing ${organization.name} generation ${toBeProcessedIndex}`, + `Syncing ${organization.name} generation ${toBeProcessedIndex} (store ${organization.orgUid})`, ); if (_.isEmpty(optimizedKvDiff)) { const auditData = { diff --git a/src/validations/audit.validations.js b/src/validations/audit.validations.js index df44011d..9a017b69 100644 --- a/src/validations/audit.validations.js +++ b/src/validations/audit.validations.js @@ -9,3 +9,14 @@ export const auditGetSchema = Joi.object() }) .with('page', 'limit') .with('limit', 'page'); + +export const auditResetToGenerationSchema = Joi.object().keys({ + generation: Joi.number(), + orgUid: Joi.string(), +}); + +export const auditResetToDateSchema = Joi.object().keys({ + date: Joi.date(), + orgUid: Joi.string().optional(), + includeHomeOrg: Joi.bool().optional(), +});