From c7180e74baeea85e20f5e8370350e5d9371dd7cd Mon Sep 17 00:00:00 2001 From: Kathleen Tuite Date: Wed, 11 Dec 2024 16:15:35 -0800 Subject: [PATCH] Beginning to add new usage metrics for 2024.3 release --- config/default.json | 2 +- lib/data/analytics.js | 21 +++- lib/model/query/analytics.js | 41 +++++-- test/integration/other/analytics-queries.js | 121 ++++++++++++++++++-- 4 files changed, 164 insertions(+), 21 deletions(-) diff --git a/config/default.json b/config/default.json index 19f31cfca..55b038ebd 100644 --- a/config/default.json +++ b/config/default.json @@ -30,7 +30,7 @@ "analytics": { "url": "https://data.getodk.cloud/v1/key/eOZ7S4bzyUW!g1PF6dIXsnSqktRuewzLTpmc6ipBtRq$LDfIMTUKswCexvE0UwJ9/projects/1/forms/odk-analytics/submissions", "formId": "odk-analytics", - "version": "v2024.2.0_1" + "version": "v2024.3.0_1" }, "s3blobStore": {} } diff --git a/lib/data/analytics.js b/lib/data/analytics.js index 83f9cdbf9..5ca8be2f1 100644 --- a/lib/data/analytics.js +++ b/lib/data/analytics.js @@ -31,6 +31,7 @@ const metricsTemplate = { "num_unique_collectors": {}, "database_size": {}, "uses_external_db": 0, + "uses_external_blob_store": {}, "sso_enabled": 0, "num_client_audit_attachments": 0, "num_client_audit_attachments_failures": 0, @@ -41,8 +42,14 @@ const metricsTemplate = { "num_offline_entity_branches": 0, "num_offline_entity_interrupted_branches": 0, "num_offline_entity_submissions_reprocessed": 0, + "num_offline_entity_submissions_force_processed": {}, "max_entity_submission_delay": 0, - "avg_entity_submission_delay": 0 + "avg_entity_submission_delay": 0, + "max_entity_branch_delay": {}, + "num_xml_only_form_defs": {}, + "num_blob_files": {}, + "num_blob_files_on_s3": {}, + "num_reset_failed_to_pending_count": {} }, "projects": [ { @@ -161,6 +168,18 @@ const metricsTemplate = { "recent": 0, "total": 0 }, + "num_entity_creates_sub": { + "recent": 0, + "total": 0 + }, + "num_entity_creates_api": { + "recent": 0, + "total": 0 + }, + "num_entity_creates_bulk": { + "recent": 0, + "total": 0 + }, "num_creation_forms": 0, "num_followup_forms": 0, "num_failed_entities": { diff --git a/lib/model/query/analytics.js b/lib/model/query/analytics.js index 7adcf676d..44f2f9ba4 100644 --- a/lib/model/query/analytics.js +++ b/lib/model/query/analytics.js @@ -523,10 +523,13 @@ FROM duplicateRuns; // Number of submissions temporarily held in backlog but were automatically // removed from backlog when preceeding submission came in -const countSubmissionReprocess = () => ({ oneFirst }) => oneFirst(sql` - SELECT COUNT(*) +const countSubmissionBacklogEvents = () => ({ one }) => one(sql` + SELECT + COUNT(CASE WHEN "action" = 'submission.backlog.hold' THEN 1 END) AS "submission.backlog.hold", + COUNT(CASE WHEN "action" = 'submission.backlog.reprocess' THEN 1 END) AS "submission.backlog.reprocess", + COUNT(CASE WHEN "action" = 'submission.backlog.force' THEN 1 END) AS "submission.backlog.force" FROM audits - WHERE "action" = 'submission.backlog.reprocess' + WHERE "action" IN ('submission.backlog.hold', 'submission.backlog.reprocess', 'submission.backlog.force') `); // Measure how much time entities whose source is a submission.create @@ -554,6 +557,21 @@ JOIN submission_defs as sd ON eds."submissionDefId" = sd.id; `); +const measureMaxEntityBranchTime = () => ({ oneFirst }) => oneFirst(sql` + SELECT + COALESCE(MAX(AGE(max_created_at, min_created_at)), '0 seconds'::interval) AS max_time_difference + FROM ( + SELECT + "branchId", + "entityId", + MIN("createdAt") AS min_created_at, + MAX("createdAt") AS max_created_at + FROM entity_defs + GROUP BY "branchId", "entityId" + HAVING "branchId" IS NOT NULL + ) AS subquery; +`); + // Other const getProjectsWithDescriptions = () => ({ all }) => all(sql` select id as "projectId", length(trim(description)) as description_length from projects where coalesce(trim(description),'')!=''`); @@ -741,13 +759,14 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([ Analytics.countClientAuditRows, Analytics.countOfflineBranches, Analytics.countInterruptedBranches, - Analytics.countSubmissionReprocess, + Analytics.countSubmissionBacklogEvents, Analytics.measureEntityProcessingTime, + Analytics.measureMaxEntityBranchTime, Analytics.projectMetrics ]).then(([db, encrypt, bigForm, admins, audits, archived, managers, viewers, collectors, caAttachments, caFailures, caRows, - oeBranches, oeInterruptedBranches, oeSubReprocess, oeProcessingTime, + oeBranches, oeInterruptedBranches, oeBacklogEvents, oeProcessingTime, oeBranchTime, projMetrics]) => { const metrics = clone(metricsTemplate); // system @@ -785,10 +804,15 @@ const previewMetrics = () => (({ Analytics }) => runSequentially([ // 2024.2.0 offline entity metrics metrics.system.num_offline_entity_branches = oeBranches; metrics.system.num_offline_entity_interrupted_branches = oeInterruptedBranches; - metrics.system.num_offline_entity_submissions_reprocessed = oeSubReprocess; + metrics.system.num_offline_entity_submissions_reprocessed = oeBacklogEvents['submission.backlog.reprocess']; + metrics.system.num_offline_entity_submissions_force_processed = oeBacklogEvents['submission.backlog.force']; + metrics.system.max_entity_submission_delay = oeProcessingTime.max_wait; metrics.system.avg_entity_submission_delay = oeProcessingTime.avg_wait; + // 2024.3.0 offline entity metrics + metrics.system.max_entity_branch_delay = oeBranchTime; + return metrics; })); @@ -842,7 +866,8 @@ module.exports = { getDatasetEvents, countOfflineBranches, countInterruptedBranches, - countSubmissionReprocess, + countSubmissionBacklogEvents, measureEntityProcessingTime, - measureElapsedEntityTime + measureElapsedEntityTime, + measureMaxEntityBranchTime }; diff --git a/test/integration/other/analytics-queries.js b/test/integration/other/analytics-queries.js index aa02493a9..5e8ab53da 100644 --- a/test/integration/other/analytics-queries.js +++ b/test/integration/other/analytics-queries.js @@ -1580,7 +1580,7 @@ describe('analytics task queries', function () { countInterruptedBranches.should.equal(4); })); - it('should count number of submission.backlog.reprocess events (submissions temporarily in the backlog)', testService(async (service, container) => { + it('should count number of submission.backlog.* events (submissions temporarily in the backlog)', testService(async (service, container) => { await createTestForm(service, container, testData.forms.offlineEntity, 1); const asAlice = await service.login('alice'); @@ -1625,8 +1625,12 @@ describe('analytics task queries', function () { backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); backlogCount.should.equal(0); - let countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(1); + let countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 1, + 'submission.backlog.reprocess': 1, + 'submission.backlog.force': 0 + }); // Send a future update that will get held in backlog await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') @@ -1643,9 +1647,13 @@ describe('analytics task queries', function () { backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); backlogCount.should.equal(1); - // A submission being put in the backlog is not what is counted so this is still 1 - countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(1); + // A submission being put in the backlog is not what is counted so reprocess count is still 1 + countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 2, + 'submission.backlog.reprocess': 1, + 'submission.backlog.force': 0 + }); // force processing the backlog await container.Entities.processBacklog(true); @@ -1653,9 +1661,13 @@ describe('analytics task queries', function () { backlogCount = await container.oneFirst(sql`select count(*) from entity_submission_backlog`); backlogCount.should.equal(0); - // Force processing also doesn't change this count so it is still 1 - countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(1); + // Force processing counted now, and reprocessing still only counted once + countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 2, + 'submission.backlog.reprocess': 1, + 'submission.backlog.force': 1 + }); //---------- @@ -1689,8 +1701,12 @@ describe('analytics task queries', function () { await exhaust(container); // Two reprocessing events logged now - countReprocess = await container.Analytics.countSubmissionReprocess(); - countReprocess.should.equal(2); + countBacklogEvents = await container.Analytics.countSubmissionBacklogEvents(); + countBacklogEvents.should.eql({ + 'submission.backlog.hold': 3, + 'submission.backlog.reprocess': 2, + 'submission.backlog.force': 1 + }); })); it('should measure time from submission creation to entity version finished processing', testService(async (service, container) => { @@ -1731,6 +1747,69 @@ describe('analytics task queries', function () { waitTime.avg_wait.should.be.greaterThan(0); })); + it('should measure max time between first submission on a branch received and last submission on that branch processed', testService(async (service, container) => { + await createTestForm(service, container, testData.forms.offlineEntity, 1); + const asAlice = await service.login('alice'); + + const emptyTime = await container.Analytics.measureMaxEntityBranchTime(); + emptyTime.should.equal(0); + + // Create entity to update + await asAlice.post('/v1/projects/1/datasets/people/entities') + .send({ + uuid: '12345678-1234-4123-8234-123456789abc', + label: 'label' + }) + .expect(200); + + const branchId = uuid(); + + // Send first update + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Send second update + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update2') + .replace('baseVersion="1"', 'baseVersion="2"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + // Send third update in its own branch + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update3') + .replace('baseVersion="1"', 'baseVersion="3"') + .replace('trunkVersion="1"', 'trunkVersion="3"') + .replace('branchId=""', `branchId="${uuid()}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + + // Make another update via the API + await asAlice.patch('/v1/projects/1/datasets/people/entities/12345678-1234-4123-8234-123456789abc?baseVersion=4') + .send({ label: 'label update' }) + .expect(200); + + const time = await container.Analytics.measureMaxEntityBranchTime(); + time.should.be.greaterThan(0); + + // Set date of entity defs in first branch to 1 day apart + await container.run(sql`UPDATE entity_defs SET "createdAt" = '1999-01-01' WHERE version = 2`); + await container.run(sql`UPDATE entity_defs SET "createdAt" = '1999-01-02' WHERE version = 3`); + const longTime = await container.Analytics.measureMaxEntityBranchTime(); + longTime.should.be.equal(86400); // number of seconds in a day + })); + it('should not see a delay for submissions processed with approvalRequired flag toggled', testService(async (service, container) => { const asAlice = await service.login('alice'); @@ -1952,6 +2031,19 @@ describe('analytics task queries', function () { await exhaust(container); + // sending in an update much later in the chain that will need to be force processed + await asAlice.post('/v1/projects/1/forms/offlineEntity/submissions') + .send(testData.instances.offlineEntity.one + .replace('one', 'one-update10') + .replace('baseVersion="1"', 'baseVersion="10"') + .replace('branchId=""', `branchId="${branchId}"`) + ) + .set('Content-Type', 'application/xml') + .expect(200); + + await exhaust(container); + await container.Entities.processBacklog(true); + // After the interesting stuff above, encrypt and archive the project // encrypting a project @@ -1984,6 +2076,13 @@ describe('analytics task queries', function () { // can't easily test this metric delete res.system.uses_external_db; delete res.system.sso_enabled; + delete res.system.uses_external_blob_store; + + // TODO stop deleting these in the tests once they are implemented + delete res.system.num_xml_only_form_defs; + delete res.system.num_blob_files; + delete res.system.num_blob_files_on_s3; + delete res.system.num_reset_failed_to_pending_count; // everything in system filled in Object.values(res.system).forEach((metric) =>