Vacms 16208 police testing (#1822)

* police testing
* correct demo data
* fix csv
* CODEOWNERS

.github/CODEOWNERS

@@ -30,6 +30,7 @@ src/site/layouts/health*.drupal.liquid @department-of-veterans-affairs/vfs-facil
 src/site/navigation/facility_no_drupal_page_sidebar_nav.drupal.liquid @department-of-veterans-affairs/vfs-facilities-frontend
 src/site/navigation/facility_sidebar_nav.drupal.liquid @department-of-veterans-affairs/vfs-facilities-frontend
 src/site/paragraphs/facilities @department-of-veterans-affairs/vfs-facilities-frontend
+src/site/stages/build/drupal/static-data-files/vaPoliceData @department-of-veterans-affairs/vfs-facilities-frontend
 
 # GraphQL Queries
 src/site/stages/build/drupal @department-of-veterans-affairs/vfs-public-websites-frontend @department-of-veterans-affairs/vfs-facilities-frontend @department-of-veterans-affairs/cms-infrastructure

src/site/stages/build/drupal/static-data-files/fetchApi.js

@@ -58,7 +58,6 @@ function getCurlClient(buildOptions, _clientOptionsArg = { verbose: true }) {
           'certs/VA-Internal-S2-RCA2.pem',
         ]);
       }
-
       return fetchWrapper(
         url,
         // eslint-disable-next-line prefer-object-spread

src/site/stages/build/drupal/static-data-files/generate.js

@@ -380,3 +380,4 @@ const generateStaticDataFiles = async (
 };
 
 module.exports.generateStaticDataFiles = generateStaticDataFiles;
+module.exports.processCurlDataFile = processCurlDataFile;

src/site/stages/build/drupal/static-data-files/vaPoliceData/index.js

@@ -1,35 +1,14 @@
-const fs = require('fs');
-const csv = require('csvtojson');
-const path = require('path');
 const { join } = require('path');
 const { pathToFileURL } = require('url');
+const { postProcessPolice } = require('./postProcessPolice');
 
 // URLs to fetch (even if they are local files)
 const query = [
-  pathToFileURL(join(__dirname, 'police-contact.csv')).toString(),
-  pathToFileURL(join(__dirname, 'police-events.csv')).toString(),
+  pathToFileURL(join(__dirname, 'police-contact.csv')).toString(), // contacts is always first
+  pathToFileURL(join(__dirname, 'police-events.csv')).toString(), // any number of events files following
 ];
 
-const postProcess = async queryResult => {
-  const processedJSON = {
-    data: {
-      statistics: {},
-      contacts: {},
-    },
-  };
-  const [contact, events] = queryResult;
-  const contactFile = path.join(__dirname, 'pre-contact-police.csv');
-  const eventsFile = path.join(__dirname, 'pre-events-police.csv');
-  // Unfortunately there's no Buffer or file-string support in csvtojson, there's read and process per-line, but this is more efficient.
-  fs.writeFileSync(contactFile, contact, { append: false });
-  fs.writeFileSync(eventsFile, events, { append: false });
-  // eslint-disable-next-line no-unused-vars
-  const contactsJson = await csv().fromFile(contactFile);
-  // eslint-disable-next-line no-unused-vars
-  const eventsJson = await csv().fromFile(eventsFile);
-  // TODO: Process jsonEvents and Join data with contact info for a Facility Police Page content
-  return processedJSON;
-};
+const postProcess = postProcessPolice;
 
 module.exports = {
   query,

src/site/stages/build/drupal/static-data-files/vaPoliceData/police-contact.csv

@@ -1,3 +1 @@
-facility_id,contact_name,contact_number
-vha_011,Abc Jameson,1234567899
-vha_021,Xyz Adamson,1123456799
+VISN,Facility API ID,Contact Name,Contact Number

src/site/stages/build/drupal/static-data-files/vaPoliceData/police-events.csv

@@ -1,7 +1 @@
-date_range,facility_id,a,b,c
-2021-01-01:2021-12-31,vha_011,2,3,4
-2022-01-01:2022-12-31,vha_011,3,4,5
-2023-01-01:2023-10-10,vha_011,1,1,1
-2021-01-01:2021-12-31,vha_021,1,3,1
-2022-01-01:2022-12-31,vha_021,2,0,1
-2023-01-01:2023-10-10,vha_021,0,1,0
+VISN,Facility API ID,Facility Name,MM/YYYY,Number of service calls (officer initiated and response to calls),Traffic and parking tickets,Non-traffic (criminal) tickets,Arrests,Complaints and investigations,Numbers of sustained allegations,Numbers of disciplinary actions

src/site/stages/build/drupal/static-data-files/vaPoliceData/postProcessPolice.js

@@ -0,0 +1,95 @@
+const csv = require('csvtojson');
+
+function processJsonPoliceData(unprocessedJson) {
+  const processedJSON = {
+    VISN: null,
+    facilityAPIId: '',
+    facilityName: '',
+    date: '',
+    numServiceCalls: null,
+    trafficParkingTickets: null,
+    criminalTickets: null,
+    arrests: null,
+    complaintsInvestigations: null,
+    sustainedAllegations: null,
+    disciplinaryActions: null,
+  };
+  for (const [key, value] of Object.entries(unprocessedJson)) {
+    switch (key) {
+      case 'VISN':
+        processedJSON.VISN = parseInt(value, 10);
+        break;
+      case 'Facility API ID':
+        processedJSON.facilityAPIId = value;
+        break;
+      case 'Facility Name':
+        processedJSON.facilityName = value;
+        break;
+      case 'MM/YYYY':
+        processedJSON.date = value;
+        break;
+      case 'Number of service calls (officer initiated and response to calls)':
+        processedJSON.numServiceCalls = parseInt(value, 10);
+        break;
+      case 'Traffic and parking tickets':
+        processedJSON.trafficParkingTickets = parseInt(value, 10);
+        break;
+      case 'Non-traffic (criminal) tickets':
+        processedJSON.criminalTickets = parseInt(value, 10);
+        break;
+      case 'Arrests':
+        processedJSON.arrests = parseInt(value, 10);
+        break;
+      case 'Complaints and investigations':
+        processedJSON.complaintsInvestigations = parseInt(value, 10);
+        break;
+      case 'Numbers of sustained allegations':
+        processedJSON.sustainedAllegations = parseInt(value, 10);
+        break;
+      case 'Numbers of disciplinary actions':
+        processedJSON.disciplinaryActions = parseInt(value, 10);
+        break;
+      default:
+        break;
+    }
+  }
+  return processedJSON;
+}
+
+async function postProcessPolice(queryResult) {
+  const processedJSON = {
+    data: {
+      statistics: {}, // keys:facilityAPIId, values: array of objects in processed format
+      contacts: {},
+    },
+  };
+  const [contact, ...events] = queryResult;
+  if (!contact || !events || events.length === 0) {
+    throw new Error(
+      'Police data files must have at least one contact file and one events file.',
+    );
+  }
+  const contactData = await csv().fromString(contact);
+  const eventsData = await Promise.all(
+    events.map(eventsFile => csv().fromString(eventsFile)),
+  ); // each file has its own header, but JSON's are the same
+  const processedEventsData = eventsData.flat().map(processJsonPoliceData); // convert keys to usable format
+  for (const processedEventsDataEntry of processedEventsData) {
+    if (processedJSON.data.statistics[processedEventsDataEntry.facilityAPIId]) {
+      processedJSON.data.statistics[
+        processedEventsDataEntry.facilityAPIId
+      ].push(processedEventsDataEntry);
+    } else {
+      processedJSON.data.statistics[processedEventsDataEntry.facilityAPIId] = [
+        processedEventsDataEntry,
+      ];
+    }
+  }
+
+  processedJSON.data.contacts = contactData;
+
+  // TODO: Process jsonEvents and Join data with contact info for a Facility Police Page content
+  return processedJSON;
+}
+module.exports.postProcessPolice = postProcessPolice;
+module.exports.processJsonPoliceData = processJsonPoliceData;

src/site/stages/build/drupal/tests/police/download.unit.spec.js

@@ -0,0 +1,75 @@
+/* eslint-disable @department-of-veterans-affairs/axe-check-required */
+import path from 'path';
+import { describe, it } from 'mocha';
+import { expect } from 'chai';
+import { processCurlDataFile } from '../../static-data-files/generate';
+import { postProcessPolice } from '../../static-data-files/vaPoliceData/postProcessPolice';
+
+const { pathToFileURL } = require('url');
+const getCurlClient = require('../../static-data-files/fetchApi');
+
+/* This is not a FE test but a unit test. */
+describe('process police csv files', () => {
+  it('should have an error if the police-contact.csv file is missing', async () => {
+    const client = getCurlClient({
+      'drupal-user': '[email protected]',
+      'drupal-password': 'drupal8',
+      'drupal-max-parallel-requests': 10,
+    });
+    expect(
+      processCurlDataFile(
+        {
+          description: 'Curl',
+          filename: 'test-police.json',
+          query: [
+            pathToFileURL(
+              path.join(
+                __dirname,
+                './fixtures/non-existant-police-contact.csv',
+              ),
+            ).toString(),
+            pathToFileURL(
+              path.join(__dirname, './fixtures/non-existant-police.csv'),
+            ).toString(),
+          ],
+          postProcess: postProcessPolice,
+        },
+        client,
+      ),
+    ).to.be.rejectedWith(Error);
+  });
+  it('should process files', async () => {
+    const client = getCurlClient({
+      'drupal-user': '[email protected]',
+      'drupal-password': 'drupal8',
+      'drupal-max-parallel-requests': 10,
+    });
+    const processedDataFile = await processCurlDataFile(
+      {
+        description: 'Curl',
+        filename: 'test-police.json',
+        query: [
+          pathToFileURL(
+            path.join(__dirname, './fixtures/police-contact.csv'),
+          ).toString(),
+          pathToFileURL(
+            path.join(__dirname, './fixtures/police.csv'),
+          ).toString(),
+        ],
+        postProcess: postProcessPolice,
+      },
+      client,
+    );
+    const keys = Object.keys(processedDataFile.data.data.statistics);
+    expect(keys).to.contain('avha_635');
+    expect(keys).to.contain('avha_523A5');
+    expect(processedDataFile.data.data.statistics.avha_635).to.be.an('array');
+    expect(processedDataFile.data.data.statistics.avha_523A5).to.be.an('array');
+    expect(processedDataFile.data.data.statistics.avha_635[0].VISN).to.be.equal(
+      19,
+    ); // that way we know processing succeeded
+    expect(
+      processedDataFile.data.data.statistics.avha_523A5[0].VISN,
+    ).to.be.equal(1);
+  });
+});

src/site/stages/build/drupal/tests/police/fixtures/police-contact.csv

@@ -0,0 +1,3 @@
+VISN,Facility API ID,Contact Name,Contact Number
+19,avha_635,Abc Jameson,12345689
+1,avha_523A5,Zyz Adamson,1123456799

src/site/stages/build/drupal/tests/police/fixtures/police.csv

@@ -0,0 +1,3 @@
+VISN,Facility API ID,Facility Name,MM/YYYY,Number of service calls (officer initiated and response to calls),Traffic and parking tickets,Non-traffic (criminal) tickets,Arrests,Complaints and investigations,Numbers of sustained allegations,Numbers of disciplinary actions
+19,avha_635,Oklahoma City VA Medical Center,12/2023,4000,100,100,600,50,10,2
+1,avha_523A5,Brockton VA Medical Center,12/2023,5200,220,325,752,78,8,3

yarn.lock

@@ -2777,9 +2777,9 @@ caniuse-api@^3.0.0:
     lodash.uniq "^4.5.0"
 
 caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001252, caniuse-lite@^1.0.30001254:
-  version "1.0.30001259"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001259.tgz#ae21691d3da9c4be6144403ac40f71d9f6efd790"
-  integrity sha512-V7mQTFhjITxuk9zBpI6nYsiTXhcPe05l+364nZjK7MFK/E7ibvYBSAXr4YcA6oPR8j3ZLM/LN+lUqUVAQEUZFg==
+  version "1.0.30001566"
+  resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001566.tgz"
+  integrity sha512-ggIhCsTxmITBAMmK8yZjEhCO5/47jKXPu6Dha/wuCS4JePVL+3uiDEBuhu2aIoT+bqTOR8L76Ip1ARL9xYsEJA==
 
 caseless@~0.12.0:
   version "0.12.0"