diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ae1e783508..10d9f4f98e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -30,6 +30,7 @@ src/site/layouts/health*.drupal.liquid @department-of-veterans-affairs/vfs-facil src/site/navigation/facility_no_drupal_page_sidebar_nav.drupal.liquid @department-of-veterans-affairs/vfs-facilities-frontend src/site/navigation/facility_sidebar_nav.drupal.liquid @department-of-veterans-affairs/vfs-facilities-frontend src/site/paragraphs/facilities @department-of-veterans-affairs/vfs-facilities-frontend +src/site/stages/build/drupal/static-data-files/vaPoliceData @department-of-veterans-affairs/vfs-facilities-frontend # GraphQL Queries src/site/stages/build/drupal @department-of-veterans-affairs/vfs-public-websites-frontend @department-of-veterans-affairs/vfs-facilities-frontend @department-of-veterans-affairs/cms-infrastructure diff --git a/src/site/stages/build/drupal/static-data-files/fetchApi.js b/src/site/stages/build/drupal/static-data-files/fetchApi.js index 6a716838ca..1a43fb2455 100644 --- a/src/site/stages/build/drupal/static-data-files/fetchApi.js +++ b/src/site/stages/build/drupal/static-data-files/fetchApi.js @@ -58,7 +58,6 @@ function getCurlClient(buildOptions, _clientOptionsArg = { verbose: true }) { 'certs/VA-Internal-S2-RCA2.pem', ]); } - return fetchWrapper( url, // eslint-disable-next-line prefer-object-spread diff --git a/src/site/stages/build/drupal/static-data-files/generate.js b/src/site/stages/build/drupal/static-data-files/generate.js index 6cbacddc01..e0d651e59d 100644 --- a/src/site/stages/build/drupal/static-data-files/generate.js +++ b/src/site/stages/build/drupal/static-data-files/generate.js @@ -380,3 +380,4 @@ const generateStaticDataFiles = async ( }; module.exports.generateStaticDataFiles = generateStaticDataFiles; +module.exports.processCurlDataFile = processCurlDataFile; diff --git a/src/site/stages/build/drupal/static-data-files/vaPoliceData/index.js b/src/site/stages/build/drupal/static-data-files/vaPoliceData/index.js index 9388718420..40d07e1d34 100644 --- a/src/site/stages/build/drupal/static-data-files/vaPoliceData/index.js +++ b/src/site/stages/build/drupal/static-data-files/vaPoliceData/index.js @@ -1,35 +1,14 @@ -const fs = require('fs'); -const csv = require('csvtojson'); -const path = require('path'); const { join } = require('path'); const { pathToFileURL } = require('url'); +const { postProcessPolice } = require('./postProcessPolice'); // URLs to fetch (even if they are local files) const query = [ - pathToFileURL(join(__dirname, 'police-contact.csv')).toString(), - pathToFileURL(join(__dirname, 'police-events.csv')).toString(), + pathToFileURL(join(__dirname, 'police-contact.csv')).toString(), // contacts is always first + pathToFileURL(join(__dirname, 'police-events.csv')).toString(), // any number of events files following ]; -const postProcess = async queryResult => { - const processedJSON = { - data: { - statistics: {}, - contacts: {}, - }, - }; - const [contact, events] = queryResult; - const contactFile = path.join(__dirname, 'pre-contact-police.csv'); - const eventsFile = path.join(__dirname, 'pre-events-police.csv'); - // Unfortunately there's no Buffer or file-string support in csvtojson, there's read and process per-line, but this is more efficient. - fs.writeFileSync(contactFile, contact, { append: false }); - fs.writeFileSync(eventsFile, events, { append: false }); - // eslint-disable-next-line no-unused-vars - const contactsJson = await csv().fromFile(contactFile); - // eslint-disable-next-line no-unused-vars - const eventsJson = await csv().fromFile(eventsFile); - // TODO: Process jsonEvents and Join data with contact info for a Facility Police Page content - return processedJSON; -}; +const postProcess = postProcessPolice; module.exports = { query, diff --git a/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-contact.csv b/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-contact.csv index da9c0203e5..c1f4e9249f 100644 --- a/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-contact.csv +++ b/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-contact.csv @@ -1,3 +1 @@ -facility_id,contact_name,contact_number -vha_011,Abc Jameson,1234567899 -vha_021,Xyz Adamson,1123456799 +VISN,Facility API ID,Contact Name,Contact Number diff --git a/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-events.csv b/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-events.csv index df790c0066..99536e981f 100644 --- a/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-events.csv +++ b/src/site/stages/build/drupal/static-data-files/vaPoliceData/police-events.csv @@ -1,7 +1 @@ -date_range,facility_id,a,b,c -2021-01-01:2021-12-31,vha_011,2,3,4 -2022-01-01:2022-12-31,vha_011,3,4,5 -2023-01-01:2023-10-10,vha_011,1,1,1 -2021-01-01:2021-12-31,vha_021,1,3,1 -2022-01-01:2022-12-31,vha_021,2,0,1 -2023-01-01:2023-10-10,vha_021,0,1,0 \ No newline at end of file +VISN,Facility API ID,Facility Name,MM/YYYY,Number of service calls (officer initiated and response to calls),Traffic and parking tickets,Non-traffic (criminal) tickets,Arrests,Complaints and investigations,Numbers of sustained allegations,Numbers of disciplinary actions diff --git a/src/site/stages/build/drupal/static-data-files/vaPoliceData/postProcessPolice.js b/src/site/stages/build/drupal/static-data-files/vaPoliceData/postProcessPolice.js new file mode 100644 index 0000000000..023d111e04 --- /dev/null +++ b/src/site/stages/build/drupal/static-data-files/vaPoliceData/postProcessPolice.js @@ -0,0 +1,95 @@ +const csv = require('csvtojson'); + +function processJsonPoliceData(unprocessedJson) { + const processedJSON = { + VISN: null, + facilityAPIId: '', + facilityName: '', + date: '', + numServiceCalls: null, + trafficParkingTickets: null, + criminalTickets: null, + arrests: null, + complaintsInvestigations: null, + sustainedAllegations: null, + disciplinaryActions: null, + }; + for (const [key, value] of Object.entries(unprocessedJson)) { + switch (key) { + case 'VISN': + processedJSON.VISN = parseInt(value, 10); + break; + case 'Facility API ID': + processedJSON.facilityAPIId = value; + break; + case 'Facility Name': + processedJSON.facilityName = value; + break; + case 'MM/YYYY': + processedJSON.date = value; + break; + case 'Number of service calls (officer initiated and response to calls)': + processedJSON.numServiceCalls = parseInt(value, 10); + break; + case 'Traffic and parking tickets': + processedJSON.trafficParkingTickets = parseInt(value, 10); + break; + case 'Non-traffic (criminal) tickets': + processedJSON.criminalTickets = parseInt(value, 10); + break; + case 'Arrests': + processedJSON.arrests = parseInt(value, 10); + break; + case 'Complaints and investigations': + processedJSON.complaintsInvestigations = parseInt(value, 10); + break; + case 'Numbers of sustained allegations': + processedJSON.sustainedAllegations = parseInt(value, 10); + break; + case 'Numbers of disciplinary actions': + processedJSON.disciplinaryActions = parseInt(value, 10); + break; + default: + break; + } + } + return processedJSON; +} + +async function postProcessPolice(queryResult) { + const processedJSON = { + data: { + statistics: {}, // keys:facilityAPIId, values: array of objects in processed format + contacts: {}, + }, + }; + const [contact, ...events] = queryResult; + if (!contact || !events || events.length === 0) { + throw new Error( + 'Police data files must have at least one contact file and one events file.', + ); + } + const contactData = await csv().fromString(contact); + const eventsData = await Promise.all( + events.map(eventsFile => csv().fromString(eventsFile)), + ); // each file has its own header, but JSON's are the same + const processedEventsData = eventsData.flat().map(processJsonPoliceData); // convert keys to usable format + for (const processedEventsDataEntry of processedEventsData) { + if (processedJSON.data.statistics[processedEventsDataEntry.facilityAPIId]) { + processedJSON.data.statistics[ + processedEventsDataEntry.facilityAPIId + ].push(processedEventsDataEntry); + } else { + processedJSON.data.statistics[processedEventsDataEntry.facilityAPIId] = [ + processedEventsDataEntry, + ]; + } + } + + processedJSON.data.contacts = contactData; + + // TODO: Process jsonEvents and Join data with contact info for a Facility Police Page content + return processedJSON; +} +module.exports.postProcessPolice = postProcessPolice; +module.exports.processJsonPoliceData = processJsonPoliceData; diff --git a/src/site/stages/build/drupal/static-data-files/vaPoliceData/pre-contact-police.csv b/src/site/stages/build/drupal/static-data-files/vaPoliceData/pre-contact-police.csv deleted file mode 100644 index da9c0203e5..0000000000 --- a/src/site/stages/build/drupal/static-data-files/vaPoliceData/pre-contact-police.csv +++ /dev/null @@ -1,3 +0,0 @@ -facility_id,contact_name,contact_number -vha_011,Abc Jameson,1234567899 -vha_021,Xyz Adamson,1123456799 diff --git a/src/site/stages/build/drupal/static-data-files/vaPoliceData/pre-events-police.csv b/src/site/stages/build/drupal/static-data-files/vaPoliceData/pre-events-police.csv deleted file mode 100644 index df790c0066..0000000000 --- a/src/site/stages/build/drupal/static-data-files/vaPoliceData/pre-events-police.csv +++ /dev/null @@ -1,7 +0,0 @@ -date_range,facility_id,a,b,c -2021-01-01:2021-12-31,vha_011,2,3,4 -2022-01-01:2022-12-31,vha_011,3,4,5 -2023-01-01:2023-10-10,vha_011,1,1,1 -2021-01-01:2021-12-31,vha_021,1,3,1 -2022-01-01:2022-12-31,vha_021,2,0,1 -2023-01-01:2023-10-10,vha_021,0,1,0 \ No newline at end of file diff --git a/src/site/stages/build/drupal/tests/police/download.unit.spec.js b/src/site/stages/build/drupal/tests/police/download.unit.spec.js new file mode 100644 index 0000000000..d8b468a775 --- /dev/null +++ b/src/site/stages/build/drupal/tests/police/download.unit.spec.js @@ -0,0 +1,75 @@ +/* eslint-disable @department-of-veterans-affairs/axe-check-required */ +import path from 'path'; +import { describe, it } from 'mocha'; +import { expect } from 'chai'; +import { processCurlDataFile } from '../../static-data-files/generate'; +import { postProcessPolice } from '../../static-data-files/vaPoliceData/postProcessPolice'; + +const { pathToFileURL } = require('url'); +const getCurlClient = require('../../static-data-files/fetchApi'); + +/* This is not a FE test but a unit test. */ +describe('process police csv files', () => { + it('should have an error if the police-contact.csv file is missing', async () => { + const client = getCurlClient({ + 'drupal-user': 'eli.selkin@va.gov', + 'drupal-password': 'drupal8', + 'drupal-max-parallel-requests': 10, + }); + expect( + processCurlDataFile( + { + description: 'Curl', + filename: 'test-police.json', + query: [ + pathToFileURL( + path.join( + __dirname, + './fixtures/non-existant-police-contact.csv', + ), + ).toString(), + pathToFileURL( + path.join(__dirname, './fixtures/non-existant-police.csv'), + ).toString(), + ], + postProcess: postProcessPolice, + }, + client, + ), + ).to.be.rejectedWith(Error); + }); + it('should process files', async () => { + const client = getCurlClient({ + 'drupal-user': 'eli.selkin@va.gov', + 'drupal-password': 'drupal8', + 'drupal-max-parallel-requests': 10, + }); + const processedDataFile = await processCurlDataFile( + { + description: 'Curl', + filename: 'test-police.json', + query: [ + pathToFileURL( + path.join(__dirname, './fixtures/police-contact.csv'), + ).toString(), + pathToFileURL( + path.join(__dirname, './fixtures/police.csv'), + ).toString(), + ], + postProcess: postProcessPolice, + }, + client, + ); + const keys = Object.keys(processedDataFile.data.data.statistics); + expect(keys).to.contain('avha_635'); + expect(keys).to.contain('avha_523A5'); + expect(processedDataFile.data.data.statistics.avha_635).to.be.an('array'); + expect(processedDataFile.data.data.statistics.avha_523A5).to.be.an('array'); + expect(processedDataFile.data.data.statistics.avha_635[0].VISN).to.be.equal( + 19, + ); // that way we know processing succeeded + expect( + processedDataFile.data.data.statistics.avha_523A5[0].VISN, + ).to.be.equal(1); + }); +}); diff --git a/src/site/stages/build/drupal/tests/police/fixtures/police-contact.csv b/src/site/stages/build/drupal/tests/police/fixtures/police-contact.csv new file mode 100644 index 0000000000..ba5cf2659e --- /dev/null +++ b/src/site/stages/build/drupal/tests/police/fixtures/police-contact.csv @@ -0,0 +1,3 @@ +VISN,Facility API ID,Contact Name,Contact Number +19,avha_635,Abc Jameson,12345689 +1,avha_523A5,Zyz Adamson,1123456799 \ No newline at end of file diff --git a/src/site/stages/build/drupal/tests/police/fixtures/police.csv b/src/site/stages/build/drupal/tests/police/fixtures/police.csv new file mode 100644 index 0000000000..96a2083620 --- /dev/null +++ b/src/site/stages/build/drupal/tests/police/fixtures/police.csv @@ -0,0 +1,3 @@ +VISN,Facility API ID,Facility Name,MM/YYYY,Number of service calls (officer initiated and response to calls),Traffic and parking tickets,Non-traffic (criminal) tickets,Arrests,Complaints and investigations,Numbers of sustained allegations,Numbers of disciplinary actions +19,avha_635,Oklahoma City VA Medical Center,12/2023,4000,100,100,600,50,10,2 +1,avha_523A5,Brockton VA Medical Center,12/2023,5200,220,325,752,78,8,3 \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 30c3f7df43..c4d363a556 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2777,9 +2777,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30000981, caniuse-lite@^1.0.30001109, caniuse-lite@^1.0.30001252, caniuse-lite@^1.0.30001254: - version "1.0.30001259" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001259.tgz#ae21691d3da9c4be6144403ac40f71d9f6efd790" - integrity sha512-V7mQTFhjITxuk9zBpI6nYsiTXhcPe05l+364nZjK7MFK/E7ibvYBSAXr4YcA6oPR8j3ZLM/LN+lUqUVAQEUZFg== + version "1.0.30001566" + resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001566.tgz" + integrity sha512-ggIhCsTxmITBAMmK8yZjEhCO5/47jKXPu6Dha/wuCS4JePVL+3uiDEBuhu2aIoT+bqTOR8L76Ip1ARL9xYsEJA== caseless@~0.12.0: version "0.12.0"