From 000067a50d5d4537ec6cc546123016eba2229fe8 Mon Sep 17 00:00:00 2001 From: Austin Tisdale Date: Mon, 16 Dec 2024 11:17:36 -0500 Subject: [PATCH 1/2] Adds long PAN functionality to send-pan task --- packages/api/lib/pdrHelpers.js | 35 +++++++++++++ packages/api/tests/lib/test-pdrHelpers.js | 63 ++++++++++++++++++++++- tasks/pdr-status-check/README.md | 9 ++++ tasks/pdr-status-check/index.js | 2 +- tasks/send-pan/schemas/config.json | 5 ++ tasks/send-pan/src/index.ts | 30 +++++++++-- tasks/send-pan/src/types.ts | 1 + 7 files changed, 140 insertions(+), 5 deletions(-) diff --git a/packages/api/lib/pdrHelpers.js b/packages/api/lib/pdrHelpers.js index 81d0dd6412b..a161e0b7ef5 100644 --- a/packages/api/lib/pdrHelpers.js +++ b/packages/api/lib/pdrHelpers.js @@ -1,6 +1,7 @@ 'use strict'; const pvl = require('@cumulus/pvl'); +const { getExecution } = require('@cumulus/api-client/executions'); /** * Generate Short PAN message @@ -17,6 +18,38 @@ function generateShortPAN(disposition) { ); } +async function getGranuleFromExecution(executionArn) { + const excObj = await getExecution({ + prefix: process.env.stackName, + arn: executionArn, + }); + return excObj.originalPayload.granules[0]; +} + +/** + * Generate Long PAN message + * + * @param {Object|string[]} executions - List of workflow executions + * @returns {string} the PAN message + */ +async function generateLongPAN(executions) { + const timeStamp = new Date(); + + const longPan = new pvl.models.PVLRoot() + .add('MESSAGE_TYPE', new pvl.models.PVLTextString('LONGPAN')) + .add('NO_OF_FILES', new pvl.models.PVLNumeric(executions.length)); + /* eslint-disable no-await-in-loop */ + for (const exc of executions) { + const granule = await getGranuleFromExecution(exc.arn || exc); + longPan.add('FILE_DIRECTORY', new pvl.models.PVLTextString(granule.files[0].path)); + longPan.add('FILE_NAME', new pvl.models.PVLTextString(granule.granuleId)); + longPan.add('DISPOSITION', new pvl.models.PVLTextString(exc.reason || 'SUCCESSFUL')); + longPan.add('TIME_STAMP', new pvl.models.PVLDateTime(timeStamp)); + } + /* eslint-enable no-await-in-loop */ + return pvl.jsToPVL(longPan); +} + /** * Generate a PDRD message with a given err * @@ -33,5 +66,7 @@ function generatePDRD(err) { module.exports = { generateShortPAN, + generateLongPAN, generatePDRD, + getGranuleFromExecution, }; diff --git a/packages/api/tests/lib/test-pdrHelpers.js b/packages/api/tests/lib/test-pdrHelpers.js index 3e421f2c89c..fc791400137 100644 --- a/packages/api/tests/lib/test-pdrHelpers.js +++ b/packages/api/tests/lib/test-pdrHelpers.js @@ -1,12 +1,61 @@ 'use strict'; const test = require('ava'); -const pdrHelpers = require('../../lib/pdrHelpers'); +const proxyquire = require('proxyquire'); + +const fakeExecutionModule = { + getExecution: () => Promise.resolve({ + originalPayload: { + granules: [ + { + files: [ + { + name: 'test_id.nc', + path: 'test', + }, + ], + granuleId: 'test_id', + }, + ], + }, + }), +}; + +const pdrHelpers = proxyquire( + '../../lib/pdrHelpers', + { + '@cumulus/api-client/executions': fakeExecutionModule, + } +); // eslint-disable-next-line max-len const regex = /MESSAGE_TYPE = "SHORTPAN";\nDISPOSITION = "SUCCESSFUL";\nTIME_STAMP = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z;\n/; // eslint-disable-next-line max-len const emptyRegex = /MESSAGE_TYPE = "SHORTPAN";\nDISPOSITION = "";\nTIME_STAMP = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z;\n/; +const longPanRegex = new RegExp( + 'MESSAGE_TYPE = "LONGPAN";\\n' + + 'NO_OF_FILES = 5;\\n' + + 'FILE_DIRECTORY = "test";\\n' + + 'FILE_NAME = "test_id";\\n' + + 'DISPOSITION = "FAILED A";\\n' + + 'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' + + 'FILE_DIRECTORY = "test";\\n' + + 'FILE_NAME = "test_id";\\n' + + 'DISPOSITION = "FAILED B";\\n' + + 'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' + + 'FILE_DIRECTORY = "test";\\n' + + 'FILE_NAME = "test_id";\\n' + + 'DISPOSITION = "FAILED C";\\n' + + 'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' + + 'FILE_DIRECTORY = "test";\\n' + + 'FILE_NAME = "test_id";\\n' + + 'DISPOSITION = "SUCCESSFUL";\\n' + + 'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' + + 'FILE_DIRECTORY = "test";\\n' + + 'FILE_NAME = "test_id";\\n' + + 'DISPOSITION = "SUCCESSFUL";\\n' + + 'TIME_STAMP = \\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}Z;\\n' +); test('generateShortPAN with a disposition', (t) => { const disposition = 'SUCCESSFUL'; @@ -19,3 +68,15 @@ test('generateShortPAN with an empty disposition', (t) => { const pan = pdrHelpers.generateShortPAN(disposition); t.regex(pan, emptyRegex); }); + +test('generateLongPAN', async (t) => { + const executions = [ + { arn: 'arn:failed:execution', reason: 'FAILED A' }, + { arn: 'arn:failed:execution', reason: 'FAILED B' }, + { arn: 'arn:failed:execution', reason: 'FAILED C' }, + 'arn:completed:execution', + 'arn:completed:execution', + ]; + const pan = await pdrHelpers.generateLongPAN(executions); + t.regex(pan, longPanRegex); +}); diff --git a/tasks/pdr-status-check/README.md b/tasks/pdr-status-check/README.md index e33af2e739a..6b2bc5adf78 100644 --- a/tasks/pdr-status-check/README.md +++ b/tasks/pdr-status-check/README.md @@ -3,6 +3,15 @@ Lambda function handler for checking the status of a workflow (step function) execution. Expects a payload object which includes the name of a PDR. The concurrency of SFN API calls is set to 10 by default, and it's configurable by setting the Lambda environment variable CONCURRENCY. +Make sure the line: `"ErrorPath": "$.exception.Cause"` is added to your workflow failed task inside your ingest granule workflow to ensure the error is properly propagated to this task +```json +"WorkflowFailed": { + "Type": "Fail", + "Cause": "Workflow failed", + "ErrorPath": "$.exception.Cause" +}, +``` + ## About Cumulus Cumulus is a cloud-based data ingest, archive, distribution and management prototype for NASA's future Earth science data streams. diff --git a/tasks/pdr-status-check/index.js b/tasks/pdr-status-check/index.js index 567ae0d5da4..d7665514a93 100644 --- a/tasks/pdr-status-check/index.js +++ b/tasks/pdr-status-check/index.js @@ -91,7 +91,7 @@ function buildOutput(event, groupedExecutions) { const parseFailedExecution = (execution) => { let reason = 'Workflow Failed'; - if (execution.output) reason = JSON.parse(execution.output).exception; + if (execution.error) reason = execution.error; return { arn: execution.executionArn, reason }; }; diff --git a/tasks/send-pan/schemas/config.json b/tasks/send-pan/schemas/config.json index 3b3ece89e3c..e8edda33c94 100644 --- a/tasks/send-pan/schemas/config.json +++ b/tasks/send-pan/schemas/config.json @@ -12,6 +12,11 @@ "type": ["string", "null"], "description": "The path in the provider to upload the file to.", "default": "pans" + }, + "panType": { + "type": ["string", "null"], + "description": "Determines which pan type to create: (shortPan, longPan, or longPanAlways)", + "default": "shortPan" } } } diff --git a/tasks/send-pan/src/index.ts b/tasks/send-pan/src/index.ts index c6d06d25570..c1cccda37a1 100644 --- a/tasks/send-pan/src/index.ts +++ b/tasks/send-pan/src/index.ts @@ -41,6 +41,7 @@ async function sendPAN(event: HandlerEvent): Promise { const { config, input } = event; const provider = config.provider; const remoteDir = config.remoteDir || 'pans'; + const panType = config.panType || 'shortPan'; const panName = input.pdr.name.replace(/\.pdr/gi, '.pan'); const uploadPath = path.join(remoteDir, panName); @@ -48,9 +49,32 @@ async function sendPAN(event: HandlerEvent): Promise { if (input.running.length !== 0) { throw new Error('Executions still running'); } - - const disposition = (input.failed.length > 0) ? 'FAILED' : 'SUCCESSFUL'; - const pan = pdrHelpers.generateShortPAN(disposition); + let pan; + switch (panType) { + case 'longPanAlways': + pan = await pdrHelpers.generateLongPAN([...input.completed, ...input.failed]); + log.debug('Created long PAN'); + break; + case 'shortPan': { + const disposition = (input.failed.length > 0) ? 'FAILED' : 'SUCCESSFUL'; + pan = pdrHelpers.generateShortPAN(disposition); + log.debug('Created short PAN'); + break; + } + case 'longPan': { + if (input.failed.length + input.completed.length <= 1) { + const disposition = (input.failed.length > 0) ? 'FAILED' : 'SUCCESSFUL'; + pan = pdrHelpers.generateShortPAN(disposition); + log.debug('Created short PAN'); + } else { + pan = await pdrHelpers.generateLongPAN([...input.completed, ...input.failed]); + log.debug('Created long PAN'); + } + break; + } + default: + throw new Error(`Unknown panType: ${panType}, must be shortPan, longPan, or longPanAlways`); + } const localPath = path.join(tmpdir(), panName); fs.writeFileSync(localPath, pan); diff --git a/tasks/send-pan/src/types.ts b/tasks/send-pan/src/types.ts index e6a8ab1e7b9..c2976111a3c 100644 --- a/tasks/send-pan/src/types.ts +++ b/tasks/send-pan/src/types.ts @@ -32,6 +32,7 @@ export type HandlerEvent = { host: string, }, remoteDir: string | null, + panType: string | null }, input: HandlerInput, }; From 8366826fed74ab3ea0db1b7efc91ad058bef9401 Mon Sep 17 00:00:00 2001 From: Austin Tisdale Date: Thu, 26 Dec 2024 12:00:07 -0500 Subject: [PATCH 2/2] Makes PAN file extension uppercase --- tasks/send-pan/src/index.ts | 2 +- tasks/send-pan/tests/index.js | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tasks/send-pan/src/index.ts b/tasks/send-pan/src/index.ts index c1cccda37a1..b1840048805 100644 --- a/tasks/send-pan/src/index.ts +++ b/tasks/send-pan/src/index.ts @@ -43,7 +43,7 @@ async function sendPAN(event: HandlerEvent): Promise { const remoteDir = config.remoteDir || 'pans'; const panType = config.panType || 'shortPan'; - const panName = input.pdr.name.replace(/\.pdr/gi, '.pan'); + const panName = input.pdr.name.replace(/\.pdr/gi, '.PAN'); const uploadPath = path.join(remoteDir, panName); if (input.running.length !== 0) { diff --git a/tasks/send-pan/tests/index.js b/tasks/send-pan/tests/index.js index 045e08bef0b..69095165e6a 100644 --- a/tasks/send-pan/tests/index.js +++ b/tasks/send-pan/tests/index.js @@ -52,7 +52,7 @@ test('SendPan task calls upload', async (t) => { }; const url = `http://${event.config.provider.host}:${port}`; - const remotePath = path.join(event.config.remoteDir, `${fileNameBase}.pan`); + const remotePath = path.join(event.config.remoteDir, `${fileNameBase}.PAN`); // Message should look like this: // MESSAGE_TYPE = "SHORTPAN"; // DISPOSITION = "SUCCESSFUL"; @@ -103,7 +103,7 @@ test('SendPan task sends PAN to HTTP server', async (t) => { test('SendPan task sends PAN to s3', async (t) => { const remoteDir = 'pan/remote-dir'; const fileNameBase = 'test-send-s3-pdr'; - const uploadPath = path.join(remoteDir, `${fileNameBase}.pan`); + const uploadPath = path.join(remoteDir, `${fileNameBase}.PAN`); const event = { config: { provider: { @@ -175,7 +175,7 @@ test('SendPan task throws error when provider protocol is not supported', async test('SendPan task sends PAN to default location when remoteDir is null', async (t) => { const fileNameBase = 'test-default-pan-path-pdr'; - const uploadPath = `pans/${fileNameBase}.pan`; + const uploadPath = `pans/${fileNameBase}.PAN`; const event = { config: { provider: { @@ -248,7 +248,7 @@ test('SendPan task fails with executions still running', async (t) => { test('SendPan task sends failed PAN to s3', async (t) => { const fileNameBase = 'test-failed-pan-path-pdr'; - const uploadPath = `pans/${fileNameBase}.pan`; + const uploadPath = `pans/${fileNameBase}.PAN`; const event = { config: { provider: {