From c021af0ce5bbd886b8f024e4036ec458aba0f913 Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Mon, 4 Nov 2024 13:57:59 -0500 Subject: [PATCH 1/2] remaking PR --- CHANGELOG.md | 2 + packages/api/bin/serve.js | 21 ------ packages/api/endpoints/elasticsearch.js | 1 - packages/api/lambdas/index-from-database.js | 67 ++----------------- packages/api/models/index.js | 2 +- .../api/tests/endpoints/test-elasticsearch.js | 2 - .../tests/lambdas/test-index-from-database.js | 43 ++++-------- tf-modules/archive/api.tf | 2 - tf-modules/archive/index_from_database.tf | 1 - tf-modules/data-persistence/dynamo.tf | 25 ------- tf-modules/data-persistence/outputs.tf | 4 -- 11 files changed, 22 insertions(+), 148 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be8416bcd21..7c7fbefd43f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,8 @@ aws lambda invoke --function-name $PREFIX-ReconciliationReportMigration $OUTFILE - **CUMULUS-3718** - Updated `reconciliation_reports` list api endpoint and added `ReconciliationReportSearch` class to query postgres - Added `reconciliationReports` type to stats endpoint, so `aggregate` query will work for reconciliation reports +- **CUMULUS-3842** + - Remove reconciliationReports DynamoDB table - **CUMULUS-3859** - Updated `@cumulus/api/bin/serveUtils` to no longer add records to ElasticSearch diff --git a/packages/api/bin/serve.js b/packages/api/bin/serve.js index 591f04ae227..72ca460c9cd 100644 --- a/packages/api/bin/serve.js +++ b/packages/api/bin/serve.js @@ -20,9 +20,6 @@ const { const { constructCollectionId } = require('@cumulus/message/Collections'); const { bootstrapElasticSearch } = require('@cumulus/es-client/bootstrap'); - -const { ReconciliationReport } = require('../models'); - const testUtils = require('../lib/testUtils'); const serveUtils = require('./serveUtils'); const { @@ -249,24 +246,6 @@ async function serveApi(user, stackName = localStackName, reseed = true) { checkEnvVariablesAreSet(requiredEnvVars); - // Create reconciliation report table - const reconciliationReportTableName = `${stackName}-ReconciliationReportsTable`; - process.env.ReconciliationReportsTable = reconciliationReportTableName; - const reconciliationReportModel = new ReconciliationReport({ - tableName: reconciliationReportTableName, - stackName: process.env.stackName, - systemBucket: process.env.system_bucket, - }); - try { - await reconciliationReportModel.createTable(); - } catch (error) { - if (error && error.name && error.name === 'ResourceInUseException') { - console.log(`${reconciliationReportTableName} is already created`); - } else { - throw error; - } - } - await prepareServices(stackName, process.env.system_bucket); await populateBucket(process.env.system_bucket, stackName); if (reseed) { diff --git a/packages/api/endpoints/elasticsearch.js b/packages/api/endpoints/elasticsearch.js index b09bb81572e..a8d79a2ac3c 100644 --- a/packages/api/endpoints/elasticsearch.js +++ b/packages/api/endpoints/elasticsearch.js @@ -194,7 +194,6 @@ async function indexFromDatabase(req, res) { operationType: 'ES Index', payload: { indexName, - reconciliationReportsTable: process.env.ReconciliationReportsTable, esHost: process.env.ES_HOST, esRequestConcurrency: esRequestConcurrency || process.env.ES_CONCURRENCY, postgresResultPageSize, diff --git a/packages/api/lambdas/index-from-database.js b/packages/api/lambdas/index-from-database.js index e0666992a18..7f5c3910866 100644 --- a/packages/api/lambdas/index-from-database.js +++ b/packages/api/lambdas/index-from-database.js @@ -3,7 +3,6 @@ const isNil = require('lodash/isNil'); const pLimit = require('p-limit'); -const DynamoDbSearchQueue = require('@cumulus/aws-client/DynamoDbSearchQueue'); const log = require('@cumulus/common/log'); const { getEsClient } = require('@cumulus/es-client/search'); @@ -13,6 +12,7 @@ const { AsyncOperationPgModel, GranulePgModel, ProviderPgModel, + ReconciliationReportPgModel, RulePgModel, PdrPgModel, getKnexClient, @@ -22,6 +22,7 @@ const { translatePostgresGranuleToApiGranule, translatePostgresProviderToApiProvider, translatePostgresPdrToApiPdr, + translatePostgresReconReportToApiReconReport, translatePostgresRuleToApiRule, } = require('@cumulus/db'); const indexer = require('@cumulus/es-client/indexer'); @@ -60,62 +61,6 @@ const getEsRequestConcurrency = (event) => { return 10; }; -// Legacy method used for indexing Reconciliation Reports only -async function indexReconciliationReports({ - esClient, - tableName, - esIndex, - indexFn, - limitEsRequests, -}) { - const scanQueue = new DynamoDbSearchQueue({ - TableName: tableName, - }); - - let itemsComplete = false; - let totalItemsIndexed = 0; - - /* eslint-disable no-await-in-loop */ - while (itemsComplete === false) { - await scanQueue.fetchItems(); - - itemsComplete = scanQueue.items[scanQueue.items.length - 1] === null; - - if (itemsComplete) { - // pop the null item off - scanQueue.items.pop(); - } - - if (scanQueue.items.length === 0) { - log.info(`No records to index for ${tableName}`); - return true; - } - - log.info(`Attempting to index ${scanQueue.items.length} records from ${tableName}`); - - const input = scanQueue.items.map( - (item) => limitEsRequests( - async () => { - try { - return await indexFn(esClient, item, esIndex); - } catch (error) { - log.error(`Error indexing record ${JSON.stringify(item)}, error: ${error}`); - return false; - } - } - ) - ); - const results = await Promise.all(input); - const successfulResults = results.filter((result) => result !== false); - totalItemsIndexed += successfulResults; - - log.info(`Completed index of ${successfulResults.length} records from ${tableName}`); - } - /* eslint-enable no-await-in-loop */ - - return totalItemsIndexed; -} - /** * indexModel - Index a postgres RDS table's contents to ElasticSearch * @@ -199,7 +144,6 @@ async function indexFromDatabase(event) { const { indexName: esIndex, esHost = process.env.ES_HOST, - reconciliationReportsTable = process.env.ReconciliationReportsTable, postgresResultPageSize, postgresConnectionPoolSize, } = event; @@ -286,12 +230,15 @@ async function indexFromDatabase(event) { knex, pageSize, }), - indexReconciliationReports({ + indexModel({ esClient, - tableName: reconciliationReportsTable, esIndex, indexFn: indexer.indexReconciliationReport, limitEsRequests, + postgresModel: new ReconciliationReportPgModel(), + translationFunction: translatePostgresReconReportToApiReconReport, + knex, + pageSize, }), indexModel({ esClient, diff --git a/packages/api/models/index.js b/packages/api/models/index.js index dd11c59c94b..c2265b57045 100644 --- a/packages/api/models/index.js +++ b/packages/api/models/index.js @@ -6,6 +6,6 @@ const ReconciliationReport = require('./reconciliation-reports'); module.exports = { AccessToken, - ReconciliationReport, Manager, + ReconciliationReport, }; diff --git a/packages/api/tests/endpoints/test-elasticsearch.js b/packages/api/tests/endpoints/test-elasticsearch.js index 811587e93c3..4bb1f6d0591 100644 --- a/packages/api/tests/endpoints/test-elasticsearch.js +++ b/packages/api/tests/endpoints/test-elasticsearch.js @@ -550,7 +550,6 @@ test.serial('Reindex from database - startAsyncOperation is called with expected const indexName = randomString(); const processEnv = { ...process.env }; process.env.ES_HOST = 'fakeEsHost'; - process.env.ReconciliationReportsTable = 'fakeReportsTable'; const asyncOperationsStub = sinon.stub(startAsyncOperation, 'invokeStartAsyncOperationLambda'); const payload = { @@ -572,7 +571,6 @@ test.serial('Reindex from database - startAsyncOperation is called with expected t.deepEqual(asyncOperationsStub.getCall(0).args[0].payload, { ...payload, esHost: process.env.ES_HOST, - reconciliationReportsTable: process.env.ReconciliationReportsTable, }); } finally { process.env = processEnv; diff --git a/packages/api/tests/lambdas/test-index-from-database.js b/packages/api/tests/lambdas/test-index-from-database.js index 8e647736154..ea404ccceba 100644 --- a/packages/api/tests/lambdas/test-index-from-database.js +++ b/packages/api/tests/lambdas/test-index-from-database.js @@ -21,59 +21,39 @@ const { fakeCollectionRecordFactory, fakeExecutionRecordFactory, fakeGranuleRecordFactory, + fakeReconciliationReportRecordFactory, fakePdrRecordFactory, fakeProviderRecordFactory, generateLocalTestDb, GranulePgModel, migrationDir, + ReconciliationReportPgModel, PdrPgModel, ProviderPgModel, translatePostgresCollectionToApiCollection, translatePostgresExecutionToApiExecution, translatePostgresGranuleToApiGranule, + translatePostgresReconReportToApiReconReport, translatePostgresPdrToApiPdr, translatePostgresProviderToApiProvider, } = require('@cumulus/db'); -const { - fakeReconciliationReportFactory, -} = require('../../lib/testUtils'); - -const models = require('../../models'); const indexFromDatabase = require('../../lambdas/index-from-database'); const { getWorkflowList, } = require('../../lib/testUtils'); const workflowList = getWorkflowList(); -process.env.ReconciliationReportsTable = randomString(); -const reconciliationReportModel = new models.ReconciliationReport(); // create all the variables needed across this test process.env.system_bucket = randomString(); process.env.stackName = randomString(); -const reconciliationReportsTable = process.env.ReconciliationReportsTable; - function sortAndFilter(input, omitList, sortKey) { return input.map((r) => omit(r, omitList)) .sort((a, b) => (a[sortKey] > b[sortKey] ? 1 : -1)); } -async function addFakeDynamoData(numItems, factory, model, factoryParams = {}) { - const items = []; - - /* eslint-disable no-await-in-loop */ - for (let i = 0; i < numItems; i += 1) { - const item = factory(factoryParams); - items.push(item); - await model.create(item); - } - /* eslint-enable no-await-in-loop */ - - return items; -} - async function addFakeData(knex, numItems, factory, model, factoryParams = {}) { const items = []; for (let i = 0; i < numItems; i += 1) { @@ -92,7 +72,6 @@ test.before(async (t) => { t.context.esIndices = []; await awsServices.s3().createBucket({ Bucket: process.env.system_bucket }); - await reconciliationReportModel.createTable(); const wKey = `${process.env.stackName}/workflows/${workflowList[0].name}.json`; const tKey = `${process.env.stackName}/workflow_template.json`; @@ -220,7 +199,6 @@ test('No error is thrown if nothing is in the database', async (t) => { await t.notThrowsAsync(() => indexFromDatabase.indexFromDatabase({ indexName: esAlias, - reconciliationReportsTable, knex, })); }); @@ -272,10 +250,12 @@ test.serial('Lambda successfully indexes records of all types', async (t) => { ...dateObject, }); - const fakeReconciliationReportRecords = await addFakeDynamoData( + const fakeReconciliationReportRecords = await addFakeData( + knex, numItems, - fakeReconciliationReportFactory, - reconciliationReportModel + fakeReconciliationReportRecordFactory, + new ReconciliationReportPgModel(), + dateObject ); await indexFromDatabase.handler({ @@ -309,6 +289,9 @@ test.serial('Lambda successfully indexes records of all types', async (t) => { knexOrTransaction: knex, })) ); + const reconciliationReportResults = await Promise.all( + fakeReconciliationReportRecords.map((r) => translatePostgresReconReportToApiReconReport(r)) + ); const pdrResults = await Promise.all( fakePdrRecords.map((r) => translatePostgresPdrToApiPdr(r, knex)) ); @@ -346,7 +329,7 @@ test.serial('Lambda successfully indexes records of all types', async (t) => { t.deepEqual( sortAndFilter(searchResults[5].results, ['timestamp'], 'name'), - sortAndFilter(fakeReconciliationReportRecords, ['timestamp'], 'name') + sortAndFilter(reconciliationReportResults, ['timestamp'], 'name') ); }); @@ -386,7 +369,6 @@ test.serial('failure in indexing record of specific type should not prevent inde try { await indexFromDatabase.handler({ indexName: esAlias, - reconciliationReportsTable, knex, }); @@ -463,7 +445,6 @@ test.serial( try { await indexFromDatabase.handler({ indexName: esAlias, - reconciliationReportsTable, knex, }); diff --git a/tf-modules/archive/api.tf b/tf-modules/archive/api.tf index 2e27fe85885..1336a5d7616 100644 --- a/tf-modules/archive/api.tf +++ b/tf-modules/archive/api.tf @@ -3,7 +3,6 @@ resource "aws_ssm_parameter" "dynamo_table_names" { type = "String" value = jsonencode({ AccessTokensTable = var.dynamo_tables.access_tokens.name - ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name }) } locals { @@ -13,7 +12,6 @@ locals { api_redirect_uri = "${local.api_uri}token" dynamo_table_namestring = jsonencode({ AccessTokensTable = var.dynamo_tables.access_tokens.name - ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name }) api_env_variables = { auth_mode = "public" diff --git a/tf-modules/archive/index_from_database.tf b/tf-modules/archive/index_from_database.tf index 61138dd4664..53889c1b049 100644 --- a/tf-modules/archive/index_from_database.tf +++ b/tf-modules/archive/index_from_database.tf @@ -14,7 +14,6 @@ resource "aws_lambda_function" "index_from_database" { databaseCredentialSecretArn = var.rds_user_access_secret_arn ES_CONCURRENCY = var.es_request_concurrency ES_HOST = var.elasticsearch_hostname - ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name stackName = var.prefix } } diff --git a/tf-modules/data-persistence/dynamo.tf b/tf-modules/data-persistence/dynamo.tf index 6f6f800e07b..3a0696e9947 100644 --- a/tf-modules/data-persistence/dynamo.tf +++ b/tf-modules/data-persistence/dynamo.tf @@ -2,7 +2,6 @@ locals { enable_point_in_time_table_names = [for x in var.enable_point_in_time_tables : "${var.prefix}-${x}"] table_names = { access_tokens_table = "${var.prefix}-AccessTokensTable" - reconciliation_reports_table = "${var.prefix}-ReconciliationReportsTable" semaphores_table = "${var.prefix}-SemaphoresTable" } } @@ -34,30 +33,6 @@ resource "aws_dynamodb_table" "access_tokens_table" { tags = var.tags } -resource "aws_dynamodb_table" "reconciliation_reports_table" { - name = local.table_names.reconciliation_reports_table - billing_mode = "PAY_PER_REQUEST" - hash_key = "name" - stream_enabled = true - stream_view_type = "NEW_AND_OLD_IMAGES" - - attribute { - name = "name" - type = "S" - } - - point_in_time_recovery { - enabled = contains(local.enable_point_in_time_table_names, local.table_names.reconciliation_reports_table) - } - - lifecycle { - prevent_destroy = true - ignore_changes = [ name ] - } - - tags = var.tags -} - resource "aws_dynamodb_table" "semaphores_table" { name = local.table_names.semaphores_table billing_mode = "PAY_PER_REQUEST" diff --git a/tf-modules/data-persistence/outputs.tf b/tf-modules/data-persistence/outputs.tf index 38313398d6d..540e07445fa 100644 --- a/tf-modules/data-persistence/outputs.tf +++ b/tf-modules/data-persistence/outputs.tf @@ -4,10 +4,6 @@ output "dynamo_tables" { name = aws_dynamodb_table.access_tokens_table.name, arn = aws_dynamodb_table.access_tokens_table.arn } - reconciliation_reports = { - name = aws_dynamodb_table.reconciliation_reports_table.name - arn = aws_dynamodb_table.reconciliation_reports_table.arn - } semaphores = { name = aws_dynamodb_table.semaphores_table.name arn = aws_dynamodb_table.semaphores_table.arn From e95441234890e29296a34b841f791e4780dbdb0f Mon Sep 17 00:00:00 2001 From: Naga Nages Date: Sun, 1 Dec 2024 22:09:16 -0500 Subject: [PATCH 2/2] removing 3833 migration work for deletion to pass --- .../.nycrc.json | 7 - .../reconciliation-report-migration/README.md | 18 -- .../reconciliation-report-migration/iam.tf | 70 ----- .../reconciliation-report-migration/main.tf | 35 --- .../outputs.tf | 3 - .../package.json | 45 ---- .../src/index.ts | 24 -- .../src/reconciliation-reports.ts | 88 ------- .../src/types.ts | 10 - .../tests/test-index.js | 104 -------- .../tests/test-reconciliation-reports.js | 245 ------------------ .../tsconfig.json | 11 - .../variables.tf | 59 ----- .../versions.tf | 9 - .../webpack.config.js | 53 ---- .../reconciliation_report_migration.tf | 21 -- 16 files changed, 802 deletions(-) delete mode 100644 lambdas/reconciliation-report-migration/.nycrc.json delete mode 100644 lambdas/reconciliation-report-migration/README.md delete mode 100644 lambdas/reconciliation-report-migration/iam.tf delete mode 100644 lambdas/reconciliation-report-migration/main.tf delete mode 100644 lambdas/reconciliation-report-migration/outputs.tf delete mode 100644 lambdas/reconciliation-report-migration/package.json delete mode 100644 lambdas/reconciliation-report-migration/src/index.ts delete mode 100644 lambdas/reconciliation-report-migration/src/reconciliation-reports.ts delete mode 100644 lambdas/reconciliation-report-migration/src/types.ts delete mode 100644 lambdas/reconciliation-report-migration/tests/test-index.js delete mode 100644 lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js delete mode 100644 lambdas/reconciliation-report-migration/tsconfig.json delete mode 100644 lambdas/reconciliation-report-migration/variables.tf delete mode 100644 lambdas/reconciliation-report-migration/versions.tf delete mode 100644 lambdas/reconciliation-report-migration/webpack.config.js delete mode 100644 tf-modules/cumulus/reconciliation_report_migration.tf diff --git a/lambdas/reconciliation-report-migration/.nycrc.json b/lambdas/reconciliation-report-migration/.nycrc.json deleted file mode 100644 index d7000c7c12b..00000000000 --- a/lambdas/reconciliation-report-migration/.nycrc.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../nyc.config.js", - "lines": 95.0, - "branches": 80.0, - "statements": 95.0, - "functions": 98.0 -} \ No newline at end of file diff --git a/lambdas/reconciliation-report-migration/README.md b/lambdas/reconciliation-report-migration/README.md deleted file mode 100644 index b117ad7a914..00000000000 --- a/lambdas/reconciliation-report-migration/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# ReconciliationReportMigration Lambda - -The lambda migrates existing ReconciliationReports data from DynamoDB to PostgreSQL. - -To invoke the Lambda and start the ReconciliationReport migration, you can use the AWS Console or CLI: - -```bash -aws lambda invoke --function-name $PREFIX-ReconciliationReportMigration $OUTFILE -``` - -- `PREFIX` is your Cumulus deployment prefix. -- `OUTFILE` (**optional**) is the filepath where the Lambda output will be saved. - -The result will be a migration summary. For example: - -``` -{"reconciliation_reports":{"total_dynamo_db_records":36,"migrated":36,"failed":0,"skipped":0}} -``` diff --git a/lambdas/reconciliation-report-migration/iam.tf b/lambdas/reconciliation-report-migration/iam.tf deleted file mode 100644 index 3fae7f2bd89..00000000000 --- a/lambdas/reconciliation-report-migration/iam.tf +++ /dev/null @@ -1,70 +0,0 @@ -data "aws_iam_policy_document" "lambda_assume_role_policy" { - statement { - actions = ["sts:AssumeRole"] - principals { - type = "Service" - identifiers = ["lambda.amazonaws.com"] - } - } -} - -resource "aws_iam_role" "reconciliation_report_migration" { - name = "${var.prefix}-reconciliation-report-migration" - assume_role_policy = data.aws_iam_policy_document.lambda_assume_role_policy.json - permissions_boundary = var.permissions_boundary_arn - - tags = var.tags -} - -data "aws_iam_policy_document" "reconciliation_report_migration" { - statement { - actions = [ - "ec2:CreateNetworkInterface", - "ec2:DeleteNetworkInterface", - "ec2:DescribeNetworkInterfaces", - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogStreams", - "logs:PutLogEvents" - ] - resources = ["*"] - } - - statement { - actions = [ - "dynamodb:Scan", - ] - resources = [ - var.dynamo_tables.reconciliation_reports.arn, - ] - } - - statement { - actions = [ - "secretsmanager:GetSecretValue" - ] - resources = [var.rds_user_access_secret_arn] - } -} - -resource "aws_iam_role_policy" "reconciliation_report_migration" { - name = "${var.prefix}_reconciliation_report_migration" - role = aws_iam_role.reconciliation_report_migration.id - policy = data.aws_iam_policy_document.reconciliation_report_migration.json -} - -resource "aws_security_group" "reconciliation_report_migration" { - count = length(var.lambda_subnet_ids) == 0 ? 0 : 1 - - name = "${var.prefix}-reconciliation-report-migration" - vpc_id = var.vpc_id - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - tags = var.tags -} diff --git a/lambdas/reconciliation-report-migration/main.tf b/lambdas/reconciliation-report-migration/main.tf deleted file mode 100644 index bb7e543fe16..00000000000 --- a/lambdas/reconciliation-report-migration/main.tf +++ /dev/null @@ -1,35 +0,0 @@ -locals { - lambda_path = "${path.module}/dist/webpack/lambda.zip" -} - -resource "aws_lambda_function" "reconciliation_report_migration" { - function_name = "${var.prefix}-ReconciliationReportMigration" - filename = local.lambda_path - source_code_hash = filebase64sha256(local.lambda_path) - handler = "index.handler" - role = aws_iam_role.reconciliation_report_migration.arn - runtime = "nodejs20.x" - timeout = lookup(var.lambda_timeouts, "ReconciliationReportMigration", 900) - memory_size = lookup(var.lambda_memory_sizes, "ReconciliationReportMigration", 1024) - - environment { - variables = { - databaseCredentialSecretArn = var.rds_user_access_secret_arn - ReconciliationReportsTable = var.dynamo_tables.reconciliation_reports.name - stackName = var.prefix - } - } - - dynamic "vpc_config" { - for_each = length(var.lambda_subnet_ids) == 0 ? [] : [1] - content { - subnet_ids = var.lambda_subnet_ids - security_group_ids = compact([ - aws_security_group.reconciliation_report_migration[0].id, - var.rds_security_group_id - ]) - } - } - - tags = var.tags -} diff --git a/lambdas/reconciliation-report-migration/outputs.tf b/lambdas/reconciliation-report-migration/outputs.tf deleted file mode 100644 index 122c24f1abc..00000000000 --- a/lambdas/reconciliation-report-migration/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "reconciliation_report_migration_function_arn" { - value = aws_lambda_function.reconciliation_report_migration.arn -} diff --git a/lambdas/reconciliation-report-migration/package.json b/lambdas/reconciliation-report-migration/package.json deleted file mode 100644 index 98bcf797212..00000000000 --- a/lambdas/reconciliation-report-migration/package.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "name": "@cumulus/reconciliation-report-migration", - "version": "19.1.0", - "description": "Lambda function for reconciliation report migration from DynamoDB to Postgres", - "author": "Cumulus Authors", - "license": "Apache-2.0", - "engines": { - "node": ">=20.12.2" - }, - "private": true, - "main": "./dist/lambda/index.js", - "types": "./dist/lambda/index.d.ts", - "scripts": { - "clean": "rm -rf dist", - "build": "rm -rf dist && mkdir dist && npm run prepare && npm run webpack", - "build-lambda-zip": "cd dist/webpack && node ../../../../bin/zip.js lambda.zip index.js", - "package": "npm run clean && npm run prepare && npm run webpack && npm run build-lambda-zip", - "test": "../../node_modules/.bin/ava", - "test:ci": "../../scripts/run_package_ci_unit.sh", - "test:coverage": "../../node_modules/.bin/nyc npm test", - "prepare": "npm run tsc", - "tsc": "../../node_modules/.bin/tsc", - "tsc:listEmittedFiles": "../../node_modules/.bin/tsc --listEmittedFiles", - "webpack": "../../node_modules/.bin/webpack" - }, - "ava": { - "files": [ - "tests/**/*.js" - ], - "timeout": "15m", - "failFast": true - }, - "dependencies": { - "@cumulus/api": "19.1.0", - "@cumulus/aws-client": "19.1.0", - "@cumulus/common": "19.1.0", - "@cumulus/db": "19.1.0", - "@cumulus/errors": "19.1.0", - "@cumulus/logger": "19.1.0", - "@cumulus/types": "19.1.0", - "knex": "2.4.1", - "lodash": "^4.17.21", - "pg": "~8.12" - } -} diff --git a/lambdas/reconciliation-report-migration/src/index.ts b/lambdas/reconciliation-report-migration/src/index.ts deleted file mode 100644 index a5c394d28fa..00000000000 --- a/lambdas/reconciliation-report-migration/src/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { getKnexClient } from '@cumulus/db'; -import Logger from '@cumulus/logger'; - -import { migrateReconciliationReports } from './reconciliation-reports'; -import { MigrationSummary } from './types'; - -const logger = new Logger({ sender: '@cumulus/reconciliation-report-migration' }); - -export interface HandlerEvent { - env?: NodeJS.ProcessEnv -} - -export const handler = async (event: HandlerEvent): Promise => { - const env = event.env ?? process.env; - const knex = await getKnexClient({ env }); - - try { - const migrationSummary = await migrateReconciliationReports(env, knex); - logger.info(JSON.stringify(migrationSummary)); - return { reconciliation_reports: migrationSummary }; - } finally { - await knex.destroy(); - } -}; diff --git a/lambdas/reconciliation-report-migration/src/reconciliation-reports.ts b/lambdas/reconciliation-report-migration/src/reconciliation-reports.ts deleted file mode 100644 index 32dec7c570e..00000000000 --- a/lambdas/reconciliation-report-migration/src/reconciliation-reports.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Knex } from 'knex'; - -import { DynamoDbSearchQueue } from '@cumulus/aws-client'; -import { envUtils } from '@cumulus/common'; -import { - ReconciliationReportPgModel, - translateApiReconReportToPostgresReconReport, -} from '@cumulus/db'; -import { RecordAlreadyMigrated, RecordDoesNotExist } from '@cumulus/errors'; -import Logger from '@cumulus/logger'; -import { ApiReconciliationReportRecord } from '@cumulus/types/api/reconciliation_reports'; - -import { MigrationResult } from './types'; - -const logger = new Logger({ sender: '@cumulus/data-migration/reconciliation-reports' }); - -export const migrateReconciliationReportRecord = async ( - dynamoRecord: ApiReconciliationReportRecord, - knex: Knex -): Promise => { - const reconReportPgModel = new ReconciliationReportPgModel(); - - let existingRecord; - try { - existingRecord = await reconReportPgModel.get(knex, { name: dynamoRecord.name }); - } catch (error) { - if (!(error instanceof RecordDoesNotExist)) { - throw error; - } - } - - if (existingRecord - && dynamoRecord.updatedAt - && existingRecord.updated_at >= new Date(dynamoRecord.updatedAt)) { - throw new RecordAlreadyMigrated(`Reconciliation report ${dynamoRecord.name} was already migrated, skipping`); - } - - const updatedRecord = translateApiReconReportToPostgresReconReport( - dynamoRecord - ); - - await reconReportPgModel.upsert(knex, updatedRecord); -}; - -export const migrateReconciliationReports = async ( - env: NodeJS.ProcessEnv, - knex: Knex -): Promise => { - const reconciliationReportsTable = envUtils.getRequiredEnvVar('ReconciliationReportsTable', env); - - const searchQueue = new DynamoDbSearchQueue({ - TableName: reconciliationReportsTable, - }); - - const migrationSummary = { - total_dynamo_db_records: 0, - migrated: 0, - failed: 0, - skipped: 0, - }; - - let record = await searchQueue.peek(); - /* eslint-disable no-await-in-loop */ - while (record) { - migrationSummary.total_dynamo_db_records += 1; - - try { - await migrateReconciliationReportRecord(record as any, knex); - migrationSummary.migrated += 1; - } catch (error) { - if (error instanceof RecordAlreadyMigrated) { - migrationSummary.skipped += 1; - } else { - migrationSummary.failed += 1; - logger.error( - `Could not create reconciliationReport record in RDS for Dynamo reconciliationReport name ${record.name}:`, - error - ); - } - } - - await searchQueue.shift(); - record = await searchQueue.peek(); - } - /* eslint-enable no-await-in-loop */ - logger.info(`successfully migrated ${migrationSummary.migrated} reconciliationReport records`); - return migrationSummary; -}; diff --git a/lambdas/reconciliation-report-migration/src/types.ts b/lambdas/reconciliation-report-migration/src/types.ts deleted file mode 100644 index 08119110cae..00000000000 --- a/lambdas/reconciliation-report-migration/src/types.ts +++ /dev/null @@ -1,10 +0,0 @@ -export type MigrationResult = { - total_dynamo_db_records: number, - skipped: number, - migrated: number, - failed: number, -}; - -export type MigrationSummary = { - reconciliation_reports: MigrationResult -}; diff --git a/lambdas/reconciliation-report-migration/tests/test-index.js b/lambdas/reconciliation-report-migration/tests/test-index.js deleted file mode 100644 index b355c0b100d..00000000000 --- a/lambdas/reconciliation-report-migration/tests/test-index.js +++ /dev/null @@ -1,104 +0,0 @@ -const test = require('ava'); -const cryptoRandomString = require('crypto-random-string'); - -const ReconciliationReport = require('@cumulus/api/models/reconciliation-reports'); - -const { - createBucket, - putJsonS3Object, - recursivelyDeleteS3Bucket, -} = require('@cumulus/aws-client/S3'); - -const { - generateLocalTestDb, - destroyLocalTestDb, - localStackConnectionEnv, - migrationDir, -} = require('@cumulus/db'); - -const { handler } = require('../dist/lambda'); -const testDbName = `reconciliation_report_migration_1_${cryptoRandomString({ length: 10 })}`; -const workflow = cryptoRandomString({ length: 10 }); - -test.before(async (t) => { - process.env = { - ...process.env, - ...localStackConnectionEnv, - PG_DATABASE: testDbName, - stackName: cryptoRandomString({ length: 10 }), - system_bucket: cryptoRandomString({ length: 10 }), - ReconciliationReportsTable: cryptoRandomString({ length: 10 }), - }; - - await createBucket(process.env.system_bucket); - - const workflowfile = `${process.env.stackName}/workflows/${workflow}.json`; - const messageTemplateKey = `${process.env.stackName}/workflow_template.json`; - - t.context.reconciliationReportsModel = new ReconciliationReport({ - stackName: process.env.stackName, - systemBucket: process.env.system_bucket, - }); - - await Promise.all([ - t.context.reconciliationReportsModel.createTable(), - ]); - - await Promise.all([ - putJsonS3Object( - process.env.system_bucket, - messageTemplateKey, - { meta: 'meta' } - ), - putJsonS3Object( - process.env.system_bucket, - workflowfile, - { testworkflow: 'workflow-config' } - ), - ]); - const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); - t.context.knex = knex; - t.context.knexAdmin = knexAdmin; -}); - -test.after.always(async (t) => { - await t.context.reconciliationReportsModel.deleteTable(); - - await recursivelyDeleteS3Bucket(process.env.system_bucket); - - await destroyLocalTestDb({ - knex: t.context.knex, - knexAdmin: t.context.knexAdmin, - testDbName, - }); -}); - -test('handler migrates reconciliation reports', async (t) => { - const { reconciliationReportsModel } = t.context; - - const fakeReconciliationReport = { - name: cryptoRandomString({ length: 5 }), - type: 'Granule Inventory', - status: 'Generated', - error: {}, - createdAt: (Date.now() - 1000), - updatedAt: Date.now(), - }; - - await Promise.all([ - reconciliationReportsModel.create(fakeReconciliationReport), - ]); - - t.teardown(() => reconciliationReportsModel.delete({ name: fakeReconciliationReport.name })); - - const call = await handler({}); - const expected = { - reconciliation_reports: { - failed: 0, - migrated: 1, - skipped: 0, - total_dynamo_db_records: 1, - }, - }; - t.deepEqual(call, expected); -}); diff --git a/lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js b/lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js deleted file mode 100644 index 79afe2d4a58..00000000000 --- a/lambdas/reconciliation-report-migration/tests/test-reconciliation-reports.js +++ /dev/null @@ -1,245 +0,0 @@ -const cryptoRandomString = require('crypto-random-string'); -const omit = require('lodash/omit'); -const test = require('ava'); - -const ReconciliationReport = require('@cumulus/api/models/reconciliation-reports'); -const { dynamodbDocClient } = require('@cumulus/aws-client/services'); -const { - createBucket, - recursivelyDeleteS3Bucket, -} = require('@cumulus/aws-client/S3'); -const { - generateLocalTestDb, - destroyLocalTestDb, - ReconciliationReportPgModel, - migrationDir, -} = require('@cumulus/db'); -const { RecordAlreadyMigrated } = require('@cumulus/errors'); - -const { - migrateReconciliationReportRecord, - migrateReconciliationReports, -} = require('../dist/lambda/reconciliation-reports'); - -const testDbName = `reconciliation_reports_migration_${cryptoRandomString({ length: 10 })}`; - -const generateFakeReconciliationReport = (params) => ({ - name: cryptoRandomString({ length: 5 }), - type: 'Granule Inventory', - status: 'Generated', - error: {}, - location: `s3://${cryptoRandomString({ length: 10 })}/${cryptoRandomString({ length: 10 })}`, - createdAt: (Date.now() - 1000), - updatedAt: Date.now(), - ...params, -}); - -let reconciliationReportsModel; - -test.before(async (t) => { - process.env.stackName = cryptoRandomString({ length: 10 }); - process.env.system_bucket = cryptoRandomString({ length: 10 }); - process.env.ReconciliationReportsTable = cryptoRandomString({ length: 10 }); - - await createBucket(process.env.system_bucket); - - reconciliationReportsModel = new ReconciliationReport({ - stackName: process.env.stackName, - systemBucket: process.env.system_bucket, - }); - await reconciliationReportsModel.createTable(); - - t.context.reconciliationReportPgModel = new ReconciliationReportPgModel(); - - const { knex, knexAdmin } = await generateLocalTestDb(testDbName, migrationDir); - t.context.knex = knex; - t.context.knexAdmin = knexAdmin; -}); - -test.afterEach.always(async (t) => { - await t.context.knex('reconciliation_reports').del(); -}); - -test.after.always(async (t) => { - await reconciliationReportsModel.deleteTable(); - await recursivelyDeleteS3Bucket(process.env.system_bucket); - await destroyLocalTestDb({ - knex: t.context.knex, - knexAdmin: t.context.knexAdmin, - testDbName, - }); -}); - -test.serial('migrateReconciliationReportRecord correctly migrates reconciliationReport record', async (t) => { - const { knex, reconciliationReportPgModel } = t.context; - - const fakeReconReport = generateFakeReconciliationReport(); - await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); - - const createdRecord = await reconciliationReportPgModel.get( - knex, - { name: fakeReconReport.name } - ); - - t.deepEqual( - omit(createdRecord, ['cumulus_id']), - omit({ - ...fakeReconReport, - created_at: new Date(fakeReconReport.createdAt), - updated_at: new Date(fakeReconReport.updatedAt), - }, ['createdAt', 'updatedAt']) - ); -}); - -test.serial('migrateReconciliationReportRecord correctly migrates reconciliationReport record where record.error is an object', async (t) => { - const error = { exception: 'there is an error' }; - const fakeReconReport = generateFakeReconciliationReport({ error }); - await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); - - const createdRecord = await t.context.knex.queryBuilder() - .select() - .table('reconciliation_reports') - .where({ name: fakeReconReport.name }) - .first(); - - t.deepEqual( - omit(createdRecord, ['cumulus_id']), - omit({ - ...fakeReconReport, - created_at: new Date(fakeReconReport.createdAt), - updated_at: new Date(fakeReconReport.updatedAt), - }, ['createdAt', 'updatedAt']) - ); -}); - -test.serial('migrateReconciliationReportRecord migrates reconciliationReport record with undefined nullables', async (t) => { - const { knex, reconciliationReportPgModel } = t.context; - - const fakeReconReport = generateFakeReconciliationReport(); - delete fakeReconReport.error; - delete fakeReconReport.location; - await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); - - const createdRecord = await reconciliationReportPgModel.get( - knex, - { name: fakeReconReport.name } - ); - - t.deepEqual( - omit(createdRecord, ['cumulus_id']), - omit({ - ...fakeReconReport, - error: null, - location: null, - created_at: new Date(fakeReconReport.createdAt), - updated_at: new Date(fakeReconReport.updatedAt), - }, ['createdAt', 'updatedAt']) - ); -}); - -test.serial('migrateReconciliationReportRecord throws RecordAlreadyMigrated error if already migrated record is newer', async (t) => { - const fakeReconReport = generateFakeReconciliationReport({ - updatedAt: Date.now(), - }); - - await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); - - const olderFakeReconReport = { - ...fakeReconReport, - updatedAt: Date.now() - 1000, // older than fakeReconReport - }; - - await t.throwsAsync( - migrateReconciliationReportRecord(olderFakeReconReport, t.context.knex), - { instanceOf: RecordAlreadyMigrated } - ); -}); - -test.serial('migrateReconciliationReportRecord updates an already migrated record if the updated date is newer', async (t) => { - const { knex, reconciliationReportPgModel } = t.context; - - const fakeReconReport = generateFakeReconciliationReport({ - updatedAt: Date.now() - 1000, - }); - await migrateReconciliationReportRecord(fakeReconReport, t.context.knex); - - const newerFakeReconReport = generateFakeReconciliationReport({ - ...fakeReconReport, - updatedAt: Date.now(), - }); - await migrateReconciliationReportRecord(newerFakeReconReport, t.context.knex); - - const createdRecord = await reconciliationReportPgModel.get( - knex, - { name: fakeReconReport.name } - ); - - t.deepEqual(createdRecord.updated_at, new Date(newerFakeReconReport.updatedAt)); -}); - -test.serial('migrateReconciliationReports processes multiple reconciliation reports', async (t) => { - const { knex, reconciliationReportPgModel } = t.context; - - const fakeReconReport1 = generateFakeReconciliationReport(); - const fakeReconReport2 = generateFakeReconciliationReport(); - - await Promise.all([ - reconciliationReportsModel.create(fakeReconReport1), - reconciliationReportsModel.create(fakeReconReport2), - ]); - t.teardown(() => Promise.all([ - reconciliationReportsModel.delete({ name: fakeReconReport1.name }), - reconciliationReportsModel.delete({ name: fakeReconReport2.name }), - ])); - - const migrationSummary = await migrateReconciliationReports(process.env, t.context.knex); - t.deepEqual(migrationSummary, { - total_dynamo_db_records: 2, - skipped: 0, - failed: 0, - migrated: 2, - }); - - const records = await reconciliationReportPgModel.search( - knex, - {} - ); - t.is(records.length, 2); -}); - -test.serial('migrateReconciliationReports processes all non-failing records', async (t) => { - const { knex, reconciliationReportPgModel } = t.context; - - const fakeReconReport1 = generateFakeReconciliationReport(); - const fakeReconReport2 = generateFakeReconciliationReport(); - - // remove required source field so that record will fail - delete fakeReconReport1.status; - - await Promise.all([ - // Have to use Dynamo client directly because creating - // via model won't allow creation of an invalid record - dynamodbDocClient().put({ - TableName: process.env.ReconciliationReportsTable, - Item: fakeReconReport1, - }), - reconciliationReportsModel.create(fakeReconReport2), - ]); - t.teardown(() => Promise.all([ - reconciliationReportsModel.delete({ name: fakeReconReport1.name }), - reconciliationReportsModel.delete({ name: fakeReconReport2.name }), - ])); - - const migrationSummary = await migrateReconciliationReports(process.env, t.context.knex); - t.deepEqual(migrationSummary, { - total_dynamo_db_records: 2, - skipped: 0, - failed: 1, - migrated: 1, - }); - const records = await reconciliationReportPgModel.search( - knex, - {} - ); - t.is(records.length, 1); -}); diff --git a/lambdas/reconciliation-report-migration/tsconfig.json b/lambdas/reconciliation-report-migration/tsconfig.json deleted file mode 100644 index 4b4ae9578ce..00000000000 --- a/lambdas/reconciliation-report-migration/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "outDir": "dist/lambda", - "declaration": false, - "declarationMap": false, - "sourceMap": true, - "removeComments": true - }, - "include": ["src"], -} diff --git a/lambdas/reconciliation-report-migration/variables.tf b/lambdas/reconciliation-report-migration/variables.tf deleted file mode 100644 index ad4ff7463cf..00000000000 --- a/lambdas/reconciliation-report-migration/variables.tf +++ /dev/null @@ -1,59 +0,0 @@ -# Required - -variable "dynamo_tables" { - description = "A map of objects with the `arn` and `name` of every DynamoDB table for your Cumulus deployment." - type = map(object({ name = string, arn = string })) -} - -variable "permissions_boundary_arn" { - type = string -} - -variable "prefix" { - type = string -} - -variable "rds_user_access_secret_arn" { - description = "RDS User Database Login Credential Secret ID" - type = string -} - -variable "system_bucket" { - description = "The name of the S3 bucket to be used for staging deployment files" - type = string -} - -# Optional - -variable "lambda_memory_sizes" { - description = "Configurable map of memory sizes for lambdas" - type = map(number) - default = {} -} - -variable "lambda_timeouts" { - description = "Configurable map of timeouts for lambdas" - type = map(number) - default = {} -} - -variable "lambda_subnet_ids" { - type = list(string) - default = [] -} - -variable "rds_security_group_id" { - description = "RDS Security Group used for access to RDS cluster" - type = string - default = "" -} - -variable "tags" { - type = map(string) - default = {} -} - -variable "vpc_id" { - type = string - default = null -} diff --git a/lambdas/reconciliation-report-migration/versions.tf b/lambdas/reconciliation-report-migration/versions.tf deleted file mode 100644 index c62a4968cfd..00000000000 --- a/lambdas/reconciliation-report-migration/versions.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } - required_version = ">= 1.5" -} diff --git a/lambdas/reconciliation-report-migration/webpack.config.js b/lambdas/reconciliation-report-migration/webpack.config.js deleted file mode 100644 index b0a194e3834..00000000000 --- a/lambdas/reconciliation-report-migration/webpack.config.js +++ /dev/null @@ -1,53 +0,0 @@ - -const path = require('path'); -const { IgnorePlugin } = require('webpack'); - -const ignoredPackages = [ - 'better-sqlite3', - 'mssql', - 'mssql/lib/base', - 'mssql/package.json', - 'mysql', - 'mysql2', - 'oracledb', - 'pg-native', - 'pg-query-stream', - 'sqlite3', - 'tedious' -]; - -module.exports = { - plugins: [ - new IgnorePlugin({ - resourceRegExp: new RegExp(`^(${ignoredPackages.join('|')})$`) - }), - ], - mode: 'production', - entry: './dist/lambda/index.js', - output: { - chunkFormat: false, - libraryTarget: 'commonjs2', - filename: 'index.js', - path: path.resolve(__dirname, 'dist', 'webpack') - }, - module: { - rules: [ - { - test: /\.js$/, - exclude: /node_modules/, - use: [ - { - loader: 'babel-loader', - options: { - cacheDirectory: true - }, - }, - ], - }, - ], - }, - target: 'node', - externals: [ - /@aws-sdk\// - ] -}; diff --git a/tf-modules/cumulus/reconciliation_report_migration.tf b/tf-modules/cumulus/reconciliation_report_migration.tf deleted file mode 100644 index c6132d136a4..00000000000 --- a/tf-modules/cumulus/reconciliation_report_migration.tf +++ /dev/null @@ -1,21 +0,0 @@ -module "reconciliation_report_migration_lambda" { - source = "../../lambdas/reconciliation-report-migration" - - prefix = var.prefix - system_bucket = var.system_bucket - - dynamo_tables = var.dynamo_tables - - lambda_subnet_ids = var.lambda_subnet_ids - lambda_timeouts = var.lambda_timeouts - lambda_memory_sizes = var.lambda_memory_sizes - - permissions_boundary_arn = var.permissions_boundary_arn - - rds_security_group_id = var.rds_security_group - rds_user_access_secret_arn = var.rds_user_access_secret_arn - - tags = var.tags - vpc_id = var.vpc_id -} -