From 94fc662b281fa7d078fd28e74dee90e532080e55 Mon Sep 17 00:00:00 2001 From: Gabe Hamilton Date: Thu, 29 Jun 2023 14:28:40 -0600 Subject: [PATCH] DPLT-1051 Do not retry errors, send them straight to the DLQ. (#117) --- .../__snapshots__/indexer.test.js.snap | 12 ++++++------ indexer-js-queue-handler/indexer.js | 5 +++-- indexer-js-queue-handler/serverless.yml | 2 ++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap b/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap index e6c354b3d..f21c79f26 100644 --- a/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap @@ -5,7 +5,7 @@ exports[`Indexer unit tests Indexer.runFunctions() allows imperative execution o [ "mock-hasura-endpoint/v1/graphql", { - "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":82699904,"message":"Running function:buildnear.testnet/test:, lag in ms is: :NaN"}}", + "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":82699904,"message":" historical backfill"}}", "headers": { "Content-Type": "application/json", "X-Hasura-Admin-Secret": "mock-hasura-secret", @@ -75,7 +75,7 @@ exports[`Indexer unit tests Indexer.runFunctions() catches errors 1`] = ` [ "mock-hasura-endpoint/v1/graphql", { - "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":456,"message":"Running function:buildnear.testnet/test:, lag in ms is: :NaN"}}", + "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":456,"message":" historical backfill"}}", "headers": { "Content-Type": "application/json", "X-Hasura-Admin-Secret": "mock-hasura-secret", @@ -132,7 +132,7 @@ exports[`Indexer unit tests Indexer.runFunctions() logs provisioning failures 1` [ "mock-hasura-endpoint/v1/graphql", { - "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"morgs.near/test","block_height":82699904,"message":"Running function:morgs.near/test:, lag in ms is: :NaN"}}", + "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"morgs.near/test","block_height":82699904,"message":" historical backfill"}}", "headers": { "Content-Type": "application/json", "X-Hasura-Admin-Secret": "mock-hasura-secret", @@ -202,7 +202,7 @@ exports[`Indexer unit tests Indexer.runFunctions() sets the current historical b [ "mock-hasura-endpoint/v1/graphql", { - "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":456,"message":"Running function:buildnear.testnet/test:, lag in ms is: :NaN"}}", + "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":456,"message":" historical backfill"}}", "headers": { "Content-Type": "application/json", "X-Hasura-Admin-Secret": "mock-hasura-secret", @@ -259,7 +259,7 @@ exports[`Indexer unit tests Indexer.runFunctions() should execute all functions [ "mock-hasura-endpoint/v1/graphql", { - "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":456,"message":"Running function:buildnear.testnet/test:, lag in ms is: :NaN"}}", + "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"buildnear.testnet/test","block_height":456,"message":" historical backfill"}}", "headers": { "Content-Type": "application/json", "X-Hasura-Admin-Secret": "mock-hasura-secret", @@ -316,7 +316,7 @@ exports[`Indexer unit tests Indexer.runFunctions() supplies the required role to [ "mock-hasura-endpoint/v1/graphql", { - "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"morgs.near/test","block_height":82699904,"message":"Running function:morgs.near/test:, lag in ms is: :NaN"}}", + "body": "{"query":"mutation writeLog($function_name: String!, $block_height: numeric!, $message: String!){\\n insert_indexer_log_entries_one(object: {function_name: $function_name, block_height: $block_height, message: $message}) {id}\\n }","variables":{"function_name":"morgs.near/test","block_height":82699904,"message":" historical backfill"}}", "headers": { "Content-Type": "application/json", "X-Hasura-Admin-Secret": "mock-hasura-secret", diff --git a/indexer-js-queue-handler/indexer.js b/indexer-js-queue-handler/indexer.js index 2e9089e57..faebaff20 100644 --- a/indexer-js-queue-handler/indexer.js +++ b/indexer-js-queue-handler/indexer.js @@ -39,11 +39,12 @@ export default class Indexer { for (const function_name in functions) { try { const indexerFunction = functions[function_name]; - console.log('Running function', function_name, ', lag in ms is: ', lag); // Lambda logs + const runningMessage = `Running function ${function_name}` + is_historical ? ' historical backfill' : `, lag is: ${lag}ms from block timestamp`; + console.log(runningMessage); // Lambda logs const segment = this.deps.awsXray.getSegment(); // segment is immutable, subsegments are mutable const functionSubsegment = segment.addNewSubsegment('indexer_function'); functionSubsegment.addAnnotation('indexer_function', function_name); - simultaneousPromises.push(this.writeLog(function_name, block_height, 'Running function', function_name, ', lag in ms is: ', lag)); + simultaneousPromises.push(this.writeLog(function_name, block_height, runningMessage)); simultaneousPromises.push(this.deps.metrics.putBlockHeight(indexerFunction.account_id, indexerFunction.function_name, block_height)); diff --git a/indexer-js-queue-handler/serverless.yml b/indexer-js-queue-handler/serverless.yml index c89b29d5c..a44657e3e 100644 --- a/indexer-js-queue-handler/serverless.yml +++ b/indexer-js-queue-handler/serverless.yml @@ -27,12 +27,14 @@ constructs: indexer-runner: type: queue fifo: true + maxRetries: 1 worker: handler: handler.consumer timeout: 15 # 1.5 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) startFromBlock-runner: type: queue fifo: true + maxRetries: 1 # batchSize: 100 worker: handler: handler.consumer