Skip to content

Commit

Permalink
Catch the exception thrown by runing tasks onTick function, sometimes…
Browse files Browse the repository at this point in the history
… the PRC node will close the connection, this will make the crust-smanager exit directly, restart the smanager will need to re-load all node information again which is very time consuming
  • Loading branch information
wuhaixian1984 committed Jun 24, 2024
1 parent a894505 commit 20ff586
Showing 1 changed file with 21 additions and 8 deletions.
29 changes: 21 additions & 8 deletions src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ const ConfigFile = process.env['SMANAGER_CONFIG'] || 'smanager-config.json';
const MaxNoNewBlockDuration = Dayjs.duration({
minutes: 30,
});
const MaxRunTaskConsecutiveErrorCount = 5;

/**
* SManager tasks:
Expand Down Expand Up @@ -146,6 +147,7 @@ async function doEventLoop(context: AppContext, tasks: Task[]): Promise<void> {
const { api } = context;
let lastBlock = api.latestFinalizedBlock();
let lastBlockTime = Dayjs();
let runTaskConsecutiveErrorCount = 0;
logger.info('running event loop');
do {
await api.ensureConnection();
Expand All @@ -164,15 +166,26 @@ async function doEventLoop(context: AppContext, tasks: Task[]): Promise<void> {
continue;
}
lastBlockTime = Dayjs();
for (let block = lastBlock + 1; block <= curBlock; block++) {
logger.info('run tasks on block %d', block);
lastBlock = block;
await timeoutOrError(
`run tasks`,
Bluebird.map(tasks, (t) => t.onTick(lastBlock)),
MaxTickTimout,
);
try {
for (let block = lastBlock + 1; block <= curBlock; block++) {
logger.info('run tasks on block %d', block);
await timeoutOrError(
`run tasks`,
Bluebird.map(tasks, (t) => t.onTick(block)),
MaxTickTimout,
);
lastBlock = block;
runTaskConsecutiveErrorCount = 0; // reset the counter if run task successfully
}
} catch(err) {
logger.error(`error in run task on tick: ${err}`);
runTaskConsecutiveErrorCount++;
if (runTaskConsecutiveErrorCount > MaxRunTaskConsecutiveErrorCount) {
logger.error(`Conesecutive run task error count exceeds the limit ${MaxRunTaskConsecutiveErrorCount}, quiting smanager!`);
throw err;
}
}

await Bluebird.delay(1 * 1000);
} while (true); // eslint-disable-line
}
Expand Down

0 comments on commit 20ff586

Please sign in to comment.