generated from hirosystems/.github
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Loading status checks…
feat: use worker threads for CPU intensive message parsing
Showing
8 changed files
with
155 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
services: | ||
postgres: | ||
image: "redis:7" | ||
ports: | ||
- "8379:6379" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
import * as WorkerThreads from 'node:worker_threads'; | ||
import { CoreNodeNakamotoBlockMessage, StackerDbChunk } from './core-node-message'; | ||
import { | ||
ParsedNakamotoBlock, | ||
ParsedStackerDbChunk, | ||
parseNakamotoBlockMsg, | ||
parseStackerDbChunk, | ||
} from './msg-parsing'; | ||
|
||
export const workerFile = __filename; | ||
|
||
export enum ThreadedParserMsgType { | ||
NakamotoBlock = 'NakamotoBlock', | ||
StackerDbChunk = 'StackerDbChunk', | ||
} | ||
|
||
interface ThreadMsg { | ||
type: ThreadedParserMsgType; | ||
msgId: number; | ||
} | ||
|
||
export interface NakamotoBlockMsgRequest extends ThreadMsg { | ||
type: ThreadedParserMsgType.NakamotoBlock; | ||
msgId: number; | ||
block: CoreNodeNakamotoBlockMessage; | ||
} | ||
|
||
export interface NakamotoBlockMsgReply extends ThreadMsg { | ||
type: ThreadedParserMsgType.NakamotoBlock; | ||
msgId: number; | ||
block: ParsedNakamotoBlock; | ||
} | ||
|
||
export interface StackerDbChunkMsgRequest extends ThreadMsg { | ||
type: ThreadedParserMsgType.StackerDbChunk; | ||
msgId: number; | ||
chunk: StackerDbChunk; | ||
} | ||
|
||
export interface StackerDbChunkMsgReply extends ThreadMsg { | ||
type: ThreadedParserMsgType.StackerDbChunk; | ||
msgId: number; | ||
chunk: ParsedStackerDbChunk[]; | ||
} | ||
|
||
export type ThreadedParserMsgRequest = NakamotoBlockMsgRequest | StackerDbChunkMsgRequest; | ||
export type ThreadedParserMsgReply = NakamotoBlockMsgReply | StackerDbChunkMsgReply; | ||
|
||
if (!WorkerThreads.isMainThread) { | ||
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion | ||
const mainThreadPort = WorkerThreads.parentPort!; | ||
mainThreadPort.on('message', (msg: ThreadedParserMsgRequest) => { | ||
let reply: ThreadedParserMsgReply; | ||
switch (msg.type) { | ||
case ThreadedParserMsgType.NakamotoBlock: { | ||
reply = { | ||
type: ThreadedParserMsgType.NakamotoBlock, | ||
msgId: msg.msgId, | ||
block: parseNakamotoBlockMsg(msg.block), | ||
} satisfies NakamotoBlockMsgReply; | ||
break; | ||
} | ||
case ThreadedParserMsgType.StackerDbChunk: { | ||
reply = { | ||
type: ThreadedParserMsgType.StackerDbChunk, | ||
msgId: msg.msgId, | ||
chunk: parseStackerDbChunk(msg.chunk), | ||
} satisfies StackerDbChunkMsgReply; | ||
break; | ||
} | ||
default: { | ||
const _exhaustiveCheck: never = msg; | ||
throw new Error(`Unhandled message type: ${msg}`); | ||
} | ||
} | ||
mainThreadPort.postMessage(reply); | ||
}); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
import * as WorkerThreads from 'node:worker_threads'; | ||
import { waiter, Waiter, logger as defaultLogger } from '@hirosystems/api-toolkit'; | ||
import { CoreNodeNakamotoBlockMessage, StackerDbChunk } from './core-node-message'; | ||
import { ParsedNakamotoBlock, ParsedStackerDbChunk } from './msg-parsing'; | ||
import { | ||
NakamotoBlockMsgReply, | ||
NakamotoBlockMsgRequest, | ||
StackerDbChunkMsgReply, | ||
StackerDbChunkMsgRequest, | ||
ThreadedParserMsgReply, | ||
ThreadedParserMsgType, | ||
workerFile, | ||
} from './threaded-parser-worker'; | ||
|
||
export class ThreadedParser { | ||
private readonly worker: WorkerThreads.Worker; | ||
private readonly msgRequests: Map<number, Waiter<ThreadedParserMsgReply>> = new Map(); | ||
private readonly logger = defaultLogger.child({ module: 'ThreadedParser' }); | ||
private lastMsgId = 0; | ||
|
||
constructor() { | ||
if (!WorkerThreads.isMainThread) { | ||
throw new Error('ThreadedParser must be instantiated in the main thread'); | ||
} | ||
this.worker = new WorkerThreads.Worker(workerFile); | ||
this.worker.on('message', (msg: ThreadedParserMsgReply) => { | ||
const waiter = this.msgRequests.get(msg.msgId); | ||
if (waiter) { | ||
waiter.finish(msg); | ||
this.msgRequests.delete(msg.msgId); | ||
} else { | ||
this.logger.warn('Received unexpected message from worker', msg); | ||
} | ||
}); | ||
} | ||
|
||
async parseNakamotoBlock(block: CoreNodeNakamotoBlockMessage): Promise<ParsedNakamotoBlock> { | ||
const replyWaiter = waiter<NakamotoBlockMsgReply>(); | ||
const msg: NakamotoBlockMsgRequest = { | ||
type: ThreadedParserMsgType.NakamotoBlock, | ||
msgId: this.lastMsgId++, | ||
block, | ||
}; | ||
this.msgRequests.set(msg.msgId, replyWaiter as Waiter<ThreadedParserMsgReply>); | ||
const reply = await replyWaiter; | ||
return reply.block; | ||
} | ||
|
||
async parseStackerDbChunk(chunk: StackerDbChunk): Promise<ParsedStackerDbChunk[]> { | ||
const replyWaiter = waiter<StackerDbChunkMsgReply>(); | ||
const msg: StackerDbChunkMsgRequest = { | ||
type: ThreadedParserMsgType.StackerDbChunk, | ||
msgId: this.lastMsgId++, | ||
chunk, | ||
}; | ||
this.msgRequests.set(msg.msgId, replyWaiter as Waiter<ThreadedParserMsgReply>); | ||
const reply = await replyWaiter; | ||
return reply.chunk; | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters