Skip to content

Commit

Permalink
Attempt to fix memory leak issues
Browse files Browse the repository at this point in the history
- Piscina was eating lots of memory
- Product frontend/backend could be leaking memory, but if one just
  wants to run a data backup node, then NODE_ENV="reconcile" may now fix
  this
  • Loading branch information
TimDaub committed Sep 26, 2024
1 parent 2551b7b commit b49110a
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 6 deletions.
9 changes: 6 additions & 3 deletions src/launch.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ cache.initializeNotifications();
const reconcileMode = env.NODE_ENV === "reconcile";
if (reconcileMode) {
log(`Running in reconciliation mode`);
log(
`In reconciliation mode, syncing with the chain and reconciling with the p2p network are enabled. The product backend/frontends are disabled.`,
);
}

const trie = await store.create();
Expand All @@ -36,11 +39,11 @@ const node = await start(config);
if (!reconcileMode) {
await api.launch(trie, node);
await http.launch(trie, node);

crawl(mintCrawlPath);
crawl(delegateCrawlPath);
}

crawl(mintCrawlPath);
crawl(delegateCrawlPath);

// NOTE: We're passing in the trie here as we don't want to make it globally
// available to run more than one node in the tests
messages.handlers.message = messages.handlers.message(trie);
Expand Down
15 changes: 12 additions & 3 deletions src/store.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ import { triggerNotification } from "./subscriptions.mjs";

const maxReaders = 500;

// NOTE: We're running the launching of Piscina workers on the top of the
// module as when this is run in a function, and the function is called
// multiple times then Piscina will continue to spawn more and more workers
// which fills up the memory fast.
const piscina = new Piscina({
filename: resolve("./src/workers/enhancer.mjs"),
});

export const upvotes = new Set();
export const commentCounts = new Map();

Expand Down Expand Up @@ -552,10 +560,11 @@ export async function posts(
return posts;
}

// NOTE: If you're wondering why the enhancer is recreated on each call of
// posts( and other calls, the reason for it is that we constantly have to
// re-query accounts and delegations as they're subject to change throughout
// the application's lifecycle (e.g. when a new person mints/delegates).
function enhance(accounts, delegations, cacheEnabled) {
const piscina = new Piscina({
filename: resolve("./src/workers/enhancer.mjs"),
});
return async (node) => {
const computeFunc = async () =>
await piscina.run({
Expand Down

0 comments on commit b49110a

Please sign in to comment.