From ee2839d889facad0acbaab3be717c688e5039282 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 11:10:50 +0200
Subject: [PATCH 01/22] Process validator performance data

---
 packages/brain/src/getSecondsToNextEpoch.ts   |  22 +++
 packages/brain/src/index.ts                   |   9 +-
 .../src/modules/apiClients/postgres/index.ts  |  74 +++++++++--
 .../src/modules/apiClients/postgres/types.ts  |  16 ++-
 packages/brain/src/modules/cron/index.ts      |   2 +-
 .../cron/trackValidatorsPerformance/index.ts  |  30 +----
 .../insertPerformanceData.ts                  |  14 +-
 .../calculateAttestationSuccessRate.ts        |  33 +++++
 .../calculateBlocksProposedSuccessRate.ts     |  25 ++++
 .../getStartAndEndEpochs.ts                   |  27 ++++
 .../modules/validatorsPerformance/index.ts    | 125 +++++++++++-------
 .../modules/validatorsPerformance/types.ts    |  14 ++
 12 files changed, 288 insertions(+), 103 deletions(-)
 create mode 100644 packages/brain/src/getSecondsToNextEpoch.ts
 create mode 100644 packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
 create mode 100644 packages/brain/src/modules/validatorsPerformance/calculateBlocksProposedSuccessRate.ts
 create mode 100644 packages/brain/src/modules/validatorsPerformance/getStartAndEndEpochs.ts
 create mode 100644 packages/brain/src/modules/validatorsPerformance/types.ts

diff --git a/packages/brain/src/getSecondsToNextEpoch.ts b/packages/brain/src/getSecondsToNextEpoch.ts
new file mode 100644
index 00000000..d12b73e1
--- /dev/null
+++ b/packages/brain/src/getSecondsToNextEpoch.ts
@@ -0,0 +1,22 @@
+/**
+ * Get the seconds to the start of the next epoch based on the current Unix time and the minimum genesis time of the chain.
+ *
+ * @param {number} minGenesisTime - Minimum genesis time of the chain.
+ * @param {number} secondsPerSlot - Seconds per slot.
+ * @returns {number} - Seconds to the start of the next epoch.
+ */
+export function getSecondsToNextEpoch({
+  minGenesisTime,
+  secondsPerSlot
+}: {
+  minGenesisTime: number;
+  secondsPerSlot: number;
+}): number {
+  const currentUnixTime = Math.floor(Date.now() / 1000);
+  const timeDifference = currentUnixTime - minGenesisTime; // Time difference in seconds
+  const stlotsSinceGenesis = timeDifference / secondsPerSlot; // Slots since genesis
+  const currentEpoch = Math.floor(stlotsSinceGenesis / 32); // Current epoch
+  const nextEpochStartSlot = (currentEpoch + 1) * 32; // Slot at the start of the next epoch
+  const nextEpochStartTime = nextEpochStartSlot * secondsPerSlot + minGenesisTime; // Time at the start of the next epoch in seconds
+  return nextEpochStartTime - currentUnixTime; // Return the difference in seconds
+}
diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts
index edaf7b92..4a665ce8 100644
--- a/packages/brain/src/index.ts
+++ b/packages/brain/src/index.ts
@@ -12,15 +12,10 @@ import { startUiServer, startLaunchpadApi } from "./modules/apiServers/index.js"
 import * as dotenv from "dotenv";
 import process from "node:process";
 import { params } from "./params.js";
-import {
-  CronJob,
-  reloadValidators,
-  trackValidatorsPerformance,
-  sendProofsOfValidation,
-  getSecondsToNextEpoch
-} from "./modules/cron/index.js";
+import { CronJob, reloadValidators, trackValidatorsPerformance, sendProofsOfValidation } from "./modules/cron/index.js";
 import { PostgresClient } from "./modules/apiClients/index.js";
 import { brainConfig } from "./modules/config/index.js";
+import { getSecondsToNextEpoch } from "./getSecondsToNextEpoch.js";
 
 logger.info(`Starting brain...`);
 
diff --git a/packages/brain/src/modules/apiClients/postgres/index.ts b/packages/brain/src/modules/apiClients/postgres/index.ts
index a1f71a8b..e2c4cf50 100644
--- a/packages/brain/src/modules/apiClients/postgres/index.ts
+++ b/packages/brain/src/modules/apiClients/postgres/index.ts
@@ -32,7 +32,7 @@ enum Columns {
   liveness = "liveness",
   blockProposalStatus = "block_proposal_status",
   syncCommitteeRewards = "sync_comittee_rewards",
-  attestationsRewards = "attestations_rewards",
+  attestationsTotalRewards = "attestations_total_rewards",
   error = "error"
 }
 
@@ -123,9 +123,9 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
   ${Columns.consensusClient} ${this.CONSENSUS_CLIENT} NOT NULL,
   ${Columns.slot} BIGINT,
   ${Columns.liveness} BOOLEAN,
-  ${Columns.blockProposalStatus} ${this.BLOCK_PROPOSAL_STATUS},
+  ${Columns.blockProposalStatus} ${this.BLOCK_PROPOSAL_STATUS} NOT NULL,
   ${Columns.syncCommitteeRewards} BIGINT,
-  ${Columns.attestationsRewards} JSONB,
+  ${Columns.attestationsTotalRewards} JSONB NOT NULL,
   ${Columns.error} TEXT,
   PRIMARY KEY (${Columns.validatorIndex}, ${Columns.epoch})
 );
@@ -150,11 +150,11 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
    * Inserts the given performance data into the database.
    *
    * @param data - The performance data to insert.
-   * @example insertPerformanceData({ validatorIndex: 1, epoch: 1, slot: 1, liveness: true, blockProposalStatus: "missed", syncCommitteeRewards: 100, attestationsRewards: { attestation1: 10, attestation2: 20 } })
+   * @example insertPerformanceData({ validatorIndex: 1, epoch: 1, slot: 1, liveness: true, blockProposalStatus: "missed", syncCommitteeRewards: 100, attestationsTotalRewards: { attestation1: 10, attestation2: 20 } })
    */
   public async insertPerformanceData(data: ValidatorPerformance): Promise<void> {
     const query = `
-INSERT INTO ${this.tableName} (${Columns.validatorIndex}, ${Columns.epoch}, ${Columns.slot}, ${Columns.liveness}, ${Columns.blockProposalStatus}, ${Columns.syncCommitteeRewards}, ${Columns.attestationsRewards}, ${Columns.error})
+INSERT INTO ${this.tableName} (${Columns.validatorIndex}, ${Columns.epoch}, ${Columns.slot}, ${Columns.liveness}, ${Columns.blockProposalStatus}, ${Columns.syncCommitteeRewards}, ${Columns.attestationsTotalRewards}, ${Columns.error})
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
     `;
 
@@ -165,15 +165,16 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
       data.consensusClient,
       data.slot ?? null,
       data.liveness ?? null,
-      data.blockProposalStatus ?? null,
+      data.blockProposalStatus,
       data.syncCommitteeRewards ?? null,
-      JSON.stringify(data.attestationsRewards) ?? null, // JSONB expects a string
+      JSON.stringify(data.attestationsTotalRewards), // JSONB expects a string
       data.error ?? null
     ]);
   }
 
   /**
-   * Get the validators data for the given validator indexes from all epochs.
+   * Get the validators data for the given validator indexes from all epochs. In order to improve data process
+   * it will return a map with the validator index as key and the performance data as value.
    *
    * @param validatorIndexes - The indexes of the validators to get the data for.
    * @returns The performance data for the given validators.
@@ -185,7 +186,7 @@ WHERE ${Columns.validatorIndex} = ANY($1)
     `;
 
     const result = await this.sql.unsafe(query, [validatorIndexes]);
-    // TODO: add type for result
+    // TODO: add type to result
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     return result.map((row: any) => ({
       validatorIndex: row.validator_index,
@@ -196,11 +197,64 @@ WHERE ${Columns.validatorIndex} = ANY($1)
       liveness: row.liveness,
       blockProposalStatus: row.block_proposal_status,
       syncCommitteeRewards: row.sync_comittee_rewards,
-      attestationsRewards: row.attestations_rewards,
+      attestationsTotalRewards: row.attestations_total_rewards,
       error: row.error
     }));
   }
 
+  /**
+   * Get tje validators data for the given validator indexes and an epoch start and end range. In order to improve data process
+   * it will return a map with the validator index as key and the performance data as value.
+   *
+   * @param validatorIndexes - The indexes of the validators to get the data for.
+   * @param startEpoch - The start epoch number.
+   * @param endEpoch - The end epoch number.
+   * @returns The performance data for the given validators.
+   */
+  public async getValidatorsDataMapForEpochRange({
+    validatorIndexes,
+    startEpoch,
+    endEpoch
+  }: {
+    validatorIndexes: string[];
+    startEpoch: number;
+    endEpoch: number;
+  }): Promise<Map<string, ValidatorPerformance[]>> {
+    const query = `
+SELECT * FROM ${this.tableName}
+WHERE ${Columns.validatorIndex} = ANY($1)
+AND ${Columns.epoch} >= $2
+AND ${Columns.epoch} <= $3
+    `;
+
+    const result = await this.sql.unsafe(query, [validatorIndexes, startEpoch, endEpoch]);
+    // TODO: add type to result
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return result.reduce((map: Map<string, ValidatorPerformance[]>, row: any) => {
+      const key = row.validator_index;
+      const performanceData = {
+        validatorIndex: row.validator_index,
+        epoch: row.epoch,
+        executionClient: row.execution_client,
+        consensusClient: row.consensus_client,
+        slot: row.slot,
+        liveness: row.liveness,
+        blockProposalStatus: row.block_proposal_status,
+        syncCommitteeRewards: row.sync_comittee_rewards,
+        attestationsTotalRewards: row.attestations_total_rewards,
+        error: row.error
+      };
+
+      if (map.has(key)) {
+        map.get(key)?.push(performanceData);
+      } else {
+        map.set(key, [performanceData]);
+      }
+
+      return map;
+    }, new Map<string, ValidatorPerformance[]>());
+  }
+
   /**
    * Method to close the database connection.
    */
diff --git a/packages/brain/src/modules/apiClients/postgres/types.ts b/packages/brain/src/modules/apiClients/postgres/types.ts
index f8653d9b..d16f1678 100644
--- a/packages/brain/src/modules/apiClients/postgres/types.ts
+++ b/packages/brain/src/modules/apiClients/postgres/types.ts
@@ -3,8 +3,16 @@ import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
 export enum BlockProposalStatus {
   Missed = "Missed",
   Proposed = "Proposed",
-  Unchosen = "Unchosen",
-  Error = "Error"
+  Unchosen = "Unchosen"
+}
+
+export interface AttestationsTotalRewards {
+  validator_index: string;
+  head: string;
+  target: string;
+  source: string;
+  inclusion_delay: string;
+  inactivity: string;
 }
 
 export interface ValidatorPerformance {
@@ -12,10 +20,10 @@ export interface ValidatorPerformance {
   epoch: number;
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
+  blockProposalStatus: BlockProposalStatus;
+  attestationsTotalRewards: AttestationsTotalRewards;
   slot?: number;
   liveness?: boolean;
-  blockProposalStatus?: BlockProposalStatus;
   syncCommitteeRewards?: number;
-  attestationsRewards?: object;
   error?: string;
 }
diff --git a/packages/brain/src/modules/cron/index.ts b/packages/brain/src/modules/cron/index.ts
index c40eb52f..2f4c1a03 100644
--- a/packages/brain/src/modules/cron/index.ts
+++ b/packages/brain/src/modules/cron/index.ts
@@ -1,4 +1,4 @@
 export { CronJob } from "./cron.js";
 export { reloadValidators } from "./reloadValidators/index.js";
 export { sendProofsOfValidation } from "./sendProofsOfValidation/index.js";
-export { trackValidatorsPerformance, getSecondsToNextEpoch } from "./trackValidatorsPerformance/index.js";
+export { trackValidatorsPerformance } from "./trackValidatorsPerformance/index.js";
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index e3e833fa..e1a4cc8d 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -11,6 +11,7 @@ import { logPrefix } from "./logPrefix.js";
 import { TotalRewards } from "../../apiClients/types.js";
 import { BlockProposalStatus } from "../../apiClients/postgres/types.js";
 import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
+import { getSecondsToNextEpoch } from "../../../getSecondsToNextEpoch.js";
 
 const MINUTE_IN_SECONDS = 60;
 
@@ -52,7 +53,7 @@ export async function trackValidatorsPerformance({
     let errorGettingValidatorData: Error | undefined;
     let newEpochFinalized = epochFinalized;
     const activeValidatorsIndexes: string[] = [];
-    const validatorsAttestationsRewards: TotalRewards[] = [];
+    const validatorsAttestationsTotalRewards: TotalRewards[] = [];
     const validatorBlockStatusMap: Map<string, BlockProposalStatus> = new Map();
 
     label: while (epochFinalized === newEpochFinalized) {
@@ -75,7 +76,7 @@ export async function trackValidatorsPerformance({
           beaconchainApi,
           epoch: epochFinalized.toString(),
           validatorIndexes: activeValidatorsIndexes,
-          totalRewards: validatorsAttestationsRewards
+          totalRewards: validatorsAttestationsTotalRewards
         });
 
         // get block proposal status
@@ -120,7 +121,7 @@ export async function trackValidatorsPerformance({
       validatorIndexes: activeValidatorsIndexes,
       epochFinalized,
       validatorBlockStatus: validatorBlockStatusMap,
-      validatorsAttestationsRewards,
+      validatorsAttestationsTotalRewards,
       error: errorGettingValidatorData,
       executionClient,
       consensusClient
@@ -131,26 +132,3 @@ export async function trackValidatorsPerformance({
     return;
   }
 }
-
-/**
- * Get the seconds to the start of the next epoch based on the current Unix time and the minimum genesis time of the chain.
- *
- * @param {number} minGenesisTime - Minimum genesis time of the chain.
- * @param {number} secondsPerSlot - Seconds per slot.
- * @returns {number} - Seconds to the start of the next epoch.
- */
-export function getSecondsToNextEpoch({
-  minGenesisTime,
-  secondsPerSlot
-}: {
-  minGenesisTime: number;
-  secondsPerSlot: number;
-}): number {
-  const currentUnixTime = Math.floor(Date.now() / 1000);
-  const timeDifference = currentUnixTime - minGenesisTime; // Time difference in seconds
-  const stlotsSinceGenesis = timeDifference / secondsPerSlot; // Slots since genesis
-  const currentEpoch = Math.floor(stlotsSinceGenesis / 32); // Current epoch
-  const nextEpochStartSlot = (currentEpoch + 1) * 32; // Slot at the start of the next epoch
-  const nextEpochStartTime = nextEpochStartSlot * secondsPerSlot + minGenesisTime; // Time at the start of the next epoch in seconds
-  return nextEpochStartTime - currentUnixTime; // Return the difference in seconds
-}
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
index e31268da..bf8d3204 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
@@ -14,14 +14,14 @@ import { logPrefix } from "./logPrefix.js";
  * @param validatorIndexes - Array of validator indexes.
  * @param epochFinalized - The epoch finalized.
  * @param validatorBlockStatus - Map with the block proposal status of each validator.
- * @param validatorsAttestationsRewards - Array of total rewards for the validators.
+ * @param validatorsAttestationsTotalRewards - Array of total rewards for the validators.
  */
 export async function insertPerformanceDataNotThrow({
   postgresClient,
   validatorIndexes,
   epochFinalized,
   validatorBlockStatus,
-  validatorsAttestationsRewards,
+  validatorsAttestationsTotalRewards,
   executionClient,
   consensusClient,
   error
@@ -30,19 +30,19 @@ export async function insertPerformanceDataNotThrow({
   validatorIndexes: string[];
   epochFinalized: number;
   validatorBlockStatus: Map<string, BlockProposalStatus>;
-  validatorsAttestationsRewards: TotalRewards[];
+  validatorsAttestationsTotalRewards: TotalRewards[];
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
   error?: Error;
 }): Promise<void> {
   for (const validatorIndex of validatorIndexes) {
     //const liveness = validatorsLiveness.find((liveness) => liveness.index === validatorIndex)?.is_live;
-    const attestationsRewards = validatorsAttestationsRewards.find(
+    const attestationsTotalRewards = validatorsAttestationsTotalRewards.find(
       (attestationReward) => attestationReward.validator_index === validatorIndex
     );
 
-    if (!attestationsRewards) {
-      logger.error(`${logPrefix}Missing data for validator ${validatorIndex}, att: ${attestationsRewards}`);
+    if (!attestationsTotalRewards) {
+      logger.error(`${logPrefix}Missing data for validator ${validatorIndex}, att: ${attestationsTotalRewards}`);
       continue;
     }
 
@@ -61,7 +61,7 @@ export async function insertPerformanceDataNotThrow({
         validatorIndex: parseInt(validatorIndex),
         epoch: epochFinalized,
         blockProposalStatus,
-        attestationsRewards,
+        attestationsTotalRewards,
         error: error?.message,
         executionClient,
         consensusClient
diff --git a/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
new file mode 100644
index 00000000..8ccb3a7c
--- /dev/null
+++ b/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
@@ -0,0 +1,33 @@
+import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
+
+/**
+ * Calculates the attestation success rate for a given validator. The attestation success rate is the percentage of successful attestations
+ * Being the total attestation opportunities the number of epochs between the first and last epoch in the data set of a specific validator.
+ * And the total successful attestations the number of epochs where the validator successfully attested: head target and source must be > 0.
+ *
+ * @param validatorData the data of the validator from the postgres database
+ * @param startEpoch the start epoch of the data set
+ * @param endEpoch the end epoch of the data set
+ */
+export function calculateAttestationSuccessRate({
+  validatorData,
+  startEpoch,
+  endEpoch
+}: {
+  validatorData: ValidatorPerformance[];
+  startEpoch: number;
+  endEpoch: number;
+}): number {
+  // Calculate the total attestation opportunities
+  const totalAttestationOpportunities = endEpoch - startEpoch;
+
+  // Calculate the total successful attestations
+  const totalSuccessfulAttestations = validatorData.filter(
+    (data) =>
+      parseInt(data.attestationsTotalRewards.head) > 0 &&
+      parseInt(data.attestationsTotalRewards.target) > 0 &&
+      parseInt(data.attestationsTotalRewards.source) > 0
+  ).length;
+
+  return (totalSuccessfulAttestations / totalAttestationOpportunities) * 100;
+}
diff --git a/packages/brain/src/modules/validatorsPerformance/calculateBlocksProposedSuccessRate.ts b/packages/brain/src/modules/validatorsPerformance/calculateBlocksProposedSuccessRate.ts
new file mode 100644
index 00000000..78f9f561
--- /dev/null
+++ b/packages/brain/src/modules/validatorsPerformance/calculateBlocksProposedSuccessRate.ts
@@ -0,0 +1,25 @@
+import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
+
+/**
+ * Calculates the blocks proposed success rate for a given validator. The blocks proposed success rate is the percentage of successful block proposals
+ * Being the total block proposal opportunities the number of epochs where the validator proposed a block or missed a block.
+ * And the total successful block proposals the number of epochs where the validator successfully proposed a block.
+ *
+ * @param validatorData the data of the validator from the postgres database
+ * @returns the blocks proposed success rate
+ */
+export function calculateBlocksProposedSuccessRate({
+  validatorData
+}: {
+  validatorData: ValidatorPerformance[];
+}): number {
+  // Calculate the total block proposal opportunities
+  const totalBlockProposalOpportunities = validatorData.filter(
+    (data) => data.blockProposalStatus === "Proposed" || data.blockProposalStatus === "Missed"
+  ).length;
+
+  // Calculate the total successful block proposals
+  const totalSuccessfulBlockProposals = validatorData.filter((data) => data.blockProposalStatus === "Proposed").length;
+
+  return (totalSuccessfulBlockProposals / totalBlockProposalOpportunities) * 100;
+}
diff --git a/packages/brain/src/modules/validatorsPerformance/getStartAndEndEpochs.ts b/packages/brain/src/modules/validatorsPerformance/getStartAndEndEpochs.ts
new file mode 100644
index 00000000..b87a0539
--- /dev/null
+++ b/packages/brain/src/modules/validatorsPerformance/getStartAndEndEpochs.ts
@@ -0,0 +1,27 @@
+export function getStartAndEndEpochs(
+  minGenesisTime: number,
+  secondsPerSlot: number,
+  dateRange?: { startDate: Date; endDate: Date }
+): { startEpoch: number; endEpoch: number } {
+  if (dateRange)
+    return {
+      startEpoch: getEpochFromDate(dateRange.startDate, minGenesisTime, secondsPerSlot),
+      endEpoch: getEpochFromDate(dateRange.endDate, minGenesisTime, secondsPerSlot)
+    };
+  else {
+    // calculate the date from 7 days ago and its epoch
+    const sevenDaysAgo = new Date();
+    sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
+    return {
+      startEpoch: getEpochFromDate(sevenDaysAgo, minGenesisTime, secondsPerSlot),
+      endEpoch: getEpochFromDate(new Date(), minGenesisTime, secondsPerSlot)
+    };
+  }
+}
+
+function getEpochFromDate(date: Date, minGenesisTime: number, secondsPerSlot: number): number {
+  const currentUnixTime = Math.floor(date.getTime() / 1000);
+  const timeDifference = currentUnixTime - minGenesisTime; // Time difference in seconds
+  const slotsSinceGenesis = timeDifference / secondsPerSlot; // Slots since genesis
+  return Math.floor(slotsSinceGenesis / 32); // Current epoch
+}
diff --git a/packages/brain/src/modules/validatorsPerformance/index.ts b/packages/brain/src/modules/validatorsPerformance/index.ts
index 7fd016a1..12288817 100644
--- a/packages/brain/src/modules/validatorsPerformance/index.ts
+++ b/packages/brain/src/modules/validatorsPerformance/index.ts
@@ -1,63 +1,92 @@
 import { PostgresClient } from "../apiClients/index.js";
+import logger from "../logger/index.js";
+import { getStartAndEndEpochs } from "./getStartAndEndEpochs.js";
+import { calculateAttestationSuccessRate } from "./calculateAttestationSuccessRate.js";
+import { calculateBlocksProposedSuccessRate } from "./calculateBlocksProposedSuccessRate.js";
+import { ValidatorsPerformanceProcessed } from "./types.js";
 
-// Module in charge of queriyng the validators attestation rewards, block proposals and sync committee rewards and
-// processing the data to be displayed in the validators performance page.
-
-// FRONTEND
-
-// Will display the following data:
-// - Attestation success rate (not chart until granularity)
-// - Blocks proposed success rate (not chart until granularity)
-// - Sync committee success rate (not chart until granularity)
-// - Balance -> No chart
-// - Means: mean attestation success rate, mean blocks proposed success rate, mean balance -> No chart
-
-// BACKEND
-
-// The frontend will call backend with arguments:
-// - startDate and endDate -> backend will translate these dates to epochs.
-//      The backend will calculate  ValidatorsPerformanceProcessed for the given dates
-//      If no arguments passeed to backend then the backend will use last 7 days epoch and latest epoch
-// - Clients (execution and consensus) -> optional
-// - Attestation/block success rate granularity (future): admit granularity of att success rate: by epoch, by day, by week, by month -> THIS enables chart visualization
-
-// Return also current balance for each validator
+// Module in charge of querying and processin the data of the validators to get the performance metrics:
+// - Attestation success rate
+// - Blocks proposed success rate
+// - Mean attestation success rate
+// - Mean blocks proposed success rate
 
 // Note: It is overkill to store in db the attestation success rate for each epoch since it is only useful froma a global perspective
 // taking into account the historical data. As for now we will calculate dynamicall the attestation success rate with the arguments: epoch start and epoch end.
 
-//  (%) = (Number of Successful Attestations + Number of Successful Proposals) / (Total Attestation Opportunities + Total Proposal Opportunities) * 100
-// Total Attestation Opportunities: is the number of epochs between the first and last epoch in the data set of a specific validator.
-// Total Proposal Opportunities:
-
-// TODO: blocksProposedByEpochAndSlot
-
-export interface ValidatorsPerformanceProcessed {
-  mapValidatorPerformance: Map<
-    string,
-    {
-      attestationSuccessRate: number;
-      blocksProposedSuccessRate: number;
-      balance: number;
-      syncCommitteeSuccessRate?: number;
-    }
-  >;
-  meanAttestationSuccessRate: number;
-  meanBlocksProposedSuccessRate: number;
-  meanBalance: number;
-}
+// TODO: return current validator balance: 2 ways of doing it: 1) get the balance from the beaconchain API, 2) store the ideal rewards with the effective balance and get the balance from the postgres DB. The second option is more efficient but it is not real time.
+// TODO: return to the frontend the remaining seconds to next epoch. In the frontend use this parameter to query the backend every time the epoch changes.
+// TODO: add to block proposed epoch and slot
 
 /**
+ * Get the processed data for the validators in the given date range and the given validators indexes.
  *
+ * @param validatorIndexes - Array of validator indexes.
+ * @param postgresClient - Postgres client to interact with the DB.
+ * @param minGenesisTime - The genesis time of the chain.
+ * @param secondsPerSlot - The number of seconds per slot.
+ * @param dateRange - The date range to get the data from.
+ * @returns the processed data for the validators
  */
-export async function processValidatorsData({
+export async function getValidatorsDataProcessed({
   validatorIndexes,
-  postgresClient
+  postgresClient,
+  minGenesisTime,
+  secondsPerSlot,
+  dateRange
 }: {
   validatorIndexes: string[];
   postgresClient: PostgresClient;
-}) {
-  console.log("Processing validators data");
-  console.log("Validator indexes: ", validatorIndexes);
-  console.log("Postgres client: ", postgresClient);
+  minGenesisTime: number;
+  secondsPerSlot: number;
+  dateRange?: { startDate: Date; endDate: Date };
+}): Promise<ValidatorsPerformanceProcessed> {
+  logger.info("Processing validators data");
+
+  const mapValidatorPerformance: Map<string, { attestationSuccessRate: number; blocksProposedSuccessRate: number }> =
+    new Map();
+
+  // Calculate the epochs for the given dates, if no dates are given then use the last 7 days epoch and the latest epoch
+  const { startEpoch, endEpoch } = getStartAndEndEpochs(minGenesisTime, secondsPerSlot, dateRange);
+
+  // Get the validators data from the postgres database
+  const validatorsDataMap = await postgresClient.getValidatorsDataMapForEpochRange({
+    validatorIndexes,
+    startEpoch,
+    endEpoch
+  });
+
+  // Calculate the attestation success rate for each validator
+  for (const [validatorIndex, validatorData] of validatorsDataMap.entries())
+    mapValidatorPerformance.set(validatorIndex, {
+      attestationSuccessRate: calculateAttestationSuccessRate({
+        validatorData,
+        startEpoch,
+        endEpoch
+      }),
+      blocksProposedSuccessRate: calculateBlocksProposedSuccessRate({
+        validatorData
+      })
+    });
+
+  // Calculate the mean attestation success rate
+  const meanAttestationSuccessRate =
+    Array.from(mapValidatorPerformance.values()).reduce(
+      (acc, { attestationSuccessRate }) => acc + attestationSuccessRate,
+      0
+    ) / mapValidatorPerformance.size;
+
+  // Calculate the mean blocks proposed success rate
+  const meanBlocksProposedSuccessRate =
+    Array.from(mapValidatorPerformance.values()).reduce(
+      (acc, { blocksProposedSuccessRate }) => acc + blocksProposedSuccessRate,
+      0
+    ) / mapValidatorPerformance.size;
+
+  // Return the processed data
+  return {
+    mapValidatorPerformance,
+    meanAttestationSuccessRate,
+    meanBlocksProposedSuccessRate
+  };
 }
diff --git a/packages/brain/src/modules/validatorsPerformance/types.ts b/packages/brain/src/modules/validatorsPerformance/types.ts
new file mode 100644
index 00000000..b552d92d
--- /dev/null
+++ b/packages/brain/src/modules/validatorsPerformance/types.ts
@@ -0,0 +1,14 @@
+export interface ValidatorsPerformanceProcessed {
+  mapValidatorPerformance: Map<
+    string,
+    {
+      attestationSuccessRate: number;
+      blocksProposedSuccessRate: number;
+      //balance: number;
+      syncCommitteeSuccessRate?: number;
+    }
+  >;
+  meanAttestationSuccessRate: number;
+  meanBlocksProposedSuccessRate: number;
+  //meanBalance: number;
+}

From 16ed646dd77411bc90e0e3c3daafdd272275054d Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 11:14:28 +0200
Subject: [PATCH 02/22] fix enum

---
 packages/brain/src/modules/apiClients/postgres/types.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/brain/src/modules/apiClients/postgres/types.ts b/packages/brain/src/modules/apiClients/postgres/types.ts
index d16f1678..99b501e3 100644
--- a/packages/brain/src/modules/apiClients/postgres/types.ts
+++ b/packages/brain/src/modules/apiClients/postgres/types.ts
@@ -3,7 +3,8 @@ import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
 export enum BlockProposalStatus {
   Missed = "Missed",
   Proposed = "Proposed",
-  Unchosen = "Unchosen"
+  Unchosen = "Unchosen",
+  Error = "Error"
 }
 
 export interface AttestationsTotalRewards {

From 40bd2bb5af1564f5ec39ac78fab87692e1dbb554 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 11:52:39 +0200
Subject: [PATCH 03/22] fix cron while loop

---
 .../src/modules/cron/trackValidatorsPerformance/index.ts   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index e1a4cc8d..e8774a58 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -56,7 +56,7 @@ export async function trackValidatorsPerformance({
     const validatorsAttestationsTotalRewards: TotalRewards[] = [];
     const validatorBlockStatusMap: Map<string, BlockProposalStatus> = new Map();
 
-    label: while (epochFinalized === newEpochFinalized) {
+    while (epochFinalized === newEpochFinalized) {
       try {
         logger.debug(`${logPrefix}Epoch finalized: ${epochFinalized}`);
 
@@ -87,8 +87,9 @@ export async function trackValidatorsPerformance({
           validatorBlockStatusMap
         });
 
-        // update error to undefined if no error occurred in last iteration
+        // update error to undefined if no error occurred in last iteration and break the loop
         errorGettingValidatorData = undefined;
+        break;
       } catch (error) {
         logger.error(`${logPrefix}Error occurred: ${error}. Updating epoch finalized and retrying in 1 minute`);
         // update error if an error occurred
@@ -102,7 +103,7 @@ export async function trackValidatorsPerformance({
           );
           // TODO: collect report of the staker setup status: el is offline, node is syncing, signer is not up and original error
           // exit the while loop and write the error to the DB
-          break label;
+          break;
         }
         // wait 1 minute without blocking the event loop and update epoch finalized
         newEpochFinalized = await new Promise((resolve) =>

From 48fc37e728e58e092febd709866f2d24e879cad9 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 13:05:43 +0200
Subject: [PATCH 04/22] initialize variables

---
 .../getActiveValidatorsLoadedInBrain.ts       | 12 +++++-----
 .../getAttestationsTotalRewards.ts            | 16 ++++++--------
 .../cron/trackValidatorsPerformance/index.ts  | 22 +++++++++----------
 .../insertPerformanceData.ts                  | 16 +++++++-------
 .../setBlockProposalStatusMap.ts              | 15 +++++++------
 .../calculateAttestationSuccessRate.ts        |  7 ++----
 6 files changed, 40 insertions(+), 48 deletions(-)

diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.ts
index 61120a58..244f9db1 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.ts
@@ -16,16 +16,14 @@ import { logPrefix } from "./logPrefix.js";
  */
 export async function getActiveValidatorsLoadedInBrain({
   beaconchainApi,
-  brainDb,
-  activeValidatorsIndexes
+  brainDb
 }: {
   beaconchainApi: BeaconchainApi;
   brainDb: BrainDataBase;
-  activeValidatorsIndexes: string[];
-}): Promise<void> {
+}): Promise<string[]> {
   const validatorIndexes = await getValidatorIndexesAndSaveInDb({ beaconchainApi, brainDb });
-  if (validatorIndexes.length === 0) return;
-  (
+  if (validatorIndexes.length === 0) return [];
+  return (
     await beaconchainApi.postStateValidators({
       body: {
         ids: validatorIndexes,
@@ -33,7 +31,7 @@ export async function getActiveValidatorsLoadedInBrain({
       },
       stateId: "finalized"
     })
-  ).data.forEach((validator) => activeValidatorsIndexes.push(validator.index));
+  ).data.map((validator) => validator.index.toString());
 }
 
 /**
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/getAttestationsTotalRewards.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/getAttestationsTotalRewards.ts
index 67299aa0..7adb0a1e 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/getAttestationsTotalRewards.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/getAttestationsTotalRewards.ts
@@ -6,24 +6,22 @@ import { TotalRewards } from "../../apiClients/types.js";
  *
  * @param {BeaconchainApi} beaconchainApi - Beaconchain API client.
  * @param {string} epoch - The epoch to get the rewards.
- * @param {string[]} validatorIndexes - Array of validator indexes.
+ * @param {string[]} activeValidatorsIndexes - Array of active validator indexes.
  * @returns {TotalRewards[]} - Array of total rewards for the validators.
  */
 export async function getAttestationsTotalRewards({
   beaconchainApi,
   epoch,
-  validatorIndexes,
-  totalRewards
+  activeValidatorsIndexes
 }: {
   beaconchainApi: BeaconchainApi;
   epoch: string;
-  validatorIndexes: string[];
-  totalRewards: TotalRewards[];
-}): Promise<void> {
-  (
+  activeValidatorsIndexes: string[];
+}): Promise<TotalRewards[]> {
+  return (
     await beaconchainApi.getAttestationsRewards({
       epoch,
-      pubkeysOrIndexes: validatorIndexes
+      pubkeysOrIndexes: activeValidatorsIndexes
     })
-  ).data.total_rewards.forEach((reward) => totalRewards.push(reward));
+  ).data.total_rewards;
 }
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index e8774a58..63cfe948 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -52,16 +52,16 @@ export async function trackValidatorsPerformance({
     const epochFinalized = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
     let errorGettingValidatorData: Error | undefined;
     let newEpochFinalized = epochFinalized;
-    const activeValidatorsIndexes: string[] = [];
-    const validatorsAttestationsTotalRewards: TotalRewards[] = [];
-    const validatorBlockStatusMap: Map<string, BlockProposalStatus> = new Map();
+    let activeValidatorsIndexes: string[] = [];
+    let validatorsAttestationsTotalRewards: TotalRewards[] = [];
+    let validatorBlockStatusMap: Map<string, BlockProposalStatus> = new Map();
 
     while (epochFinalized === newEpochFinalized) {
       try {
         logger.debug(`${logPrefix}Epoch finalized: ${epochFinalized}`);
 
         // active validators indexes
-        await getActiveValidatorsLoadedInBrain({ beaconchainApi, brainDb, activeValidatorsIndexes });
+        activeValidatorsIndexes = await getActiveValidatorsLoadedInBrain({ beaconchainApi, brainDb });
         if (activeValidatorsIndexes.length === 0) {
           logger.info(`${logPrefix}No active validators found`);
           return;
@@ -72,19 +72,17 @@ export async function trackValidatorsPerformance({
         await checkNodeHealth({ beaconchainApi });
 
         // get block attestations rewards
-        await getAttestationsTotalRewards({
+        validatorsAttestationsTotalRewards = await getAttestationsTotalRewards({
           beaconchainApi,
           epoch: epochFinalized.toString(),
-          validatorIndexes: activeValidatorsIndexes,
-          totalRewards: validatorsAttestationsTotalRewards
+          activeValidatorsIndexes
         });
 
         // get block proposal status
-        await setBlockProposalStatusMap({
+        validatorBlockStatusMap = await setBlockProposalStatusMap({
           beaconchainApi,
           epoch: epochFinalized.toString(),
-          validatorIndexes: activeValidatorsIndexes,
-          validatorBlockStatusMap
+          activeValidatorsIndexes
         });
 
         // update error to undefined if no error occurred in last iteration and break the loop
@@ -119,9 +117,9 @@ export async function trackValidatorsPerformance({
     // insert performance data or each validator
     await insertPerformanceDataNotThrow({
       postgresClient,
-      validatorIndexes: activeValidatorsIndexes,
+      activeValidatorsIndexes,
       epochFinalized,
-      validatorBlockStatus: validatorBlockStatusMap,
+      validatorBlockStatusMap,
       validatorsAttestationsTotalRewards,
       error: errorGettingValidatorData,
       executionClient,
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
index bf8d3204..ad761fc8 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
@@ -11,31 +11,31 @@ import { logPrefix } from "./logPrefix.js";
  * with the next validator.
  *
  * @param postgresClient - Postgres client to interact with the DB.
- * @param validatorIndexes - Array of validator indexes.
+ * @param activeValidatorIndexes - Array of validator indexes.
  * @param epochFinalized - The epoch finalized.
- * @param validatorBlockStatus - Map with the block proposal status of each validator.
+ * @param validatorBlockStatusMap - Map with the block proposal status of each validator.
  * @param validatorsAttestationsTotalRewards - Array of total rewards for the validators.
  */
 export async function insertPerformanceDataNotThrow({
   postgresClient,
-  validatorIndexes,
+  activeValidatorsIndexes,
   epochFinalized,
-  validatorBlockStatus,
+  validatorBlockStatusMap,
   validatorsAttestationsTotalRewards,
   executionClient,
   consensusClient,
   error
 }: {
   postgresClient: PostgresClient;
-  validatorIndexes: string[];
+  activeValidatorsIndexes: string[];
   epochFinalized: number;
-  validatorBlockStatus: Map<string, BlockProposalStatus>;
+  validatorBlockStatusMap: Map<string, BlockProposalStatus>;
   validatorsAttestationsTotalRewards: TotalRewards[];
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
   error?: Error;
 }): Promise<void> {
-  for (const validatorIndex of validatorIndexes) {
+  for (const validatorIndex of activeValidatorsIndexes) {
     //const liveness = validatorsLiveness.find((liveness) => liveness.index === validatorIndex)?.is_live;
     const attestationsTotalRewards = validatorsAttestationsTotalRewards.find(
       (attestationReward) => attestationReward.validator_index === validatorIndex
@@ -46,7 +46,7 @@ export async function insertPerformanceDataNotThrow({
       continue;
     }
 
-    const blockProposalStatus = validatorBlockStatus.get(validatorIndex);
+    const blockProposalStatus = validatorBlockStatusMap.get(validatorIndex);
     if (!blockProposalStatus) {
       logger.error(
         `${logPrefix}Missing block proposal data for validator ${validatorIndex}, block: ${blockProposalStatus}`
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts
index d06aebb3..c5c40f16 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts
@@ -8,19 +8,19 @@ import { logPrefix } from "./logPrefix.js";
  *
  * @param {BeaconchainApi} beaconchainApi - Beaconchain API client.
  * @param {string} epoch - The epoch to get the block proposal duties.
- * @param {string[]} validatorIndexes - Array of validator indexes.
+ * @param {string[]} activeValidatorIndexes - Array of validator indexes.
  */
 export async function setBlockProposalStatusMap({
   beaconchainApi,
   epoch,
-  validatorIndexes,
-  validatorBlockStatusMap
+  activeValidatorsIndexes
 }: {
   beaconchainApi: BeaconchainApi;
   epoch: string;
-  validatorIndexes: string[];
-  validatorBlockStatusMap: Map<string, BlockProposalStatus>;
-}): Promise<void> {
+  activeValidatorsIndexes: string[];
+}): Promise<Map<string, BlockProposalStatus>> {
+  // Initialize the map with the block proposal status of each validator.
+  const validatorBlockStatusMap = new Map<string, BlockProposalStatus>();
   // Get the block proposal duties for the given epoch. Which validators
   // are supposed to propose a block in which slot?
   const blockProposalsResponse = await beaconchainApi.getProposerDuties({
@@ -28,7 +28,7 @@ export async function setBlockProposalStatusMap({
   });
 
   // Utilize a Set for quick lookup. We assume that the validator indexes are unique.
-  const validatorIndexesSet = new Set(validatorIndexes);
+  const validatorIndexesSet = new Set(activeValidatorsIndexes);
 
   // Initialize all validator's status to Unchosen.
   validatorIndexesSet.forEach((validatorIndex) => {
@@ -64,4 +64,5 @@ export async function setBlockProposalStatusMap({
       }
     }
   }
+  return validatorBlockStatusMap;
 }
diff --git a/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
index 8ccb3a7c..079d867d 100644
--- a/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
+++ b/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
@@ -3,7 +3,7 @@ import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
 /**
  * Calculates the attestation success rate for a given validator. The attestation success rate is the percentage of successful attestations
  * Being the total attestation opportunities the number of epochs between the first and last epoch in the data set of a specific validator.
- * And the total successful attestations the number of epochs where the validator successfully attested: head target and source must be > 0.
+ * And the total successful attestations the number of epochs where the validator successfully attested: source must be >= 0.
  *
  * @param validatorData the data of the validator from the postgres database
  * @param startEpoch the start epoch of the data set
@@ -23,10 +23,7 @@ export function calculateAttestationSuccessRate({
 
   // Calculate the total successful attestations
   const totalSuccessfulAttestations = validatorData.filter(
-    (data) =>
-      parseInt(data.attestationsTotalRewards.head) > 0 &&
-      parseInt(data.attestationsTotalRewards.target) > 0 &&
-      parseInt(data.attestationsTotalRewards.source) > 0
+    (data) => parseInt(data.attestationsTotalRewards.source) >= 0
   ).length;
 
   return (totalSuccessfulAttestations / totalAttestationOpportunities) * 100;

From 8eb46f714a9b80c5fec358c4e977dfc074e3d9fc Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 13:33:18 +0200
Subject: [PATCH 05/22] fix data insert

---
 packages/brain/src/modules/apiClients/postgres/index.ts       | 2 +-
 .../src/modules/cron/trackValidatorsPerformance/index.ts      | 1 -
 .../cron/trackValidatorsPerformance/insertPerformanceData.ts  | 4 ++--
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/packages/brain/src/modules/apiClients/postgres/index.ts b/packages/brain/src/modules/apiClients/postgres/index.ts
index e2c4cf50..35ad9fcd 100644
--- a/packages/brain/src/modules/apiClients/postgres/index.ts
+++ b/packages/brain/src/modules/apiClients/postgres/index.ts
@@ -154,7 +154,7 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
    */
   public async insertPerformanceData(data: ValidatorPerformance): Promise<void> {
     const query = `
-INSERT INTO ${this.tableName} (${Columns.validatorIndex}, ${Columns.epoch}, ${Columns.slot}, ${Columns.liveness}, ${Columns.blockProposalStatus}, ${Columns.syncCommitteeRewards}, ${Columns.attestationsTotalRewards}, ${Columns.error})
+INSERT INTO ${this.tableName} (${Columns.validatorIndex}, ${Columns.epoch}, ${Columns.executionClient}, ${Columns.consensusClient}, ${Columns.slot}, ${Columns.liveness}, ${Columns.blockProposalStatus}, ${Columns.syncCommitteeRewards}, ${Columns.attestationsTotalRewards}, ${Columns.error})
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
     `;
 
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index 63cfe948..bd44ea54 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -125,7 +125,6 @@ export async function trackValidatorsPerformance({
       executionClient,
       consensusClient
     });
-    logger.debug(`${logPrefix}Performance data inserted for epoch ${epochFinalized}`);
   } catch (e) {
     logger.error(`${logPrefix}Error in trackValidatorsPerformance: ${e}`);
     return;
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
index ad761fc8..c591650a 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
@@ -54,9 +54,8 @@ export async function insertPerformanceDataNotThrow({
       continue;
     }
 
-    // write on db
-    logger.debug(`${logPrefix}Inserting performance data for validator ${validatorIndex}`);
     try {
+      logger.debug(`${logPrefix}Inserting performance data for validator ${validatorIndex}`);
       await postgresClient.insertPerformanceData({
         validatorIndex: parseInt(validatorIndex),
         epoch: epochFinalized,
@@ -66,6 +65,7 @@ export async function insertPerformanceDataNotThrow({
         executionClient,
         consensusClient
       });
+      logger.debug(`${logPrefix}Performance data inserted for epoch ${epochFinalized}`);
     } catch (e) {
       logger.error(`${logPrefix}Error inserting performance data for validator ${validatorIndex}: ${e}`);
       continue;

From 2fe5ef0fd06856603e45e2b50819215b1deb2b55 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 13:44:27 +0200
Subject: [PATCH 06/22] add unit testing files

---
 .../calculateAttestationSuccessRate.ts              |  0
 .../calculateBlocksProposedSuccessRate.ts           |  0
 .../getStartAndEndEpochs.ts                         |  0
 .../index.ts                                        |  2 +-
 .../types.ts                                        |  0
 .../calculateAttestationSuccessRate.unit.test.ts    |  0
 .../calculateBlocksProposedSuccessRate.unit.test.ts |  0
 .../getStartAndEndEpochs.unit.test.ts               |  0
 .../modules/validatorsDataIngest/index.test.unit.ts | 13 +++++++++++++
 9 files changed, 14 insertions(+), 1 deletion(-)
 rename packages/brain/src/modules/{validatorsPerformance => validatorsDataIngest}/calculateAttestationSuccessRate.ts (100%)
 rename packages/brain/src/modules/{validatorsPerformance => validatorsDataIngest}/calculateBlocksProposedSuccessRate.ts (100%)
 rename packages/brain/src/modules/{validatorsPerformance => validatorsDataIngest}/getStartAndEndEpochs.ts (100%)
 rename packages/brain/src/modules/{validatorsPerformance => validatorsDataIngest}/index.ts (98%)
 rename packages/brain/src/modules/{validatorsPerformance => validatorsDataIngest}/types.ts (100%)
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/calculateAttestationSuccessRate.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts

diff --git a/packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
similarity index 100%
rename from packages/brain/src/modules/validatorsPerformance/calculateAttestationSuccessRate.ts
rename to packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
diff --git a/packages/brain/src/modules/validatorsPerformance/calculateBlocksProposedSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.ts
similarity index 100%
rename from packages/brain/src/modules/validatorsPerformance/calculateBlocksProposedSuccessRate.ts
rename to packages/brain/src/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.ts
diff --git a/packages/brain/src/modules/validatorsPerformance/getStartAndEndEpochs.ts b/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
similarity index 100%
rename from packages/brain/src/modules/validatorsPerformance/getStartAndEndEpochs.ts
rename to packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
diff --git a/packages/brain/src/modules/validatorsPerformance/index.ts b/packages/brain/src/modules/validatorsDataIngest/index.ts
similarity index 98%
rename from packages/brain/src/modules/validatorsPerformance/index.ts
rename to packages/brain/src/modules/validatorsDataIngest/index.ts
index 12288817..49efeb6e 100644
--- a/packages/brain/src/modules/validatorsPerformance/index.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/index.ts
@@ -28,7 +28,7 @@ import { ValidatorsPerformanceProcessed } from "./types.js";
  * @param dateRange - The date range to get the data from.
  * @returns the processed data for the validators
  */
-export async function getValidatorsDataProcessed({
+export async function fetchAndProcessValidatorsData({
   validatorIndexes,
   postgresClient,
   minGenesisTime,
diff --git a/packages/brain/src/modules/validatorsPerformance/types.ts b/packages/brain/src/modules/validatorsDataIngest/types.ts
similarity index 100%
rename from packages/brain/src/modules/validatorsPerformance/types.ts
rename to packages/brain/src/modules/validatorsDataIngest/types.ts
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/calculateAttestationSuccessRate.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/calculateAttestationSuccessRate.unit.test.ts
new file mode 100644
index 00000000..e69de29b
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.unit.test.ts
new file mode 100644
index 00000000..e69de29b
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
new file mode 100644
index 00000000..e69de29b
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts b/packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts
new file mode 100644
index 00000000..3fae9701
--- /dev/null
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts
@@ -0,0 +1,13 @@
+import { PostgresClient } from "../../../../src/modules/apiClients/index.js";
+import { fetchAndProcessValidatorsData } from "../../../../src/modules/validatorsDataIngest/index.js";
+
+// This test must be executed with a real database connection
+
+describe("Validators data ingest", function () {
+  this.timeout(10 * 1000);
+  // change the dbUrl on demmand
+  const dbUrl = "postgres://postgres:password@postgres.web3signer-holesky.dappnode:5432/web3signer";
+  const postgresClient = new PostgresClient(dbUrl);
+
+  it("should fetch and process validators data", async () => {});
+});

From 5652ea71f2ef140c7b59f30ba71f96d53e7a633b Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 13:45:51 +0200
Subject: [PATCH 07/22] fix insert data number of columns

---
 packages/brain/src/modules/apiClients/postgres/index.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/brain/src/modules/apiClients/postgres/index.ts b/packages/brain/src/modules/apiClients/postgres/index.ts
index 35ad9fcd..7ad24e64 100644
--- a/packages/brain/src/modules/apiClients/postgres/index.ts
+++ b/packages/brain/src/modules/apiClients/postgres/index.ts
@@ -155,7 +155,7 @@ CREATE TABLE IF NOT EXISTS ${this.tableName} (
   public async insertPerformanceData(data: ValidatorPerformance): Promise<void> {
     const query = `
 INSERT INTO ${this.tableName} (${Columns.validatorIndex}, ${Columns.epoch}, ${Columns.executionClient}, ${Columns.consensusClient}, ${Columns.slot}, ${Columns.liveness}, ${Columns.blockProposalStatus}, ${Columns.syncCommitteeRewards}, ${Columns.attestationsTotalRewards}, ${Columns.error})
-VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
     `;
 
     await this.sql.unsafe(query, [

From c8f9af55d4d080ccb671fc9ba9a3165380ddf573 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 21:35:57 +0200
Subject: [PATCH 08/22] implement getIntervalsEpochs

---
 .../calculateBlocksProposedSuccessRate.ts     | 25 ------
 .../getIntervalsEpochs.ts                     | 41 +++++++++
 .../getStartAndEndEpochs.ts                   | 34 ++++----
 .../src/modules/validatorsDataIngest/index.ts | 85 ++++++++++---------
 .../src/modules/validatorsDataIngest/types.ts | 33 ++++---
 ...est.ts => getIntervalsEpochs.unit.test.ts} |  0
 6 files changed, 121 insertions(+), 97 deletions(-)
 delete mode 100644 packages/brain/src/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.ts
 create mode 100644 packages/brain/src/modules/validatorsDataIngest/getIntervalsEpochs.ts
 rename packages/brain/test/unit/modules/validatorsDataIngest/{calculateBlocksProposedSuccessRate.unit.test.ts => getIntervalsEpochs.unit.test.ts} (100%)

diff --git a/packages/brain/src/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.ts
deleted file mode 100644
index 78f9f561..00000000
--- a/packages/brain/src/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
-
-/**
- * Calculates the blocks proposed success rate for a given validator. The blocks proposed success rate is the percentage of successful block proposals
- * Being the total block proposal opportunities the number of epochs where the validator proposed a block or missed a block.
- * And the total successful block proposals the number of epochs where the validator successfully proposed a block.
- *
- * @param validatorData the data of the validator from the postgres database
- * @returns the blocks proposed success rate
- */
-export function calculateBlocksProposedSuccessRate({
-  validatorData
-}: {
-  validatorData: ValidatorPerformance[];
-}): number {
-  // Calculate the total block proposal opportunities
-  const totalBlockProposalOpportunities = validatorData.filter(
-    (data) => data.blockProposalStatus === "Proposed" || data.blockProposalStatus === "Missed"
-  ).length;
-
-  // Calculate the total successful block proposals
-  const totalSuccessfulBlockProposals = validatorData.filter((data) => data.blockProposalStatus === "Proposed").length;
-
-  return (totalSuccessfulBlockProposals / totalBlockProposalOpportunities) * 100;
-}
diff --git a/packages/brain/src/modules/validatorsDataIngest/getIntervalsEpochs.ts b/packages/brain/src/modules/validatorsDataIngest/getIntervalsEpochs.ts
new file mode 100644
index 00000000..10bef2c9
--- /dev/null
+++ b/packages/brain/src/modules/validatorsDataIngest/getIntervalsEpochs.ts
@@ -0,0 +1,41 @@
+import { getStartAndEndEpochs } from "./getStartAndEndEpochs.js";
+import { Granularity } from "./types.js";
+
+export function getIntervalsEpochs({
+  startDate,
+  endDate,
+  granularity,
+  minGenesisTime,
+  secondsPerSlot
+}: {
+  startDate: Date;
+  endDate: Date;
+  granularity: Granularity;
+  minGenesisTime: number;
+  secondsPerSlot: number;
+}): { startEpoch: number; endEpoch: number }[] {
+  // Calculate the number of intervals based on the granularity
+  const numberOfIntervals = getNumberOfIntervals({ startDate, endDate, granularity });
+  return Array.from({ length: numberOfIntervals }, (_, idx) => {
+    return getStartAndEndEpochs({
+      minGenesisTime,
+      secondsPerSlot,
+      startDate: new Date(startDate.getTime() + idx * granularity),
+      endDate: new Date(startDate.getTime() + (idx + 1) * granularity)
+    });
+  });
+}
+
+function getNumberOfIntervals({
+  startDate,
+  endDate,
+  granularity
+}: {
+  startDate: Date;
+  endDate: Date;
+  granularity: Granularity;
+}): number {
+  // Calculate the total amount of time based on the granularity
+  const totalAmountOfTime = endDate.getTime() - startDate.getTime();
+  return Math.floor(totalAmountOfTime / granularity); // Use Math.floor for proper interval count
+}
diff --git a/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts b/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
index b87a0539..30fdd16a 100644
--- a/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
@@ -1,22 +1,18 @@
-export function getStartAndEndEpochs(
-  minGenesisTime: number,
-  secondsPerSlot: number,
-  dateRange?: { startDate: Date; endDate: Date }
-): { startEpoch: number; endEpoch: number } {
-  if (dateRange)
-    return {
-      startEpoch: getEpochFromDate(dateRange.startDate, minGenesisTime, secondsPerSlot),
-      endEpoch: getEpochFromDate(dateRange.endDate, minGenesisTime, secondsPerSlot)
-    };
-  else {
-    // calculate the date from 7 days ago and its epoch
-    const sevenDaysAgo = new Date();
-    sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
-    return {
-      startEpoch: getEpochFromDate(sevenDaysAgo, minGenesisTime, secondsPerSlot),
-      endEpoch: getEpochFromDate(new Date(), minGenesisTime, secondsPerSlot)
-    };
-  }
+export function getStartAndEndEpochs({
+  minGenesisTime,
+  secondsPerSlot,
+  startDate,
+  endDate
+}: {
+  minGenesisTime: number;
+  secondsPerSlot: number;
+  startDate: Date;
+  endDate: Date;
+}): { startEpoch: number; endEpoch: number } {
+  return {
+    startEpoch: getEpochFromDate(startDate, minGenesisTime, secondsPerSlot),
+    endEpoch: getEpochFromDate(endDate, minGenesisTime, secondsPerSlot)
+  };
 }
 
 function getEpochFromDate(date: Date, minGenesisTime: number, secondsPerSlot: number): number {
diff --git a/packages/brain/src/modules/validatorsDataIngest/index.ts b/packages/brain/src/modules/validatorsDataIngest/index.ts
index 49efeb6e..ca14ef2c 100644
--- a/packages/brain/src/modules/validatorsDataIngest/index.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/index.ts
@@ -2,8 +2,8 @@ import { PostgresClient } from "../apiClients/index.js";
 import logger from "../logger/index.js";
 import { getStartAndEndEpochs } from "./getStartAndEndEpochs.js";
 import { calculateAttestationSuccessRate } from "./calculateAttestationSuccessRate.js";
-import { calculateBlocksProposedSuccessRate } from "./calculateBlocksProposedSuccessRate.js";
-import { ValidatorsPerformanceProcessed } from "./types.js";
+import { Granularity, NumberOfDaysToQuery, ValidatorsDataProcessed } from "./types.js";
+import { getIntervalsEpochs } from "./getIntervalsEpochs.js";
 
 // Module in charge of querying and processin the data of the validators to get the performance metrics:
 // - Attestation success rate
@@ -14,9 +14,15 @@ import { ValidatorsPerformanceProcessed } from "./types.js";
 // Note: It is overkill to store in db the attestation success rate for each epoch since it is only useful froma a global perspective
 // taking into account the historical data. As for now we will calculate dynamicall the attestation success rate with the arguments: epoch start and epoch end.
 
-// TODO: return current validator balance: 2 ways of doing it: 1) get the balance from the beaconchain API, 2) store the ideal rewards with the effective balance and get the balance from the postgres DB. The second option is more efficient but it is not real time.
+// TODO: return current validator balance: 2 ways of doing it: 1) **get the balance from the beaconchain API**, 2) store the ideal rewards with the effective balance and get the balance from the postgres DB. The second option is more efficient but it is not real time.
 // TODO: return to the frontend the remaining seconds to next epoch. In the frontend use this parameter to query the backend every time the epoch changes.
 // TODO: add to block proposed epoch and slot
+// TODO: implement first epoch limit?
+
+// GRANULARITY AND START/END DATE ALLOWED -> only from past to present not from past to past
+// - 1 day: granularity allowed HOURLY
+// - 7 days: granularity allowed HOURLY and DAILY
+// - 1 month (28 days): granularity allowed HOURLY, DAILY and WEEKLY
 
 /**
  * Get the processed data for the validators in the given date range and the given validators indexes.
@@ -33,23 +39,36 @@ export async function fetchAndProcessValidatorsData({
   postgresClient,
   minGenesisTime,
   secondsPerSlot,
-  dateRange
+  numberOfDaysToQuery = 1,
+  granularity = Granularity.Hourly
 }: {
   validatorIndexes: string[];
-  postgresClient: PostgresClient;
-  minGenesisTime: number;
-  secondsPerSlot: number;
-  dateRange?: { startDate: Date; endDate: Date };
-}): Promise<ValidatorsPerformanceProcessed> {
+  postgresClient: PostgresClient; // import from backend index
+  minGenesisTime: number; // import from backend index
+  secondsPerSlot: number; // immport from backend index
+  numberOfDaysToQuery?: NumberOfDaysToQuery;
+  granularity?: Granularity;
+}): Promise<
+  Map<
+    string, // validatorIndex
+    ValidatorsDataProcessed // processed data of the validator
+  >
+> {
   logger.info("Processing validators data");
+  const mapValidatorPerformance = new Map<string, ValidatorsDataProcessed>();
+
+  // Get start timestamp and end timestamp
+  const endDate = new Date();
+  const startDate = new Date();
+  startDate.setDate(endDate.getDate() - numberOfDaysToQuery);
 
-  const mapValidatorPerformance: Map<string, { attestationSuccessRate: number; blocksProposedSuccessRate: number }> =
-    new Map();
+  // Calculate the epochs for the given dates
+  const { startEpoch, endEpoch } = getStartAndEndEpochs({ minGenesisTime, secondsPerSlot, startDate, endDate });
 
-  // Calculate the epochs for the given dates, if no dates are given then use the last 7 days epoch and the latest epoch
-  const { startEpoch, endEpoch } = getStartAndEndEpochs(minGenesisTime, secondsPerSlot, dateRange);
+  // Get the start and end epochs for each interval
+  const intervals = getIntervalsEpochs({ startDate, endDate, granularity, minGenesisTime, secondsPerSlot });
 
-  // Get the validators data from the postgres database
+  // Get the validators data from the postgres database with the start and end epoch
   const validatorsDataMap = await postgresClient.getValidatorsDataMapForEpochRange({
     validatorIndexes,
     startEpoch,
@@ -59,34 +78,20 @@ export async function fetchAndProcessValidatorsData({
   // Calculate the attestation success rate for each validator
   for (const [validatorIndex, validatorData] of validatorsDataMap.entries())
     mapValidatorPerformance.set(validatorIndex, {
-      attestationSuccessRate: calculateAttestationSuccessRate({
-        validatorData,
-        startEpoch,
-        endEpoch
+      attestationSuccessRate: calculateAttestationSuccessRate({ validatorData, startEpoch, endEpoch }),
+      attestationSuccessRatePerInterval: intervals.map(({ startEpoch, endEpoch }) => {
+        return {
+          startEpoch,
+          endEpoch,
+          attestationSuccessRate: calculateAttestationSuccessRate({ validatorData, startEpoch, endEpoch })
+        };
       }),
-      blocksProposedSuccessRate: calculateBlocksProposedSuccessRate({
-        validatorData
-      })
+      blocks: {
+        proposed: validatorData.filter((data) => data.blockProposalStatus === "Proposed").length,
+        missed: validatorData.filter((data) => data.blockProposalStatus === "Missed").length
+      }
     });
 
-  // Calculate the mean attestation success rate
-  const meanAttestationSuccessRate =
-    Array.from(mapValidatorPerformance.values()).reduce(
-      (acc, { attestationSuccessRate }) => acc + attestationSuccessRate,
-      0
-    ) / mapValidatorPerformance.size;
-
-  // Calculate the mean blocks proposed success rate
-  const meanBlocksProposedSuccessRate =
-    Array.from(mapValidatorPerformance.values()).reduce(
-      (acc, { blocksProposedSuccessRate }) => acc + blocksProposedSuccessRate,
-      0
-    ) / mapValidatorPerformance.size;
-
   // Return the processed data
-  return {
-    mapValidatorPerformance,
-    meanAttestationSuccessRate,
-    meanBlocksProposedSuccessRate
-  };
+  return mapValidatorPerformance;
 }
diff --git a/packages/brain/src/modules/validatorsDataIngest/types.ts b/packages/brain/src/modules/validatorsDataIngest/types.ts
index b552d92d..8cc25fa0 100644
--- a/packages/brain/src/modules/validatorsDataIngest/types.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/types.ts
@@ -1,14 +1,21 @@
-export interface ValidatorsPerformanceProcessed {
-  mapValidatorPerformance: Map<
-    string,
-    {
-      attestationSuccessRate: number;
-      blocksProposedSuccessRate: number;
-      //balance: number;
-      syncCommitteeSuccessRate?: number;
-    }
-  >;
-  meanAttestationSuccessRate: number;
-  meanBlocksProposedSuccessRate: number;
-  //meanBalance: number;
+export interface ValidatorsDataProcessed {
+  attestationSuccessRate: number; // mean attestationSuccessRate of the validator
+  // attestationSuccessRate in each interval
+  attestationSuccessRatePerInterval: {
+    startEpoch: number; // start epoch of the interval
+    endEpoch: number; // end epoch of the interval
+    attestationSuccessRate: number | null; // attestationSuccessRate in the interval
+  }[];
+  blocks: {
+    proposed: number; // number of blocks proposed
+    missed: number; // number of blocks missed
+  };
 }
+
+export enum Granularity {
+  Hourly = 3600000, // 1 hour in milliseconds
+  Daily = 86400000, // 1 day in milliseconds
+  Weekly = 604800000 // 7 days in milliseconds
+}
+
+export type NumberOfDaysToQuery = 1 | 7 | 28;
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getIntervalsEpochs.unit.test.ts
similarity index 100%
rename from packages/brain/test/unit/modules/validatorsDataIngest/calculateBlocksProposedSuccessRate.unit.test.ts
rename to packages/brain/test/unit/modules/validatorsDataIngest/getIntervalsEpochs.unit.test.ts

From 27d27e003a8e22b0f50a548940fd281dae42b4f8 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 22:24:20 +0200
Subject: [PATCH 09/22] rename test

---
 .../{index.test.unit.ts => index.unit.test.ts}   | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)
 rename packages/brain/test/unit/modules/validatorsDataIngest/{index.test.unit.ts => index.unit.test.ts} (56%)

diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts b/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
similarity index 56%
rename from packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts
rename to packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
index 3fae9701..21b53bd3 100644
--- a/packages/brain/test/unit/modules/validatorsDataIngest/index.test.unit.ts
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
@@ -9,5 +9,19 @@ describe("Validators data ingest", function () {
   const dbUrl = "postgres://postgres:password@postgres.web3signer-holesky.dappnode:5432/web3signer";
   const postgresClient = new PostgresClient(dbUrl);
 
-  it("should fetch and process validators data", async () => {});
+  it("should fetch and process validators data", async () => {
+    const validatorIndexes = ["1802289", "1802258"];
+    const minGenesisTime = 1695902100;
+    const secondsPerSlot = 12;
+
+    const data = await fetchAndProcessValidatorsData({
+      validatorIndexes,
+      postgresClient,
+      minGenesisTime,
+      secondsPerSlot,
+      numberOfDaysToQuery: 1
+    });
+
+    console.log(data);
+  });
 });

From 06c4dae1b9ac6f6b7f3f379e074d488bf0e39980 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 22:46:25 +0200
Subject: [PATCH 10/22] fix type

---
 packages/brain/src/modules/apiClients/postgres/types.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/brain/src/modules/apiClients/postgres/types.ts b/packages/brain/src/modules/apiClients/postgres/types.ts
index 99b501e3..84335fdd 100644
--- a/packages/brain/src/modules/apiClients/postgres/types.ts
+++ b/packages/brain/src/modules/apiClients/postgres/types.ts
@@ -22,7 +22,7 @@ export interface ValidatorPerformance {
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
   blockProposalStatus: BlockProposalStatus;
-  attestationsTotalRewards: AttestationsTotalRewards;
+  attestationsTotalRewards: string; // JSON string AttestationsTotalRewards
   slot?: number;
   liveness?: boolean;
   syncCommitteeRewards?: number;

From 8a14a1c106896fa1ac8018013f5ebf762b1a87d6 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 22:46:35 +0200
Subject: [PATCH 11/22] parse attestationreward

---
 .../calculateAttestationSuccessRate.ts               | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
index 079d867d..c2985d5e 100644
--- a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
@@ -1,4 +1,5 @@
 import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
+import logger from "../logger/index.js";
 
 /**
  * Calculates the attestation success rate for a given validator. The attestation success rate is the percentage of successful attestations
@@ -20,11 +21,16 @@ export function calculateAttestationSuccessRate({
 }): number {
   // Calculate the total attestation opportunities
   const totalAttestationOpportunities = endEpoch - startEpoch;
+  if (totalAttestationOpportunities <= 0) {
+    logger.warn("totalAttestationOpportunities is less than or equal to 0");
+    return 0;
+  }
 
   // Calculate the total successful attestations
-  const totalSuccessfulAttestations = validatorData.filter(
-    (data) => parseInt(data.attestationsTotalRewards.source) >= 0
-  ).length;
+  const totalSuccessfulAttestations = validatorData.filter((data) => {
+    const rewards = JSON.parse(data.attestationsTotalRewards); // Parse the JSON
+    return rewards.source >= 0; // Check if source is non-negative
+  }).length;
 
   return (totalSuccessfulAttestations / totalAttestationOpportunities) * 100;
 }

From 9c470c203b44482991e7f7a9959062440268dc1b Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Mon, 23 Sep 2024 22:47:11 +0200
Subject: [PATCH 12/22] use math round

---
 .../validatorsDataIngest/calculateAttestationSuccessRate.ts     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
index c2985d5e..a236e82e 100644
--- a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
@@ -32,5 +32,5 @@ export function calculateAttestationSuccessRate({
     return rewards.source >= 0; // Check if source is non-negative
   }).length;
 
-  return (totalSuccessfulAttestations / totalAttestationOpportunities) * 100;
+  return Math.round(totalSuccessfulAttestations / totalAttestationOpportunities) * 100;
 }

From 03d758096db575f26f3d41f10ebab4d4863d4ca3 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Tue, 24 Sep 2024 09:22:31 +0200
Subject: [PATCH 13/22] parse total rewards

---
 .../brain/src/modules/apiClients/postgres/index.ts    |  4 ++--
 .../brain/src/modules/apiClients/postgres/types.ts    |  2 +-
 .../calculateAttestationSuccessRate.ts                | 11 ++++++-----
 .../brain/src/modules/validatorsDataIngest/index.ts   |  6 ------
 4 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/packages/brain/src/modules/apiClients/postgres/index.ts b/packages/brain/src/modules/apiClients/postgres/index.ts
index 7ad24e64..cd82a004 100644
--- a/packages/brain/src/modules/apiClients/postgres/index.ts
+++ b/packages/brain/src/modules/apiClients/postgres/index.ts
@@ -197,7 +197,7 @@ WHERE ${Columns.validatorIndex} = ANY($1)
       liveness: row.liveness,
       blockProposalStatus: row.block_proposal_status,
       syncCommitteeRewards: row.sync_comittee_rewards,
-      attestationsTotalRewards: row.attestations_total_rewards,
+      attestationsTotalRewards: JSON.parse(row.attestations_total_rewards),
       error: row.error
     }));
   }
@@ -241,7 +241,7 @@ AND ${Columns.epoch} <= $3
         liveness: row.liveness,
         blockProposalStatus: row.block_proposal_status,
         syncCommitteeRewards: row.sync_comittee_rewards,
-        attestationsTotalRewards: row.attestations_total_rewards,
+        attestationsTotalRewards: JSON.parse(row.attestations_total_rewards),
         error: row.error
       };
 
diff --git a/packages/brain/src/modules/apiClients/postgres/types.ts b/packages/brain/src/modules/apiClients/postgres/types.ts
index 84335fdd..99b501e3 100644
--- a/packages/brain/src/modules/apiClients/postgres/types.ts
+++ b/packages/brain/src/modules/apiClients/postgres/types.ts
@@ -22,7 +22,7 @@ export interface ValidatorPerformance {
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
   blockProposalStatus: BlockProposalStatus;
-  attestationsTotalRewards: string; // JSON string AttestationsTotalRewards
+  attestationsTotalRewards: AttestationsTotalRewards;
   slot?: number;
   liveness?: boolean;
   syncCommitteeRewards?: number;
diff --git a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
index a236e82e..959f9dd8 100644
--- a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
@@ -6,6 +6,8 @@ import logger from "../logger/index.js";
  * Being the total attestation opportunities the number of epochs between the first and last epoch in the data set of a specific validator.
  * And the total successful attestations the number of epochs where the validator successfully attested: source must be >= 0.
  *
+ * The epoch must be greater or equal to the startEpoch and less than the endEpoch.
+ *
  * @param validatorData the data of the validator from the postgres database
  * @param startEpoch the start epoch of the data set
  * @param endEpoch the end epoch of the data set
@@ -27,10 +29,9 @@ export function calculateAttestationSuccessRate({
   }
 
   // Calculate the total successful attestations
-  const totalSuccessfulAttestations = validatorData.filter((data) => {
-    const rewards = JSON.parse(data.attestationsTotalRewards); // Parse the JSON
-    return rewards.source >= 0; // Check if source is non-negative
-  }).length;
+  const totalSuccessfulAttestations = validatorData.filter(
+    (data) => data.epoch >= startEpoch && data.epoch < endEpoch && parseInt(data.attestationsTotalRewards.source) >= 0
+  ).length;
 
-  return Math.round(totalSuccessfulAttestations / totalAttestationOpportunities) * 100;
+  return Math.round((totalSuccessfulAttestations / totalAttestationOpportunities) * 100);
 }
diff --git a/packages/brain/src/modules/validatorsDataIngest/index.ts b/packages/brain/src/modules/validatorsDataIngest/index.ts
index ca14ef2c..fbb320e0 100644
--- a/packages/brain/src/modules/validatorsDataIngest/index.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/index.ts
@@ -17,12 +17,6 @@ import { getIntervalsEpochs } from "./getIntervalsEpochs.js";
 // TODO: return current validator balance: 2 ways of doing it: 1) **get the balance from the beaconchain API**, 2) store the ideal rewards with the effective balance and get the balance from the postgres DB. The second option is more efficient but it is not real time.
 // TODO: return to the frontend the remaining seconds to next epoch. In the frontend use this parameter to query the backend every time the epoch changes.
 // TODO: add to block proposed epoch and slot
-// TODO: implement first epoch limit?
-
-// GRANULARITY AND START/END DATE ALLOWED -> only from past to present not from past to past
-// - 1 day: granularity allowed HOURLY
-// - 7 days: granularity allowed HOURLY and DAILY
-// - 1 month (28 days): granularity allowed HOURLY, DAILY and WEEKLY
 
 /**
  * Get the processed data for the validators in the given date range and the given validators indexes.

From 59de4cb4e7c13903f2572a439dcbe01dad08260f Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Tue, 24 Sep 2024 09:47:48 +0200
Subject: [PATCH 14/22] add start and end epoch unit testing

---
 .../getStartAndEndEpochs.ts                   |  9 ++++++
 .../getStartAndEndEpochs.unit.test.ts         | 32 +++++++++++++++++++
 2 files changed, 41 insertions(+)

diff --git a/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts b/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
index 30fdd16a..604c21b0 100644
--- a/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/getStartAndEndEpochs.ts
@@ -1,3 +1,12 @@
+/**
+ * Get the start and end epochs for the given date range
+ *
+ * @param minGenesisTime - The minimum genesis time of the chain
+ * @param secondsPerSlot - The number of seconds per slot in the chain
+ * @param startDate - The start date of the date range
+ * @param endDate - The end date of the date range
+ * @returns the start and end epochs for the given date range
+ */
 export function getStartAndEndEpochs({
   minGenesisTime,
   secondsPerSlot,
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
index e69de29b..b764a06a 100644
--- a/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
@@ -0,0 +1,32 @@
+import { getStartAndEndEpochs } from "../../../../dist/modules/validatorsDataIngest/getStartAndEndEpochs.js";
+import { expect } from "chai";
+
+describe("getStartAndEndEpochs", () => {
+  it("should return correct start and end epochs for the given date range", () => {
+    // Define constants
+    const minGenesisTime = 1695902100; // Use the provided minGenesisTime
+    const secondsPerSlot = 12; // Use the provided secondsPerSlot
+
+    // Define date range for testing
+    const startDate = new Date("2024-09-24T06:40:00.000Z");
+    const endDate = new Date("2024-09-24T07:05:36.000Z");
+
+    // Calculate expected epochs
+    const expectedStartEpoch = 81400;
+    const expectedEndEpoch = 81404;
+
+    // Call the function
+    const result = getStartAndEndEpochs({
+      minGenesisTime,
+      secondsPerSlot,
+      startDate,
+      endDate
+    });
+
+    // Assert the results
+    expect(result).to.deep.equal({
+      startEpoch: expectedStartEpoch,
+      endEpoch: expectedEndEpoch
+    });
+  });
+});

From c3b0ff33b5859c10b58543a0f562f27b64674c93 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Tue, 24 Sep 2024 11:57:18 +0200
Subject: [PATCH 15/22] add missing returns att rate per clients

---
 ...ssRate.ts => getAttestationSuccessRate.ts} |  2 +-
 .../getAttestationSuccessRatePerClients.ts    | 35 +++++++++++++++++++
 .../getClientsUsedInInterval.ts               | 27 ++++++++++++++
 .../src/modules/validatorsDataIngest/index.ts | 10 ++++--
 .../src/modules/validatorsDataIngest/types.ts | 11 ++++++
 .../validatorsDataIngest/index.unit.test.ts   |  4 ++-
 6 files changed, 84 insertions(+), 5 deletions(-)
 rename packages/brain/src/modules/validatorsDataIngest/{calculateAttestationSuccessRate.ts => getAttestationSuccessRate.ts} (96%)
 create mode 100644 packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.ts
 create mode 100644 packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts

diff --git a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts b/packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRate.ts
similarity index 96%
rename from packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
rename to packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRate.ts
index 959f9dd8..0a531d7d 100644
--- a/packages/brain/src/modules/validatorsDataIngest/calculateAttestationSuccessRate.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRate.ts
@@ -12,7 +12,7 @@ import logger from "../logger/index.js";
  * @param startEpoch the start epoch of the data set
  * @param endEpoch the end epoch of the data set
  */
-export function calculateAttestationSuccessRate({
+export function getAttestationSuccessRate({
   validatorData,
   startEpoch,
   endEpoch
diff --git a/packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.ts b/packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.ts
new file mode 100644
index 00000000..f7fd9fe6
--- /dev/null
+++ b/packages/brain/src/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.ts
@@ -0,0 +1,35 @@
+import { getAttestationSuccessRate } from "./getAttestationSuccessRate.js";
+import type { ExecutionConsensusConcatenated } from "./types.js";
+import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
+
+/**
+ * Calculates the attestation success rate for a given validator per Execution and Consensus client.
+ *
+ * @param validatorData the data of the validator from the postgres database
+ * @param startEpoch the start epoch of the data set
+ * @param endEpoch the end epoch of the data set
+ */
+export function getAttestationSuccessRatePerClients({
+  validatorData,
+  startEpoch,
+  endEpoch
+}: {
+  validatorData: ValidatorPerformance[];
+  startEpoch: number;
+  endEpoch: number;
+}): Map<ExecutionConsensusConcatenated, number> {
+  const attestationSuccessRatePerClients = new Map<ExecutionConsensusConcatenated, number>();
+
+  const dataByClient = new Map<ExecutionConsensusConcatenated, ValidatorPerformance[]>();
+  for (const data of validatorData) {
+    const key: ExecutionConsensusConcatenated = `${data.executionClient}-${data.consensusClient}`;
+    if (!dataByClient.has(key)) dataByClient.set(key, []);
+    dataByClient.get(key)?.push(data);
+  }
+
+  // calculate the attestation success rate for each client combination
+  for (const [key, data] of dataByClient.entries())
+    attestationSuccessRatePerClients.set(key, getAttestationSuccessRate({ validatorData: data, startEpoch, endEpoch }));
+
+  return attestationSuccessRatePerClients;
+}
diff --git a/packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts b/packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts
new file mode 100644
index 00000000..996cd86b
--- /dev/null
+++ b/packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts
@@ -0,0 +1,27 @@
+import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
+import type { ExecutionConsensusConcatenated } from "./types.js";
+
+export function getClientsUsedInIntervals({
+  validatorData,
+  startEpoch,
+  endEpoch
+}: {
+  validatorData: ValidatorPerformance[];
+  startEpoch: number;
+  endEpoch: number;
+}): Map<ExecutionConsensusConcatenated, number> {
+  const clientsUsedInInterval = new Map<ExecutionConsensusConcatenated, number>();
+
+  const dataByClient = new Map<ExecutionConsensusConcatenated, ValidatorPerformance[]>();
+  for (const data of validatorData) {
+    const key: ExecutionConsensusConcatenated = `${data.executionClient}-${data.consensusClient}`;
+    if (!dataByClient.has(key)) dataByClient.set(key, []);
+    dataByClient.get(key)?.push(data);
+  }
+
+  // calculate the number of epochs the client was used in the interval
+  for (const [key, data] of dataByClient.entries())
+    clientsUsedInInterval.set(key, data.filter((data) => data.epoch >= startEpoch && data.epoch < endEpoch).length);
+
+  return clientsUsedInInterval;
+}
diff --git a/packages/brain/src/modules/validatorsDataIngest/index.ts b/packages/brain/src/modules/validatorsDataIngest/index.ts
index fbb320e0..41a578d0 100644
--- a/packages/brain/src/modules/validatorsDataIngest/index.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/index.ts
@@ -1,9 +1,11 @@
 import { PostgresClient } from "../apiClients/index.js";
 import logger from "../logger/index.js";
 import { getStartAndEndEpochs } from "./getStartAndEndEpochs.js";
-import { calculateAttestationSuccessRate } from "./calculateAttestationSuccessRate.js";
+import { getAttestationSuccessRate } from "./getAttestationSuccessRate.js";
 import { Granularity, NumberOfDaysToQuery, ValidatorsDataProcessed } from "./types.js";
 import { getIntervalsEpochs } from "./getIntervalsEpochs.js";
+import { getAttestationSuccessRatePerClients } from "./getAttestationSuccessRatePerClients.js";
+import { getClientsUsedInIntervals } from "./getClientsUsedInInterval.js";
 
 // Module in charge of querying and processin the data of the validators to get the performance metrics:
 // - Attestation success rate
@@ -72,12 +74,14 @@ export async function fetchAndProcessValidatorsData({
   // Calculate the attestation success rate for each validator
   for (const [validatorIndex, validatorData] of validatorsDataMap.entries())
     mapValidatorPerformance.set(validatorIndex, {
-      attestationSuccessRate: calculateAttestationSuccessRate({ validatorData, startEpoch, endEpoch }),
+      attestationSuccessRate: getAttestationSuccessRate({ validatorData, startEpoch, endEpoch }),
+      attestationSuccessRatePerClients: getAttestationSuccessRatePerClients({ validatorData, startEpoch, endEpoch }),
       attestationSuccessRatePerInterval: intervals.map(({ startEpoch, endEpoch }) => {
         return {
           startEpoch,
           endEpoch,
-          attestationSuccessRate: calculateAttestationSuccessRate({ validatorData, startEpoch, endEpoch })
+          attestationSuccessRate: getAttestationSuccessRate({ validatorData, startEpoch, endEpoch }),
+          clientsUsedInInterval: getClientsUsedInIntervals({ validatorData, startEpoch, endEpoch })
         };
       }),
       blocks: {
diff --git a/packages/brain/src/modules/validatorsDataIngest/types.ts b/packages/brain/src/modules/validatorsDataIngest/types.ts
index 8cc25fa0..5a9cd314 100644
--- a/packages/brain/src/modules/validatorsDataIngest/types.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/types.ts
@@ -1,15 +1,26 @@
+import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
+
+export type ExecutionConsensusConcatenated = `${ExecutionClient}-${ConsensusClient}`;
+
 export interface ValidatorsDataProcessed {
   attestationSuccessRate: number; // mean attestationSuccessRate of the validator
+  attestationSuccessRatePerClients: Map<ExecutionConsensusConcatenated, number>;
   // attestationSuccessRate in each interval
   attestationSuccessRatePerInterval: {
     startEpoch: number; // start epoch of the interval
     endEpoch: number; // end epoch of the interval
     attestationSuccessRate: number | null; // attestationSuccessRate in the interval
+    clientsUsedInInterval: Map<ExecutionConsensusConcatenated, number>; // Map indexed by ["execution-consensus"] (i.e "geth-lighthouse") with the number of epochs the client was used in the interval
   }[];
   blocks: {
     proposed: number; // number of blocks proposed
     missed: number; // number of blocks missed
   };
+  // TODO: instead blocks should look as folloes
+  // blocks: {
+  //   proposed: {epoch: number, slot: number}[],
+  //   missed: {epoch: number, slot: number}[]
+  // }
 }
 
 export enum Granularity {
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
index 21b53bd3..7d16be54 100644
--- a/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
@@ -1,5 +1,6 @@
 import { PostgresClient } from "../../../../src/modules/apiClients/index.js";
 import { fetchAndProcessValidatorsData } from "../../../../src/modules/validatorsDataIngest/index.js";
+import { Granularity } from "../../../../src/modules/validatorsDataIngest/types.js";
 
 // This test must be executed with a real database connection
 
@@ -19,7 +20,8 @@ describe("Validators data ingest", function () {
       postgresClient,
       minGenesisTime,
       secondsPerSlot,
-      numberOfDaysToQuery: 1
+      numberOfDaysToQuery: 1,
+      granularity: Granularity.Hourly
     });
 
     console.log(data);

From 02f284549a8c0593e6d6e23c85db18f939e1ae03 Mon Sep 17 00:00:00 2001
From: Marc Font <36164126+Marketen@users.noreply.github.com>
Date: Wed, 25 Sep 2024 09:54:21 +0200
Subject: [PATCH 16/22] refactor cron (#347)

* refactor cron

* retry mechanism

* Execute cron once every epoch within the first 10% of time of the epoch  (#349)

* Propose changes

* add cron to stop handling

---------

Co-authored-by: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com>
---
 packages/brain/src/index.ts                   |  39 +++--
 .../cron/trackValidatorsPerformance/index.ts  | 138 +++++++-----------
 .../insertPerformanceData.ts                  |   8 +-
 3 files changed, 83 insertions(+), 102 deletions(-)

diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts
index 4a665ce8..44703bb8 100644
--- a/packages/brain/src/index.ts
+++ b/packages/brain/src/index.ts
@@ -97,29 +97,36 @@ const proofOfValidationCron = new CronJob(shareCronInterval, () =>
   sendProofsOfValidation(signerApi, brainDb, dappnodeSignatureVerifierApi, shareDataWithDappnode)
 );
 proofOfValidationCron.start();
-const trackValidatorsPerformanceCron = new CronJob(slotsPerEpoch * secondsPerSlot * 1000, () =>
-  // once every epoch
-  trackValidatorsPerformance({
-    brainDb,
-    postgresClient,
-    beaconchainApi,
-    minGenesisTime,
-    secondsPerSlot,
-    executionClient,
-    consensusClient
-  })
-);
+
+// executes once every epoch
+const trackValidatorsPerformanceCron = new CronJob(slotsPerEpoch * secondsPerSlot * 1000, async () => {
+  try {
+    const currentEpoch = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
+    await trackValidatorsPerformance({
+      currentEpoch,
+      brainDb,
+      postgresClient,
+      beaconchainApi,
+      executionClient,
+      consensusClient
+    });
+  } catch (e) {
+    logger.error(`Error tracking validator performance: ${e}`);
+  }
+});
+// if we are in the first 10% of the epoch we start the cron job if not we wait until the next epoch with a timeout.
+// gnosis chain 80 seconds per epoch -> 8 seconds
+// ethereum 384 seconds per epoch -> 38.4 seconds
 const secondsToNextEpoch = getSecondsToNextEpoch({ minGenesisTime, secondsPerSlot });
-// start the cron within the first minute of an epoch
-// If it remains more than 1 minute then wait for the next epoch (+ 10 seconds of margin)
-if (secondsToNextEpoch > 60) setTimeout(() => trackValidatorsPerformanceCron.start(), (secondsToNextEpoch + 10) * 1000);
-else trackValidatorsPerformanceCron.start();
+if (secondsToNextEpoch <= slotsPerEpoch * secondsPerSlot * 0.1) trackValidatorsPerformanceCron.start();
+else setTimeout(() => trackValidatorsPerformanceCron.start(), (secondsToNextEpoch + 3) * 1000);
 
 // Graceful shutdown
 function handle(signal: string): void {
   logger.info(`${signal} received. Shutting down...`);
   reloadValidatorsCron.stop();
   proofOfValidationCron.stop();
+  trackValidatorsPerformanceCron.stop();
   brainDb.close();
   postgresClient.close().catch((err) => logger.error(`Error closing postgres client`, err)); // postgresClient db connection is the only external resource that needs to be closed
   uiServer.close();
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index bd44ea54..f3018ad8 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -8,12 +8,7 @@ import { setBlockProposalStatusMap } from "./setBlockProposalStatusMap.js";
 import { checkNodeHealth } from "./checkNodeHealth.js";
 import { getActiveValidatorsLoadedInBrain } from "./getActiveValidatorsLoadedInBrain.js";
 import { logPrefix } from "./logPrefix.js";
-import { TotalRewards } from "../../apiClients/types.js";
-import { BlockProposalStatus } from "../../apiClients/postgres/types.js";
 import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
-import { getSecondsToNextEpoch } from "../../../getSecondsToNextEpoch.js";
-
-const MINUTE_IN_SECONDS = 60;
 
 // TODO: at this moment Lighthouse client does not support retrieving:
 // - liveness of validator from finalized epoch:
@@ -23,110 +18,89 @@ const MINUTE_IN_SECONDS = 60;
 
 /**
  * Cron task that will track validators performance for the epoch finalized and store it in the Postgres DB.
- * If any issue is arisen during the process, it will be retried after 1 minute. If the issue persists until the epoch
+ * If any issue is arisen during the process, it will be retried after 30 seconds. If the issue persists until the epoch
  * finalized changes, the issue will be logged and stored in the DB.
  *
  * @param validatorPubkeys - The pubkeys of the validators to track.
  * @param postgresClient - Postgres client to interact with the DB.
  * @param beaconchainApi - Beaconchain API client to interact with the Beaconchain API.
- * @param minGenesisTime - The minimum genesis time of the chain.
+ * @param executionClient - The execution client to interact with.
+ * @param consensusClient - The consensus client to interact with.
+ *
+ * @throws {Error} If there is an error when updating the latestEpoch in the error handling
  */
 export async function trackValidatorsPerformance({
+  currentEpoch,
   brainDb,
   postgresClient,
   beaconchainApi,
-  minGenesisTime,
-  secondsPerSlot,
   executionClient,
   consensusClient
 }: {
+  currentEpoch: number;
   brainDb: BrainDataBase;
   postgresClient: PostgresClient;
   beaconchainApi: BeaconchainApi;
-  minGenesisTime: number;
-  secondsPerSlot: number;
   executionClient: ExecutionClient;
   consensusClient: ConsensusClient;
 }): Promise<void> {
-  try {
-    const epochFinalized = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
-    let errorGettingValidatorData: Error | undefined;
-    let newEpochFinalized = epochFinalized;
-    let activeValidatorsIndexes: string[] = [];
-    let validatorsAttestationsTotalRewards: TotalRewards[] = [];
-    let validatorBlockStatusMap: Map<string, BlockProposalStatus> = new Map();
+  let latestEpoch = currentEpoch;
 
-    while (epochFinalized === newEpochFinalized) {
-      try {
-        logger.debug(`${logPrefix}Epoch finalized: ${epochFinalized}`);
+  while (currentEpoch === latestEpoch) {
+    try {
+      logger.debug(`${logPrefix}Starting to track performance for epoch: ${currentEpoch}`);
 
-        // active validators indexes
-        activeValidatorsIndexes = await getActiveValidatorsLoadedInBrain({ beaconchainApi, brainDb });
-        if (activeValidatorsIndexes.length === 0) {
-          logger.info(`${logPrefix}No active validators found`);
-          return;
-        }
-        logger.debug(`${logPrefix}Active validators: ${activeValidatorsIndexes}`);
+      const activeValidatorsIndexes = await getActiveValidatorsLoadedInBrain({ beaconchainApi, brainDb });
+      if (activeValidatorsIndexes.length === 0) {
+        logger.info(`${logPrefix}No active validators found`);
+        return; // Exit if no active validators are found
+      }
 
-        // check node health
-        await checkNodeHealth({ beaconchainApi });
+      await checkNodeHealth({ beaconchainApi });
 
-        // get block attestations rewards
-        validatorsAttestationsTotalRewards = await getAttestationsTotalRewards({
-          beaconchainApi,
-          epoch: epochFinalized.toString(),
-          activeValidatorsIndexes
-        });
+      const validatorsAttestationsTotalRewards = await getAttestationsTotalRewards({
+        beaconchainApi,
+        epoch: currentEpoch.toString(),
+        activeValidatorsIndexes
+      });
 
-        // get block proposal status
-        validatorBlockStatusMap = await setBlockProposalStatusMap({
-          beaconchainApi,
-          epoch: epochFinalized.toString(),
-          activeValidatorsIndexes
-        });
+      const validatorBlockStatusMap = await setBlockProposalStatusMap({
+        beaconchainApi,
+        epoch: currentEpoch.toString(),
+        activeValidatorsIndexes
+      });
 
-        // update error to undefined if no error occurred in last iteration and break the loop
-        errorGettingValidatorData = undefined;
-        break;
-      } catch (error) {
-        logger.error(`${logPrefix}Error occurred: ${error}. Updating epoch finalized and retrying in 1 minute`);
-        // update error if an error occurred
-        errorGettingValidatorData = error;
+      await insertPerformanceDataNotThrow({
+        postgresClient,
+        activeValidatorsIndexes,
+        currentEpoch,
+        validatorBlockStatusMap,
+        validatorsAttestationsTotalRewards,
+        error: undefined,
+        executionClient,
+        consensusClient
+      });
 
-        // skip if the seconds to the next epoch is less than 1 minute
-        const secondsToNextEpoch = getSecondsToNextEpoch({ minGenesisTime, secondsPerSlot });
-        if (secondsToNextEpoch < MINUTE_IN_SECONDS) {
-          logger.warn(
-            `${logPrefix}Could not get validator data for epoch ${epochFinalized}. Writing error and skipping to next epoch.`
-          );
-          // TODO: collect report of the staker setup status: el is offline, node is syncing, signer is not up and original error
-          // exit the while loop and write the error to the DB
-          break;
-        }
-        // wait 1 minute without blocking the event loop and update epoch finalized
-        newEpochFinalized = await new Promise((resolve) =>
-          setTimeout(
-            async () => resolve(await beaconchainApi.getEpochHeader({ blockId: "finalized" })),
-            MINUTE_IN_SECONDS * 1000
-          )
-        );
+      logger.debug(`${logPrefix}Finished tracking performance for epoch: ${currentEpoch}`);
+      return; // Success, exit function
+    } catch (e) {
+      logger.error(`${logPrefix}Error tracking validator peformance for epoch ${currentEpoch}: ${e}`);
+      latestEpoch = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
+      if (latestEpoch !== currentEpoch) {
+        logger.info(`${logPrefix}Epoch has changed from ${currentEpoch} to ${latestEpoch}, aborting retry.`);
+        await insertPerformanceDataNotThrow({
+          postgresClient,
+          activeValidatorsIndexes: [],
+          currentEpoch,
+          validatorBlockStatusMap: new Map(),
+          validatorsAttestationsTotalRewards: [],
+          error: e.message, // Store the error in the DB after all retries are exhausted
+          executionClient,
+          consensusClient
+        });
+        return; // Exit after final attempt
       }
+      await new Promise((resolve) => setTimeout(resolve, 30 * 1000)); // Wait 30 seconds before retrying
     }
-    logger.debug(`${logPrefix}Epoch finalized changed: ${newEpochFinalized}`);
-
-    // insert performance data or each validator
-    await insertPerformanceDataNotThrow({
-      postgresClient,
-      activeValidatorsIndexes,
-      epochFinalized,
-      validatorBlockStatusMap,
-      validatorsAttestationsTotalRewards,
-      error: errorGettingValidatorData,
-      executionClient,
-      consensusClient
-    });
-  } catch (e) {
-    logger.error(`${logPrefix}Error in trackValidatorsPerformance: ${e}`);
-    return;
   }
 }
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
index c591650a..0e31d9f7 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/insertPerformanceData.ts
@@ -19,7 +19,7 @@ import { logPrefix } from "./logPrefix.js";
 export async function insertPerformanceDataNotThrow({
   postgresClient,
   activeValidatorsIndexes,
-  epochFinalized,
+  currentEpoch,
   validatorBlockStatusMap,
   validatorsAttestationsTotalRewards,
   executionClient,
@@ -28,7 +28,7 @@ export async function insertPerformanceDataNotThrow({
 }: {
   postgresClient: PostgresClient;
   activeValidatorsIndexes: string[];
-  epochFinalized: number;
+  currentEpoch: number;
   validatorBlockStatusMap: Map<string, BlockProposalStatus>;
   validatorsAttestationsTotalRewards: TotalRewards[];
   executionClient: ExecutionClient;
@@ -58,14 +58,14 @@ export async function insertPerformanceDataNotThrow({
       logger.debug(`${logPrefix}Inserting performance data for validator ${validatorIndex}`);
       await postgresClient.insertPerformanceData({
         validatorIndex: parseInt(validatorIndex),
-        epoch: epochFinalized,
+        epoch: currentEpoch,
         blockProposalStatus,
         attestationsTotalRewards,
         error: error?.message,
         executionClient,
         consensusClient
       });
-      logger.debug(`${logPrefix}Performance data inserted for epoch ${epochFinalized}`);
+      logger.debug(`${logPrefix}Performance data inserted for epoch ${currentEpoch}`);
     } catch (e) {
       logger.error(`${logPrefix}Error inserting performance data for validator ${validatorIndex}: ${e}`);
       continue;

From 97b08c2b4efd164512e866430fa051176c7f612f Mon Sep 17 00:00:00 2001
From: pablomendezroyo <mendez4a@gmail.com>
Date: Wed, 25 Sep 2024 09:57:41 +0200
Subject: [PATCH 17/22] skip unit test

---
 .../test/unit/modules/validatorsDataIngest/index.unit.test.ts   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
index 7d16be54..ce211523 100644
--- a/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/index.unit.test.ts
@@ -4,7 +4,7 @@ import { Granularity } from "../../../../src/modules/validatorsDataIngest/types.
 
 // This test must be executed with a real database connection
 
-describe("Validators data ingest", function () {
+describe.skip("Validators data ingest", function () {
   this.timeout(10 * 1000);
   // change the dbUrl on demmand
   const dbUrl = "postgres://postgres:password@postgres.web3signer-holesky.dappnode:5432/web3signer";

From b3d4b94018b00703707b22475ef7ed19a4ca12fb Mon Sep 17 00:00:00 2001
From: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com>
Date: Wed, 25 Sep 2024 10:27:22 +0200
Subject: [PATCH 18/22] use protocol https in teku (#350)

---
 packages/brain/src/modules/config/networks/gnosis.ts  | 2 +-
 packages/brain/src/modules/config/networks/holesky.ts | 2 +-
 packages/brain/src/modules/config/networks/lukso.ts   | 2 +-
 packages/brain/src/modules/config/networks/mainnet.ts | 2 +-
 packages/brain/src/modules/config/networks/prater.ts  | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/brain/src/modules/config/networks/gnosis.ts b/packages/brain/src/modules/config/networks/gnosis.ts
index bc69db4e..97e0fb1f 100644
--- a/packages/brain/src/modules/config/networks/gnosis.ts
+++ b/packages/brain/src/modules/config/networks/gnosis.ts
@@ -15,7 +15,7 @@ export const gnosisBrainConfig = (
     consensusClient,
     isMevBoostSet,
     executionClientUrl: "http://execution.gnosis.dncore.dappnode:8545",
-    validatorUrl: "http://validator.gnosis.dncore.dappnode:3500",
+    validatorUrl: `${consensusClient === "teku" ? "https" : "http"}://validator.gnosis.dncore.dappnode:3500`,
     beaconchainUrl: "http:/beacon-chain.gnosis.dncore.dappnode:3500",
     blockExplorerUrl: "https://gnosischa.in",
     signerUrl: "http://web3signer.web3signer-gnosis.dappnode:9000",
diff --git a/packages/brain/src/modules/config/networks/holesky.ts b/packages/brain/src/modules/config/networks/holesky.ts
index 15eb5a38..744b1032 100644
--- a/packages/brain/src/modules/config/networks/holesky.ts
+++ b/packages/brain/src/modules/config/networks/holesky.ts
@@ -15,7 +15,7 @@ export const holeskyBrainConfig = (
     consensusClient,
     isMevBoostSet,
     executionClientUrl: "http://execution.holesky.dncore.dappnode:8545",
-    validatorUrl: "http://validator.holesky.dncore.dappnode:3500",
+    validatorUrl: `${consensusClient === "teku" ? "https" : "http"}://validator.holesky.dncore.dappnode:3500`,
     beaconchainUrl: "http:/beacon-chain.holesky.dncore.dappnode:3500",
     blockExplorerUrl: "https://holesky.beaconcha.in",
     signerUrl: "http://web3signer.web3signer-holesky.dappnode:9000",
diff --git a/packages/brain/src/modules/config/networks/lukso.ts b/packages/brain/src/modules/config/networks/lukso.ts
index 388e95d1..bb8e0bb3 100644
--- a/packages/brain/src/modules/config/networks/lukso.ts
+++ b/packages/brain/src/modules/config/networks/lukso.ts
@@ -15,7 +15,7 @@ export const luksoBrainConfig = (
     consensusClient,
     isMevBoostSet,
     executionClientUrl: "http://execution.lukso.dncore.dappnode:8545",
-    validatorUrl: "http://validator.lukso.dncore.dappnode:3500",
+    validatorUrl: `${consensusClient === "teku" ? "https" : "http"}://validator.lukso.dncore.dappnode:3500`,
     beaconchainUrl: "http:/beacon-chain.lukso.dncore.dappnode:3500",
     blockExplorerUrl: "https://explorer.consensus.mainnet.lukso.network/",
     signerUrl: "http://web3signer.web3signer-lukso.dappnode:9000",
diff --git a/packages/brain/src/modules/config/networks/mainnet.ts b/packages/brain/src/modules/config/networks/mainnet.ts
index d8e4776a..39954586 100644
--- a/packages/brain/src/modules/config/networks/mainnet.ts
+++ b/packages/brain/src/modules/config/networks/mainnet.ts
@@ -15,7 +15,7 @@ export const mainnetBrainConfig = (
     consensusClient,
     isMevBoostSet,
     executionClientUrl: "http://execution.mainnet.dncore.dappnode:8545",
-    validatorUrl: "http://validator.mainnet.dncore.dappnode:3500",
+    validatorUrl: `${consensusClient === "teku" ? "https" : "http"}://validator.mainnet.dncore.dappnode:3500`,
     beaconchainUrl: "http:/beacon-chain.mainnet.dncore.dappnode:3500",
     blockExplorerUrl: "https://beaconcha.in",
     signerUrl: "http://web3signer.web3signer.dappnode:9000",
diff --git a/packages/brain/src/modules/config/networks/prater.ts b/packages/brain/src/modules/config/networks/prater.ts
index 213bd088..54ecd81b 100644
--- a/packages/brain/src/modules/config/networks/prater.ts
+++ b/packages/brain/src/modules/config/networks/prater.ts
@@ -15,7 +15,7 @@ export const praterBrainConfig = (
     consensusClient,
     isMevBoostSet,
     executionClientUrl: "http://execution.prater.dncore.dappnode:8545",
-    validatorUrl: "http://validator.prater.dncore.dappnode:3500",
+    validatorUrl: `${consensusClient === "teku" ? "https" : "http"}://validator.prater.dncore.dappnode:3500`,
     beaconchainUrl: "http:/beacon-chain.prater.dncore.dappnode:3500",
     blockExplorerUrl: "https://prater.beaconcha.in",
     signerUrl: "http://web3signer.web3signer-prater.dappnode:9000",

From 21a420033d50ed3ce36157b48fc5654fa3ae302e Mon Sep 17 00:00:00 2001
From: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com>
Date: Wed, 25 Sep 2024 15:35:35 +0200
Subject: [PATCH 19/22] Skip epoch if node is syncing and retry if el is
 offline (#351)

* reorg check node health

* relocate
---
 .../checkNodeHealth.ts                           | 16 ----------------
 .../cron/trackValidatorsPerformance/index.ts     |  8 ++++++--
 2 files changed, 6 insertions(+), 18 deletions(-)
 delete mode 100644 packages/brain/src/modules/cron/trackValidatorsPerformance/checkNodeHealth.ts

diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/checkNodeHealth.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/checkNodeHealth.ts
deleted file mode 100644
index 2d576227..00000000
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/checkNodeHealth.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import { BeaconchainApi } from "../../apiClients/index.js";
-import logger from "../../logger/index.js";
-import { logPrefix } from "./logPrefix.js";
-
-/**
- * Check the health of the node, if the EL node is offline or the node is syncing, an error will be thrown.
- *
- * @param {BeaconchainApi} beaconchainApi - Beaconchain API client.
- * @throws {Error} - If the EL node is offline or the node is syncing.
- */
-export async function checkNodeHealth({ beaconchainApi }: { beaconchainApi: BeaconchainApi }): Promise<void> {
-  const { el_offline, is_syncing } = (await beaconchainApi.getSyncingStatus()).data;
-  logger.debug(`${logPrefix}EL Node offline: ${el_offline}, Node syncing: ${is_syncing}`);
-  if (el_offline) throw Error("EL Node is offline");
-  if (is_syncing) throw Error("Node is syncing");
-}
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index f3018ad8..097acd28 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -5,7 +5,6 @@ import { BrainDataBase } from "../../db/index.js";
 import { insertPerformanceDataNotThrow } from "./insertPerformanceData.js";
 import { getAttestationsTotalRewards } from "./getAttestationsTotalRewards.js";
 import { setBlockProposalStatusMap } from "./setBlockProposalStatusMap.js";
-import { checkNodeHealth } from "./checkNodeHealth.js";
 import { getActiveValidatorsLoadedInBrain } from "./getActiveValidatorsLoadedInBrain.js";
 import { logPrefix } from "./logPrefix.js";
 import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
@@ -56,7 +55,12 @@ export async function trackValidatorsPerformance({
         return; // Exit if no active validators are found
       }
 
-      await checkNodeHealth({ beaconchainApi });
+      const { el_offline, is_syncing } = (await beaconchainApi.getSyncingStatus()).data;
+      if (is_syncing) {
+        logger.debug(`${logPrefix}Node is syncing, skipping epoch ${currentEpoch}`);
+        return; // Exit if the node is syncing. Head finalized will change
+      }
+      if (el_offline) throw new Error("EL Node offline"); // throw error and retry
 
       const validatorsAttestationsTotalRewards = await getAttestationsTotalRewards({
         beaconchainApi,

From 970c2e7f464ec2263977f0f402211894d374f168 Mon Sep 17 00:00:00 2001
From: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com>
Date: Wed, 25 Sep 2024 17:44:07 +0200
Subject: [PATCH 20/22] Add unit testing to trackValidatorsPerformance cron
 module (#352)

* Add unit testing to getActiveValidators

* Implement missing unit testing

* remove only
---
 .../modules/apiClients/beaconchain/index.ts   |   6 +-
 .../modules/apiClients/beaconchain/types.ts   |   2 +
 ...tusMap.ts => getBlockProposalStatusMap.ts} |  11 +-
 .../cron/trackValidatorsPerformance/index.ts  |   4 +-
 ...ActiveValidatorsLoadedInBrain.unit.test.ts | 145 ++++++++++++++++++
 .../getBlockProposalStatusMap.unit.test.ts    | 109 +++++++++++++
 6 files changed, 265 insertions(+), 12 deletions(-)
 rename packages/brain/src/modules/cron/trackValidatorsPerformance/{setBlockProposalStatusMap.ts => getBlockProposalStatusMap.ts} (87%)
 create mode 100644 packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getBlockProposalStatusMap.unit.test.ts

diff --git a/packages/brain/src/modules/apiClients/beaconchain/index.ts b/packages/brain/src/modules/apiClients/beaconchain/index.ts
index 7d0c0a90..a5409d6e 100644
--- a/packages/brain/src/modules/apiClients/beaconchain/index.ts
+++ b/packages/brain/src/modules/apiClients/beaconchain/index.ts
@@ -13,7 +13,8 @@ import {
   BeaconchainSyncingStatusGetResponse,
   BeaconchainSyncCommitteePostResponse,
   BeaconchainBlockRewardsGetResponse,
-  BeaconchainProposerDutiesGetResponse
+  BeaconchainProposerDutiesGetResponse,
+  BlockId
 } from "./types.js";
 import { StandardApi } from "../standard.js";
 import path from "path";
@@ -21,9 +22,6 @@ import { ApiParams } from "../types.js";
 import { Network } from "@stakingbrain/common";
 import { BeaconchainApiError } from "./error.js";
 
-// TODO: BlockId can also be a simple slot in the form of a string. Is this type still necessary?
-type BlockId = "head" | "genesis" | "finalized" | string | `0x${string}`;
-
 export class BeaconchainApi extends StandardApi {
   private SLOTS_PER_EPOCH: number;
 
diff --git a/packages/brain/src/modules/apiClients/beaconchain/types.ts b/packages/brain/src/modules/apiClients/beaconchain/types.ts
index 40632582..074e49f1 100644
--- a/packages/brain/src/modules/apiClients/beaconchain/types.ts
+++ b/packages/brain/src/modules/apiClients/beaconchain/types.ts
@@ -207,3 +207,5 @@ export interface BeaconchainLivenessPostResponse {
     is_live: boolean;
   }[];
 }
+
+export type BlockId = "head" | "genesis" | "finalized" | string | `0x${string}`;
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/getBlockProposalStatusMap.ts
similarity index 87%
rename from packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts
rename to packages/brain/src/modules/cron/trackValidatorsPerformance/getBlockProposalStatusMap.ts
index c5c40f16..7ee85900 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/setBlockProposalStatusMap.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/getBlockProposalStatusMap.ts
@@ -10,7 +10,7 @@ import { logPrefix } from "./logPrefix.js";
  * @param {string} epoch - The epoch to get the block proposal duties.
  * @param {string[]} activeValidatorIndexes - Array of validator indexes.
  */
-export async function setBlockProposalStatusMap({
+export async function getBlockProposalStatusMap({
   beaconchainApi,
   epoch,
   activeValidatorsIndexes
@@ -43,14 +43,13 @@ export async function setBlockProposalStatusMap({
     // enter loop if one of our monitored validators had to propose in this slot
     if (validatorIndexesSet.has(validator_index)) {
       try {
-        // Get the block header for the slot. It has the proposer index.
-        const blockHeader = await beaconchainApi.getBlockHeader({ blockId: slot });
+        // Get the proposer index from the block header for the slot
+        const proposerIndex = (await beaconchainApi.getBlockHeader({ blockId: slot })).data.header.message
+          .proposer_index;
         // If the proposer index in the block header matches the validator index, the block was proposed correctly.
         validatorBlockStatusMap.set(
           validator_index,
-          blockHeader.data.header.message.proposer_index === validator_index
-            ? BlockProposalStatus.Proposed
-            : BlockProposalStatus.Error
+          proposerIndex === validator_index ? BlockProposalStatus.Proposed : BlockProposalStatus.Error
         );
       } catch (error) {
         if (error.status === 404) {
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index 097acd28..be318860 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -4,7 +4,7 @@ import logger from "../../logger/index.js";
 import { BrainDataBase } from "../../db/index.js";
 import { insertPerformanceDataNotThrow } from "./insertPerformanceData.js";
 import { getAttestationsTotalRewards } from "./getAttestationsTotalRewards.js";
-import { setBlockProposalStatusMap } from "./setBlockProposalStatusMap.js";
+import { getBlockProposalStatusMap } from "./getBlockProposalStatusMap.js";
 import { getActiveValidatorsLoadedInBrain } from "./getActiveValidatorsLoadedInBrain.js";
 import { logPrefix } from "./logPrefix.js";
 import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
@@ -68,7 +68,7 @@ export async function trackValidatorsPerformance({
         activeValidatorsIndexes
       });
 
-      const validatorBlockStatusMap = await setBlockProposalStatusMap({
+      const validatorBlockStatusMap = await getBlockProposalStatusMap({
         beaconchainApi,
         epoch: currentEpoch.toString(),
         activeValidatorsIndexes
diff --git a/packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.unit.test.ts b/packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.unit.test.ts
new file mode 100644
index 00000000..62ae74af
--- /dev/null
+++ b/packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.unit.test.ts	
@@ -0,0 +1,145 @@
+import { expect } from "chai";
+import { BeaconchainApi } from "../../../../../src/modules/apiClients/index.js";
+import {
+  BeaconchainValidatorFromStateGetResponse,
+  BeaconchainValidatorStatePostResponse,
+  BlockId,
+  ValidatorStatus
+} from "../../../../../src/modules/apiClients/types.js";
+import { BrainDataBase } from "../../../../../src/modules/db/index.js";
+import { StakingBrainDb } from "../../../../../src/modules/db/types.js";
+import { getActiveValidatorsLoadedInBrain } from "../../../../../src/modules/cron/trackValidatorsPerformance/getActiveValidatorsLoadedInBrain.js";
+import { Network } from "@stakingbrain/common";
+
+const validatorIndexOne = 1802289;
+const pubkeyOne = "0x86531f35f71730767e72692442a2020a6f252c15bc73d11e201d658ed90dde0dd15d9614e6c115b2dd0221ce35dcdcb3";
+const validatorIndexTwo = 1802291;
+const pubkeyTwo = "0x86531f35f71730767e72692442a2020a6f252c15bc73d11e201d658ed90dde0dd15d9614e6c115b2dd0221ce35dcdcb4";
+
+// Create class mock that implements BeaconchainApi and overwrites the method postStateValidators
+class BeaconchainApiMock extends BeaconchainApi {
+  async getStateValidator({
+    state,
+    pubkey
+  }: {
+    state: BlockId;
+    pubkey: string;
+  }): Promise<BeaconchainValidatorFromStateGetResponse> {
+    console.log(`state: ${state}, pubkey: ${pubkey}`);
+    if (pubkey === pubkeyOne)
+      return {
+        execution_optimistic: false,
+        data: {
+          index: validatorIndexOne.toString(),
+          balance: "0",
+          status: ValidatorStatus.ACTIVE_EXITING,
+          validator: {
+            pubkey: pubkeyOne,
+            withdrawal_credentials: "",
+            effective_balance: "",
+            slashed: false,
+            activation_eligibility_epoch: "",
+            activation_epoch: "",
+            exit_epoch: "",
+            withdrawable_epoch: ""
+          }
+        }
+      };
+    if (pubkey === pubkeyTwo)
+      return {
+        execution_optimistic: false,
+        data: {
+          index: validatorIndexTwo.toString(),
+          balance: "0",
+          status: ValidatorStatus.ACTIVE_EXITING,
+          validator: {
+            pubkey: pubkeyTwo,
+            withdrawal_credentials: "",
+            effective_balance: "",
+            slashed: false,
+            activation_eligibility_epoch: "",
+            activation_epoch: "",
+            exit_epoch: "",
+            withdrawable_epoch: ""
+          }
+        }
+      };
+    throw new Error("pubkey not found");
+  }
+
+  async postStateValidators({
+    stateId,
+    body
+  }: {
+    stateId: BlockId;
+    body: { ids: string[]; statuses: ValidatorStatus[] };
+  }): Promise<BeaconchainValidatorStatePostResponse> {
+    console.log(`stateId: ${stateId}, body: ${JSON.stringify(body)}`);
+    return {
+      execution_optimistic: false,
+      finalized: true,
+      data: body.ids.map((id) => ({
+        index: id,
+        balance: "0",
+        status: ValidatorStatus.ACTIVE_ONGOING,
+        validator: {
+          pubkey: "",
+          withdrawal_credentials: "",
+          effective_balance: "",
+          slashed: false,
+          activation_eligibility_epoch: "",
+          activation_epoch: "",
+          exit_epoch: "",
+          withdrawable_epoch: ""
+        }
+      }))
+    };
+  }
+}
+
+// Mock the BrainDataBase class
+class BrainDataBaseMock extends BrainDataBase {
+  data: StakingBrainDb = {
+    [pubkeyOne]: {
+      tag: "obol",
+      feeRecipient: "0x52908400098527886E0F7030069857D2E4169EE7",
+      automaticImport: true,
+      index: validatorIndexOne // validator index exists
+    },
+    [pubkeyTwo]: {
+      tag: "solo",
+      feeRecipient: "0x52908400098527886E0F7030069857D2E4169EE6",
+      automaticImport: true
+      // validator index does not exist
+    }
+  };
+
+  getData() {
+    return this.data;
+  }
+
+  updateValidators({ validators }: { validators: StakingBrainDb }) {
+    this.data = { ...this.data, ...validators };
+  }
+}
+
+describe("Cron - trackValidatorsPerformance - getActiveValidatorsLoadedInBrain", () => {
+  it("should return the active validators loaded in the brain and not update validator index one and update validator index two in db", async () => {
+    const beaconchainApi = new BeaconchainApiMock(
+      { baseUrl: "http://localhost:3000", apiPath: "", authToken: "" },
+      Network.Holesky
+    );
+    const brainDb = new BrainDataBaseMock("test.json");
+
+    const activeValidatorsIndexes = await getActiveValidatorsLoadedInBrain({
+      beaconchainApi,
+      brainDb
+    });
+
+    expect(activeValidatorsIndexes).to.be.an("array").that.includes(validatorIndexOne.toString());
+    expect(activeValidatorsIndexes).to.be.an("array").that.includes(validatorIndexTwo.toString());
+    expect(activeValidatorsIndexes.length).to.be.equal(2);
+    expect(brainDb.getData()[pubkeyOne].index).to.be.equal(validatorIndexOne);
+    expect(brainDb.getData()[pubkeyTwo].index).to.be.equal(validatorIndexTwo);
+  });
+});
diff --git a/packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getBlockProposalStatusMap.unit.test.ts b/packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getBlockProposalStatusMap.unit.test.ts
new file mode 100644
index 00000000..f0632e95
--- /dev/null
+++ b/packages/brain/test/unit/modules/ cron/trackValidatorsPerformance/getBlockProposalStatusMap.unit.test.ts	
@@ -0,0 +1,109 @@
+import { expect } from "chai";
+import { BeaconchainApi } from "../../../../../src/modules/apiClients/index.js";
+import type {
+  BeaconchainBlockHeaderGetResponse,
+  BeaconchainProposerDutiesGetResponse,
+  BlockId
+} from "../../../../../src/modules/apiClients/types.js";
+import { BlockProposalStatus } from "../../../../../src/modules/apiClients/postgres/types.js";
+import { Network } from "@stakingbrain/common";
+import { getBlockProposalStatusMap } from "../../../../../src/modules/cron/trackValidatorsPerformance/getBlockProposalStatusMap.js";
+
+// validator index 1802289 is supposed to propose in slot 1
+// validator index 1802291 is supposed to propose in slot 2
+// validator index 1802292 is not supposed to propose in any slot
+const validatorsBlockProposal: {
+  index: string;
+  slot: string;
+}[] = [
+  {
+    index: "1802289",
+    slot: "1"
+  },
+  {
+    index: "1802291",
+    slot: "2"
+  },
+  {
+    index: "1802292",
+    slot: ""
+  }
+];
+
+const validatorMissedBlockProposal = {
+  index: "1802283",
+  slot: "3"
+};
+
+class BeaconchainApiMock extends BeaconchainApi {
+  async getProposerDuties({ epoch }: { epoch: string }): Promise<BeaconchainProposerDutiesGetResponse> {
+    console.log(`epoch: ${epoch}`);
+    return {
+      dependent_root: "",
+      execution_optimistic: false,
+      data: [
+        {
+          pubkey: "",
+          validator_index: validatorsBlockProposal[0].index,
+          slot: validatorsBlockProposal[0].slot
+        },
+        {
+          pubkey: "",
+          validator_index: validatorsBlockProposal[1].index,
+          slot: validatorsBlockProposal[1].slot
+        },
+        {
+          pubkey: "",
+          validator_index: validatorMissedBlockProposal.index,
+          slot: validatorMissedBlockProposal.slot
+        }
+      ]
+    };
+  }
+
+  async getBlockHeader({ blockId }: { blockId: BlockId }): Promise<BeaconchainBlockHeaderGetResponse> {
+    console.log(`blockId: ${blockId}`);
+
+    // find in the validatorsBlockProposal array the slot that matches the blockId if not found reject with an error with code 404
+    const foundValidator = validatorsBlockProposal.find((validator) => validator.slot === blockId);
+    if (!foundValidator) return Promise.reject({ status: 404 });
+    return {
+      execution_optimistic: true,
+      data: {
+        root: "",
+        canonical: true,
+        header: {
+          message: {
+            slot: foundValidator.slot,
+            proposer_index: foundValidator.index,
+            parent_root: "",
+            state_root: "",
+            body_root: ""
+          },
+          signature: ""
+        }
+      }
+    };
+  }
+}
+
+describe("Cron - trackValidatorsPerformance - getBlockProposalStatusMap", () => {
+  const beaconchainApi = new BeaconchainApiMock({ baseUrl: "http://localhost:3000" }, Network.Mainnet);
+
+  it("should return the block proposal status of each validator: ", async () => {
+    const epoch = "1";
+    const blockProposalStatusMap = await getBlockProposalStatusMap({
+      beaconchainApi,
+      epoch,
+      activeValidatorsIndexes: [
+        ...validatorsBlockProposal.map((validator) => validator.index),
+        validatorMissedBlockProposal.index
+      ]
+    });
+
+    expect(blockProposalStatusMap.get(validatorsBlockProposal[0].index)).to.equal(BlockProposalStatus.Proposed);
+    expect(blockProposalStatusMap.get(validatorsBlockProposal[1].index)).to.equal(BlockProposalStatus.Proposed);
+    expect(blockProposalStatusMap.get(validatorsBlockProposal[2].index)).to.equal(BlockProposalStatus.Unchosen);
+    expect(blockProposalStatusMap.get(validatorMissedBlockProposal.index)).to.equal(BlockProposalStatus.Missed);
+  });
+});

From da5a30b31d3f00374e2b29ce7cef4c509a331c9d Mon Sep 17 00:00:00 2001
From: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com>
Date: Wed, 25 Sep 2024 22:04:57 +0200
Subject: [PATCH 21/22] Add missing test to validators data ingest. WIP (#353)

* Add missing test to validators data ingest

* add unit testing to getClientsUsedPerIntervalsMap

* add unit testing for getAttestationSuccessRatePerClients
---
 ...al.ts => getClientsUsedPerIntervalsMap.ts} |   2 +-
 .../src/modules/validatorsDataIngest/index.ts |   4 +-
 ...lculateAttestationSuccessRate.unit.test.ts |   0
 .../getAttestationSuccessRate.unit.test.ts    | 112 +++++++++
 ...estationSuccessRatePerClients.unit.test.ts | 177 ++++++++++++++
 ...getClientsUsedPerIntervalsMap.unit.test.ts | 219 ++++++++++++++++++
 .../getIntervalsEpochs.unit.test.ts           | 108 +++++++++
 .../getStartAndEndEpochs.unit.test.ts         |  28 ++-
 8 files changed, 645 insertions(+), 5 deletions(-)
 rename packages/brain/src/modules/validatorsDataIngest/{getClientsUsedInInterval.ts => getClientsUsedPerIntervalsMap.ts} (95%)
 delete mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/calculateAttestationSuccessRate.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRate.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.unit.test.ts
 create mode 100644 packages/brain/test/unit/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.unit.test.ts

diff --git a/packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts b/packages/brain/src/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.ts
similarity index 95%
rename from packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts
rename to packages/brain/src/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.ts
index 996cd86b..6ca32e23 100644
--- a/packages/brain/src/modules/validatorsDataIngest/getClientsUsedInInterval.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.ts
@@ -1,7 +1,7 @@
 import type { ValidatorPerformance } from "../apiClients/postgres/types.js";
 import type { ExecutionConsensusConcatenated } from "./types.js";
 
-export function getClientsUsedInIntervals({
+export function getClientsUsedPerIntervalsMap({
   validatorData,
   startEpoch,
   endEpoch
diff --git a/packages/brain/src/modules/validatorsDataIngest/index.ts b/packages/brain/src/modules/validatorsDataIngest/index.ts
index 41a578d0..488a504f 100644
--- a/packages/brain/src/modules/validatorsDataIngest/index.ts
+++ b/packages/brain/src/modules/validatorsDataIngest/index.ts
@@ -5,7 +5,7 @@ import { getAttestationSuccessRate } from "./getAttestationSuccessRate.js";
 import { Granularity, NumberOfDaysToQuery, ValidatorsDataProcessed } from "./types.js";
 import { getIntervalsEpochs } from "./getIntervalsEpochs.js";
 import { getAttestationSuccessRatePerClients } from "./getAttestationSuccessRatePerClients.js";
-import { getClientsUsedInIntervals } from "./getClientsUsedInInterval.js";
+import { getClientsUsedPerIntervalsMap } from "./getClientsUsedPerIntervalsMap.js";
 
 // Module in charge of querying and processin the data of the validators to get the performance metrics:
 // - Attestation success rate
@@ -81,7 +81,7 @@ export async function fetchAndProcessValidatorsData({
           startEpoch,
           endEpoch,
           attestationSuccessRate: getAttestationSuccessRate({ validatorData, startEpoch, endEpoch }),
-          clientsUsedInInterval: getClientsUsedInIntervals({ validatorData, startEpoch, endEpoch })
+          clientsUsedInInterval: getClientsUsedPerIntervalsMap({ validatorData, startEpoch, endEpoch })
         };
       }),
       blocks: {
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/calculateAttestationSuccessRate.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/calculateAttestationSuccessRate.unit.test.ts
deleted file mode 100644
index e69de29b..00000000
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRate.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRate.unit.test.ts
new file mode 100644
index 00000000..43b674d5
--- /dev/null
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRate.unit.test.ts
@@ -0,0 +1,112 @@
+import { ExecutionClient, ConsensusClient } from "@stakingbrain/common";
+import { expect } from "chai";
+import { ValidatorPerformance, BlockProposalStatus } from "../../../../src/modules/apiClients/postgres/types.js";
+import { getAttestationSuccessRate } from "../../../../src/modules/validatorsDataIngest/getAttestationSuccessRate";
+
+describe("validatorsDataIngest - getAttestationSuccessRate", () => {
+  it("should return the attestation success rate for a given validator", () => {
+    const validatorData: ValidatorPerformance[] = [
+      {
+        validatorIndex: 0,
+        epoch: 1,
+        executionClient: ExecutionClient.Geth,
+        consensusClient: ConsensusClient.Lighthouse,
+        blockProposalStatus: BlockProposalStatus.Proposed,
+        attestationsTotalRewards: {
+          validator_index: "0",
+          head: "someHead",
+          target: "someTarget",
+          source: "1",
+          inclusion_delay: "0",
+          inactivity: "0"
+        }
+      },
+      {
+        validatorIndex: 0,
+        epoch: 2,
+        executionClient: ExecutionClient.Geth,
+        consensusClient: ConsensusClient.Lighthouse,
+        blockProposalStatus: BlockProposalStatus.Proposed,
+        attestationsTotalRewards: {
+          validator_index: "0",
+          head: "someHead",
+          target: "someTarget",
+          source: "0",
+          inclusion_delay: "0",
+          inactivity: "0"
+        }
+      },
+      {
+        validatorIndex: 0,
+        epoch: 3,
+        executionClient: ExecutionClient.Geth,
+        consensusClient: ConsensusClient.Lighthouse,
+        blockProposalStatus: BlockProposalStatus.Proposed,
+        attestationsTotalRewards: {
+          validator_index: "0",
+          head: "someHead",
+          target: "someTarget",
+          source: "-1",
+          inclusion_delay: "0",
+          inactivity: "0"
+        }
+      }
+    ];
+
+    const startEpoch = 1;
+    const endEpoch = 4; // Total opportunities: 3 (1, 2, 3)
+
+    const successRate = getAttestationSuccessRate({
+      validatorData,
+      startEpoch,
+      endEpoch
+    });
+
+    expect(successRate).to.equal(67); // 2 successful attestations out of 3 opportunities
+  });
+
+  it("should return 0 if the total attestation opportunities are less than or equal to 0", () => {
+    const validatorData: ValidatorPerformance[] = [];
+    const startEpoch = 3;
+    const endEpoch = 3; // Total opportunities: 0
+
+    const successRate = getAttestationSuccessRate({
+      validatorData,
+      startEpoch,
+      endEpoch
+    });
+
+    expect(successRate).to.equal(0);
+  });
+
+  it("should correctly handle edge case with no successful attestations", () => {
+    const validatorData: ValidatorPerformance[] = [
+      {
+        validatorIndex: 0,
+        epoch: 1,
+        executionClient: ExecutionClient.Geth,
+        consensusClient: ConsensusClient.Lighthouse,
+        blockProposalStatus: BlockProposalStatus.Proposed,
+        attestationsTotalRewards: {
+          validator_index: "0",
+          head: "someHead",
+          target: "someTarget",
+          source: "-1", // Unsuccessful
+          inclusion_delay: "0",
+          inactivity: "0"
+        }
+      }
+    ];
+
+    const startEpoch = 1;
+    const endEpoch = 2; // Total opportunities: 1
+
+    const successRate = getAttestationSuccessRate({
+      validatorData,
+      startEpoch,
+      endEpoch
+    });
+
+    expect(successRate).to.equal(0); // No successful attestations
+  });
+});
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.unit.test.ts
new file mode 100644
index 00000000..5ed167a0
--- /dev/null
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.unit.test.ts
@@ -0,0 +1,177 @@
+import { expect } from "chai";
+import { getAttestationSuccessRatePerClients } from "../../../../src/modules/validatorsDataIngest/getAttestationSuccessRatePerClients.js";
+import { ExecutionClient, ConsensusClient } from "@stakingbrain/common";
+import { ValidatorPerformance, BlockProposalStatus } from "../../../../src/modules/apiClients/postgres/types.js";
+
+describe("validatorsDataIngest - getAttestationSuccessRatePerClients", () => {
+  // Sample validator data
+  const validatorData: ValidatorPerformance[] = [
+    {
+      validatorIndex: 1,
+      epoch: 0,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "1",
+        head: "head1",
+        target: "target1",
+        source: "0", // Successful attestation
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 1,
+      epoch: 1,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "1",
+        head: "head2",
+        target: "target2",
+        source: "-1", // Failed attestation
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 1,
+      epoch: 2,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "1",
+        head: "head3",
+        target: "target3",
+        source: "1", // Successful attestation
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 2,
+      epoch: 0,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Prysm,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "2",
+        head: "head1",
+        target: "target1",
+        source: "0", // Successful attestation
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 2,
+      epoch: 1,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Prysm,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "2",
+        head: "head2",
+        target: "target2",
+        source: "0", // Successful attestation
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 2,
+      epoch: 2,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Prysm,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "2",
+        head: "head3",
+        target: "target3",
+        source: "-1", // Failed attestation
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    }
+  ];
+
+  it("should calculate the attestation success rate per clients correctly", () => {
+    const startEpoch = 0;
+    const endEpoch = 3; // Covering epochs 0, 1, and 2
+
+    const result = getAttestationSuccessRatePerClients({
+      validatorData,
+      startEpoch,
+      endEpoch
+    });
+
+    // Check success rates for Geth-Teku
+    expect(result.get(`${ExecutionClient.Geth}-${ConsensusClient.Teku}`)).to.equal(67); // 2 successful out of 3
+
+    // Check success rates for Besu-Prysm
+    expect(result.get(`${ExecutionClient.Besu}-${ConsensusClient.Prysm}`)).to.equal(67); // 2 successful out of 3
+  });
+
+  it("should return 0% success rate if there are no opportunities", () => {
+    const startEpoch = 3;
+    const endEpoch = 3; // No opportunities
+
+    const result = getAttestationSuccessRatePerClients({
+      validatorData,
+      startEpoch,
+      endEpoch
+    });
+
+    expect(result.get(`${ExecutionClient.Geth}-${ConsensusClient.Teku}`)).to.equal(0); // No opportunities
+    expect(result.get(`${ExecutionClient.Besu}-${ConsensusClient.Prysm}`)).to.equal(0); // No opportunities
+  });
+
+  it("should handle a scenario where there are no successful attestations", () => {
+    const validatorDataNoSuccess: ValidatorPerformance[] = [
+      {
+        validatorIndex: 1,
+        epoch: 0,
+        executionClient: ExecutionClient.Geth,
+        consensusClient: ConsensusClient.Teku,
+        blockProposalStatus: BlockProposalStatus.Proposed,
+        attestationsTotalRewards: {
+          validator_index: "1",
+          head: "head1",
+          target: "target1",
+          source: "-1", // Failed attestation
+          inclusion_delay: "0",
+          inactivity: "0"
+        }
+      },
+      {
+        validatorIndex: 1,
+        epoch: 1,
+        executionClient: ExecutionClient.Geth,
+        consensusClient: ConsensusClient.Teku,
+        blockProposalStatus: BlockProposalStatus.Proposed,
+        attestationsTotalRewards: {
+          validator_index: "1",
+          head: "head2",
+          target: "target2",
+          source: "-1", // Failed attestation
+          inclusion_delay: "0",
+          inactivity: "0"
+        }
+      }
+    ];
+
+    const startEpoch = 0;
+    const endEpoch = 2; // Covering epochs 0 and 1
+
+    const result = getAttestationSuccessRatePerClients({
+      validatorData: validatorDataNoSuccess,
+      startEpoch,
+      endEpoch
+    });
+
+    expect(result.get(`${ExecutionClient.Geth}-${ConsensusClient.Teku}`)).to.equal(0); // No successful attestations
+  });
+});
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.unit.test.ts
new file mode 100644
index 00000000..22709a93
--- /dev/null
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.unit.test.ts
@@ -0,0 +1,219 @@
+import { expect } from "chai";
+import { getClientsUsedPerIntervalsMap } from "../../../../src/modules/validatorsDataIngest/getClientsUsedPerIntervalsMap.js";
+import { ExecutionClient, ConsensusClient } from "@stakingbrain/common";
+import { ValidatorPerformance, BlockProposalStatus } from "../../../../src/modules/apiClients/postgres/types.js";
+
+describe("validatorsDataIngest - getClientsUsedPerIntervalsMap", () => {
+  const validatorData: ValidatorPerformance[] = [
+    {
+      validatorIndex: 1,
+      epoch: 0,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "1",
+        head: "head1",
+        target: "target1",
+        source: "source1",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 2,
+      epoch: 1,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Prysm,
+      blockProposalStatus: BlockProposalStatus.Missed,
+      attestationsTotalRewards: {
+        validator_index: "2",
+        head: "head2",
+        target: "target2",
+        source: "source2",
+        inclusion_delay: "1",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 3,
+      epoch: 1,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Lighthouse,
+      blockProposalStatus: BlockProposalStatus.Unchosen,
+      attestationsTotalRewards: {
+        validator_index: "3",
+        head: "head3",
+        target: "target3",
+        source: "source3",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 4,
+      epoch: 2,
+      executionClient: ExecutionClient.Erigon,
+      consensusClient: ConsensusClient.Nimbus,
+      blockProposalStatus: BlockProposalStatus.Error,
+      attestationsTotalRewards: {
+        validator_index: "4",
+        head: "head4",
+        target: "target4",
+        source: "source4",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 5,
+      epoch: 1,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "5",
+        head: "head5",
+        target: "target5",
+        source: "source5",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 6,
+      epoch: 2,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "6",
+        head: "head6",
+        target: "target6",
+        source: "source6",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 7,
+      epoch: 2,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Prysm,
+      blockProposalStatus: BlockProposalStatus.Missed,
+      attestationsTotalRewards: {
+        validator_index: "7",
+        head: "head7",
+        target: "target7",
+        source: "source7",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 8,
+      epoch: 3,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Lighthouse,
+      blockProposalStatus: BlockProposalStatus.Unchosen,
+      attestationsTotalRewards: {
+        validator_index: "8",
+        head: "head8",
+        target: "target8",
+        source: "source8",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 9,
+      epoch: 3,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Nimbus,
+      blockProposalStatus: BlockProposalStatus.Error,
+      attestationsTotalRewards: {
+        validator_index: "9",
+        head: "head9",
+        target: "target9",
+        source: "source9",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 10,
+      epoch: 4,
+      executionClient: ExecutionClient.Erigon,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Proposed,
+      attestationsTotalRewards: {
+        validator_index: "10",
+        head: "head10",
+        target: "target10",
+        source: "source10",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 11,
+      epoch: 4,
+      executionClient: ExecutionClient.Besu,
+      consensusClient: ConsensusClient.Prysm,
+      blockProposalStatus: BlockProposalStatus.Missed,
+      attestationsTotalRewards: {
+        validator_index: "11",
+        head: "head11",
+        target: "target11",
+        source: "source11",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    },
+    {
+      validatorIndex: 12,
+      epoch: 4,
+      executionClient: ExecutionClient.Geth,
+      consensusClient: ConsensusClient.Teku,
+      blockProposalStatus: BlockProposalStatus.Unchosen,
+      attestationsTotalRewards: {
+        validator_index: "12",
+        head: "head12",
+        target: "target12",
+        source: "source12",
+        inclusion_delay: "0",
+        inactivity: "0"
+      }
+    }
+  ];
+
+  it("should return correct counts for a given epoch range", () => {
+    const startEpoch = 1;
+    const endEpoch = 4;
+
+    const result = getClientsUsedPerIntervalsMap({ validatorData, startEpoch, endEpoch });
+    expect(result.size).to.equal(7); // Geth-Teku, Besu-Prysm, Geth-Lighthouse, Erigon-Teku
+    expect(result.get(`${ExecutionClient.Geth}-${ConsensusClient.Teku}`)).to.equal(2);
+    expect(result.get(`${ExecutionClient.Besu}-${ConsensusClient.Prysm}`)).to.equal(2);
+    expect(result.get(`${ExecutionClient.Geth}-${ConsensusClient.Lighthouse}`)).to.equal(1);
+    expect(result.get(`${ExecutionClient.Erigon}-${ConsensusClient.Nimbus}`)).to.equal(1);
+    expect(result.get(`${ExecutionClient.Geth}-${ConsensusClient.Nimbus}`)).to.equal(1);
+    expect(result.get(`${ExecutionClient.Erigon}-${ConsensusClient.Teku}`)).to.equal(0);
+  });
+
+  it("should return zero counts for an epoch range with no data", () => {
+    const startEpoch = 12;
+    const endEpoch = 15;
+
+    const result = getClientsUsedPerIntervalsMap({ validatorData, startEpoch, endEpoch });
+    for (const value of result.values()) expect(value).to.equal(0);
+  });
+
+  it("should handle cases where startEpoch is equal to endEpoch. Nothing should be displayed since the intervals takes the first epoch and not the last one", () => {
+    const startEpoch = 1;
+    const endEpoch = 1;
+
+    const result = getClientsUsedPerIntervalsMap({ validatorData, startEpoch, endEpoch });
+    for (const value of result.values()) expect(value).to.equal(0);
+  });
+});
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getIntervalsEpochs.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getIntervalsEpochs.unit.test.ts
index e69de29b..858961b3 100644
--- a/packages/brain/test/unit/modules/validatorsDataIngest/getIntervalsEpochs.unit.test.ts
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/getIntervalsEpochs.unit.test.ts
@@ -0,0 +1,108 @@
+import { expect } from "chai";
+import { getIntervalsEpochs } from "../../../../src/modules/validatorsDataIngest/getIntervalsEpochs.js";
+import { Granularity } from "../../../../src/modules/validatorsDataIngest/types.js";
+
+describe("validatorsDataIngest - getIntervalsEpochs", () => {
+  const minGenesisTime = 1695902100; // Min genesis time Holesky
+  const secondsPerSlot = 12; // Seconds per slot
+
+  it("should return correct intervals for daily granularity", () => {
+    const startDate = new Date("2024-09-22T00:00:00Z");
+    const endDate = new Date("2024-09-23T00:00:00Z");
+    const granularity = Granularity.Daily;
+
+    const intervals = getIntervalsEpochs({
+      startDate,
+      endDate,
+      granularity,
+      minGenesisTime,
+      secondsPerSlot
+    });
+
+    expect(intervals.length).to.equal(1);
+    expect(intervals[0]).to.deep.equal({
+      startEpoch: 80888,
+      endEpoch: 81113
+    });
+  });
+
+  it("should return correct intervals for hourly granularity", () => {
+    const startDate = new Date("2024-09-22T00:00:00Z");
+    const endDate = new Date("2024-09-22T02:00:00Z");
+    const granularity = Granularity.Hourly;
+
+    const intervals = getIntervalsEpochs({
+      startDate,
+      endDate,
+      granularity,
+      minGenesisTime,
+      secondsPerSlot
+    });
+
+    expect(intervals.length).to.equal(2);
+    expect(intervals[0]).to.deep.equal({
+      startEpoch: 80888,
+      endEpoch: 80897
+    });
+    expect(intervals[1]).to.deep.equal({
+      startEpoch: 80897,
+      endEpoch: 80907
+    });
+  });
+
+  it("should return correct intervals for weekly granularity", () => {
+    const startDate = new Date("2024-08-01T00:00:00Z");
+    const endDate = new Date("2024-08-15T00:00:00Z");
+    const granularity = Granularity.Weekly;
+
+    const intervals = getIntervalsEpochs({
+      startDate,
+      endDate,
+      granularity,
+      minGenesisTime,
+      secondsPerSlot
+    });
+
+    expect(intervals.length).to.equal(2);
+    expect(intervals[0]).to.deep.equal({
+      startEpoch: 69188,
+      endEpoch: 70763
+    });
+    expect(intervals[1]).to.deep.equal({
+      startEpoch: 70763,
+      endEpoch: 72338
+    });
+  });
+
+  it("should handle cases where endDate is the same as startDate", () => {
+    const startDate = new Date("2023-01-01T00:00:00Z");
+    const endDate = new Date("2023-01-01T00:00:00Z");
+    const granularity = Granularity.Hourly;
+
+    const intervals = getIntervalsEpochs({
+      startDate,
+      endDate,
+      granularity,
+      minGenesisTime,
+      secondsPerSlot
+    });
+
+    expect(intervals.length).to.equal(0);
+  });
+
+  it("should return an empty array for invalid date ranges", () => {
+    const startDate = new Date("2023-01-02T00:00:00Z");
+    const endDate = new Date("2023-01-01T00:00:00Z");
+    const granularity = Granularity.Hourly;
+
+    const intervals = getIntervalsEpochs({
+      startDate,
+      endDate,
+      granularity,
+      minGenesisTime,
+      secondsPerSlot
+    });
+
+    expect(intervals.length).to.equal(0);
+  });
+});
diff --git a/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts b/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
index b764a06a..55f90369 100644
--- a/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
+++ b/packages/brain/test/unit/modules/validatorsDataIngest/getStartAndEndEpochs.unit.test.ts
@@ -1,8 +1,8 @@
 import { getStartAndEndEpochs } from "../../../../dist/modules/validatorsDataIngest/getStartAndEndEpochs.js";
 import { expect } from "chai";
 
-describe("getStartAndEndEpochs", () => {
-  it("should return correct start and end epochs for the given date range", () => {
+describe("validatorsDataIngest - getStartAndEndEpochs", () => {
+  it("should return correct start and end epochs for the given date range in Holesky", () => {
     // Define constants
     const minGenesisTime = 1695902100; // Use the provided minGenesisTime
     const secondsPerSlot = 12; // Use the provided secondsPerSlot
@@ -29,4 +29,28 @@ describe("getStartAndEndEpochs", () => {
       endEpoch: expectedEndEpoch
     });
   });
+
+  it("should return correct start and end epochs for the given date range in Holesky", () => {
+    // Define constants
+    const minGenesisTime = 1606824000; // Min genesis time ethereum
+    const secondsPerSlot = 12; // Use the provided secondsPerSlot
+
+    // Define date range for testing
+    const startDate = new Date("Sep-25-2024 16:46:47 UTC+2");
+    const endDate = new Date("Sep-25-2024 17:57:11 UTC+2");
+
+    // Call the function
+    const result = getStartAndEndEpochs({
+      minGenesisTime,
+      secondsPerSlot,
+      startDate,
+      endDate
+    });
+
+    // Assert the results
+    expect(result).to.deep.equal({
+      startEpoch: 313676,
+      endEpoch: 313687
+    });
+  });
 });

From 47e0ea0319a6a9aef1a2bd5c1e171eb9330b099a Mon Sep 17 00:00:00 2001
From: pablomendezroyo <41727368+pablomendezroyo@users.noreply.github.com>
Date: Wed, 25 Sep 2024 22:22:02 +0200
Subject: [PATCH 22/22] Reorg track validators performance cron (#355)

* Reorg track validators performance cron

* remove unused comment
---
 packages/brain/src/index.ts                   |  39 +++---
 packages/brain/src/modules/cron/index.ts      |   5 +-
 .../cron/trackValidatorsPerformance/index.ts  | 112 +--------------
 .../startWithinTenFirstPercentageOfEpoch.ts}  |  30 +++-
 .../trackValidatorsPerformance.ts             | 132 ++++++++++++++++++
 5 files changed, 183 insertions(+), 135 deletions(-)
 rename packages/brain/src/{getSecondsToNextEpoch.ts => modules/cron/trackValidatorsPerformance/startWithinTenFirstPercentageOfEpoch.ts} (51%)
 create mode 100644 packages/brain/src/modules/cron/trackValidatorsPerformance/trackValidatorsPerformance.ts

diff --git a/packages/brain/src/index.ts b/packages/brain/src/index.ts
index 44703bb8..ada2d640 100644
--- a/packages/brain/src/index.ts
+++ b/packages/brain/src/index.ts
@@ -12,10 +12,15 @@ import { startUiServer, startLaunchpadApi } from "./modules/apiServers/index.js"
 import * as dotenv from "dotenv";
 import process from "node:process";
 import { params } from "./params.js";
-import { CronJob, reloadValidators, trackValidatorsPerformance, sendProofsOfValidation } from "./modules/cron/index.js";
+import {
+  CronJob,
+  reloadValidators,
+  trackValidatorsPerformanceCron,
+  sendProofsOfValidation,
+  startWithinTenFirstPercentageOfEpoch
+} from "./modules/cron/index.js";
 import { PostgresClient } from "./modules/apiClients/index.js";
 import { brainConfig } from "./modules/config/index.js";
-import { getSecondsToNextEpoch } from "./getSecondsToNextEpoch.js";
 
 logger.info(`Starting brain...`);
 
@@ -99,34 +104,22 @@ const proofOfValidationCron = new CronJob(shareCronInterval, () =>
 proofOfValidationCron.start();
 
 // executes once every epoch
-const trackValidatorsPerformanceCron = new CronJob(slotsPerEpoch * secondsPerSlot * 1000, async () => {
-  try {
-    const currentEpoch = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
-    await trackValidatorsPerformance({
-      currentEpoch,
-      brainDb,
-      postgresClient,
-      beaconchainApi,
-      executionClient,
-      consensusClient
-    });
-  } catch (e) {
-    logger.error(`Error tracking validator performance: ${e}`);
-  }
+export const trackValidatorsPerformanceCronTask = new CronJob(slotsPerEpoch * secondsPerSlot * 1000, () =>
+  trackValidatorsPerformanceCron({ brainDb, postgresClient, beaconchainApi, executionClient, consensusClient })
+);
+startWithinTenFirstPercentageOfEpoch({
+  minGenesisTime,
+  secondsPerSlot,
+  slotsPerEpoch,
+  jobFunction: trackValidatorsPerformanceCronTask
 });
-// if we are in the first 10% of the epoch we start the cron job if not we wait until the next epoch with a timeout.
-// gnosis chain 80 seconds per epoch -> 8 seconds
-// ethereum 384 seconds per epoch -> 38.4 seconds
-const secondsToNextEpoch = getSecondsToNextEpoch({ minGenesisTime, secondsPerSlot });
-if (secondsToNextEpoch <= slotsPerEpoch * secondsPerSlot * 0.1) trackValidatorsPerformanceCron.start();
-else setTimeout(() => trackValidatorsPerformanceCron.start(), (secondsToNextEpoch + 3) * 1000);
 
 // Graceful shutdown
 function handle(signal: string): void {
   logger.info(`${signal} received. Shutting down...`);
   reloadValidatorsCron.stop();
   proofOfValidationCron.stop();
-  trackValidatorsPerformanceCron.stop();
+  trackValidatorsPerformanceCronTask.stop();
   brainDb.close();
   postgresClient.close().catch((err) => logger.error(`Error closing postgres client`, err)); // postgresClient db connection is the only external resource that needs to be closed
   uiServer.close();
diff --git a/packages/brain/src/modules/cron/index.ts b/packages/brain/src/modules/cron/index.ts
index 2f4c1a03..55cdd064 100644
--- a/packages/brain/src/modules/cron/index.ts
+++ b/packages/brain/src/modules/cron/index.ts
@@ -1,4 +1,7 @@
 export { CronJob } from "./cron.js";
 export { reloadValidators } from "./reloadValidators/index.js";
 export { sendProofsOfValidation } from "./sendProofsOfValidation/index.js";
-export { trackValidatorsPerformance } from "./trackValidatorsPerformance/index.js";
+export {
+  trackValidatorsPerformanceCron,
+  startWithinTenFirstPercentageOfEpoch
+} from "./trackValidatorsPerformance/index.js";
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
index be318860..8714c2c4 100644
--- a/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/index.ts
@@ -1,110 +1,2 @@
-import { BeaconchainApi } from "../../apiClients/beaconchain/index.js";
-import { PostgresClient } from "../../apiClients/postgres/index.js";
-import logger from "../../logger/index.js";
-import { BrainDataBase } from "../../db/index.js";
-import { insertPerformanceDataNotThrow } from "./insertPerformanceData.js";
-import { getAttestationsTotalRewards } from "./getAttestationsTotalRewards.js";
-import { getBlockProposalStatusMap } from "./getBlockProposalStatusMap.js";
-import { getActiveValidatorsLoadedInBrain } from "./getActiveValidatorsLoadedInBrain.js";
-import { logPrefix } from "./logPrefix.js";
-import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
-
-// TODO: at this moment Lighthouse client does not support retrieving:
-// - liveness of validator from finalized epoch:
-// ```400: BAD_REQUEST: request epoch 79833 is more than one epoch from the current epoch 79835```
-// - sync committee rewards:
-// ```404: NOT_FOUND: Parent state is not available! MissingBeaconState(0xa9592014ad4aa3d5dcc4ef67b669278a85fb4dbe80f12364f2486444b7db3927)```
-
-/**
- * Cron task that will track validators performance for the epoch finalized and store it in the Postgres DB.
- * If any issue is arisen during the process, it will be retried after 30 seconds. If the issue persists until the epoch
- * finalized changes, the issue will be logged and stored in the DB.
- *
- * @param validatorPubkeys - The pubkeys of the validators to track.
- * @param postgresClient - Postgres client to interact with the DB.
- * @param beaconchainApi - Beaconchain API client to interact with the Beaconchain API.
- * @param executionClient - The execution client to interact with.
- * @param consensusClient - The consensus client to interact with.
- *
- * @throws {Error} If there is an error when updating the latestEpoch in the error handling
- */
-export async function trackValidatorsPerformance({
-  currentEpoch,
-  brainDb,
-  postgresClient,
-  beaconchainApi,
-  executionClient,
-  consensusClient
-}: {
-  currentEpoch: number;
-  brainDb: BrainDataBase;
-  postgresClient: PostgresClient;
-  beaconchainApi: BeaconchainApi;
-  executionClient: ExecutionClient;
-  consensusClient: ConsensusClient;
-}): Promise<void> {
-  let latestEpoch = currentEpoch;
-
-  while (currentEpoch === latestEpoch) {
-    try {
-      logger.debug(`${logPrefix}Starting to track performance for epoch: ${currentEpoch}`);
-
-      const activeValidatorsIndexes = await getActiveValidatorsLoadedInBrain({ beaconchainApi, brainDb });
-      if (activeValidatorsIndexes.length === 0) {
-        logger.info(`${logPrefix}No active validators found`);
-        return; // Exit if no active validators are found
-      }
-
-      const { el_offline, is_syncing } = (await beaconchainApi.getSyncingStatus()).data;
-      if (is_syncing) {
-        logger.debug(`${logPrefix}Node is syncing, skipping epoch ${currentEpoch}`);
-        return; // Exit if the node is syncing. Head finalized will change
-      }
-      if (el_offline) throw new Error("EL Node offline"); // throw error and retry
-
-      const validatorsAttestationsTotalRewards = await getAttestationsTotalRewards({
-        beaconchainApi,
-        epoch: currentEpoch.toString(),
-        activeValidatorsIndexes
-      });
-
-      const validatorBlockStatusMap = await getBlockProposalStatusMap({
-        beaconchainApi,
-        epoch: currentEpoch.toString(),
-        activeValidatorsIndexes
-      });
-
-      await insertPerformanceDataNotThrow({
-        postgresClient,
-        activeValidatorsIndexes,
-        currentEpoch,
-        validatorBlockStatusMap,
-        validatorsAttestationsTotalRewards,
-        error: undefined,
-        executionClient,
-        consensusClient
-      });
-
-      logger.debug(`${logPrefix}Finished tracking performance for epoch: ${currentEpoch}`);
-      return; // Success, exit function
-    } catch (e) {
-      logger.error(`${logPrefix}Error tracking validator peformance for epoch ${currentEpoch}: ${e}`);
-      latestEpoch = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
-      if (latestEpoch !== currentEpoch) {
-        logger.info(`${logPrefix}Epoch has changed from ${currentEpoch} to ${latestEpoch}, aborting retry.`);
-        await insertPerformanceDataNotThrow({
-          postgresClient,
-          activeValidatorsIndexes: [],
-          currentEpoch,
-          validatorBlockStatusMap: new Map(),
-          validatorsAttestationsTotalRewards: [],
-          error: e.message, // Store the error in the DB after all retries are exhausted
-          executionClient,
-          consensusClient
-        });
-        return; // Exit after final attempt
-      }
-      await new Promise((resolve) => setTimeout(resolve, 30 * 1000)); // Wait 30 seconds before retrying
-    }
-  }
-}
+export { trackValidatorsPerformanceCron } from "./trackValidatorsPerformance.js";
+export { startWithinTenFirstPercentageOfEpoch } from "./startWithinTenFirstPercentageOfEpoch.js";
diff --git a/packages/brain/src/getSecondsToNextEpoch.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/startWithinTenFirstPercentageOfEpoch.ts
similarity index 51%
rename from packages/brain/src/getSecondsToNextEpoch.ts
rename to packages/brain/src/modules/cron/trackValidatorsPerformance/startWithinTenFirstPercentageOfEpoch.ts
index d12b73e1..8b4b239c 100644
--- a/packages/brain/src/getSecondsToNextEpoch.ts
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/startWithinTenFirstPercentageOfEpoch.ts
@@ -1,3 +1,31 @@
+import { CronJob } from "../cron";
+
+/**
+ * Start a cron job within the first 10% of the epoch.
+ * if we are in the first 10% of the epoch we start the cron job if not we wait until the next epoch with a timeout.
+ * - gnosis chain 80 seconds per epoch -> 8 seconds
+ * - ethereum 384 seconds per epoch -> 38.4 seconds
+ *
+ * @param minGenesisTime - Minimum genesis time of the chain.
+ * @param secondsPerSlot - Seconds per slot.
+ * @param jobFunction - Cron job function.
+ */
+export function startWithinTenFirstPercentageOfEpoch({
+  minGenesisTime,
+  secondsPerSlot,
+  slotsPerEpoch,
+  jobFunction
+}: {
+  minGenesisTime: number;
+  secondsPerSlot: number;
+  slotsPerEpoch: number;
+  jobFunction: CronJob;
+}): void {
+  const secondsToNextEpoch = getSecondsToNextEpoch({ minGenesisTime, secondsPerSlot });
+  if (secondsToNextEpoch <= slotsPerEpoch * secondsPerSlot * 0.1) jobFunction.start();
+  else setTimeout(() => jobFunction.start(), (secondsToNextEpoch + 3) * 1000);
+}
+
 /**
  * Get the seconds to the start of the next epoch based on the current Unix time and the minimum genesis time of the chain.
  *
@@ -5,7 +33,7 @@
  * @param {number} secondsPerSlot - Seconds per slot.
  * @returns {number} - Seconds to the start of the next epoch.
  */
-export function getSecondsToNextEpoch({
+function getSecondsToNextEpoch({
   minGenesisTime,
   secondsPerSlot
 }: {
diff --git a/packages/brain/src/modules/cron/trackValidatorsPerformance/trackValidatorsPerformance.ts b/packages/brain/src/modules/cron/trackValidatorsPerformance/trackValidatorsPerformance.ts
new file mode 100644
index 00000000..c05d8ba7
--- /dev/null
+++ b/packages/brain/src/modules/cron/trackValidatorsPerformance/trackValidatorsPerformance.ts
@@ -0,0 +1,132 @@
+import { BeaconchainApi } from "../../apiClients/beaconchain/index.js";
+import { PostgresClient } from "../../apiClients/postgres/index.js";
+import logger from "../../logger/index.js";
+import { BrainDataBase } from "../../db/index.js";
+import { insertPerformanceDataNotThrow } from "./insertPerformanceData.js";
+import { getAttestationsTotalRewards } from "./getAttestationsTotalRewards.js";
+import { getBlockProposalStatusMap } from "./getBlockProposalStatusMap.js";
+import { getActiveValidatorsLoadedInBrain } from "./getActiveValidatorsLoadedInBrain.js";
+import { logPrefix } from "./logPrefix.js";
+import { ConsensusClient, ExecutionClient } from "@stakingbrain/common";
+
+export async function trackValidatorsPerformanceCron({
+  brainDb,
+  postgresClient,
+  beaconchainApi,
+  executionClient,
+  consensusClient
+}: {
+  brainDb: BrainDataBase;
+  postgresClient: PostgresClient;
+  beaconchainApi: BeaconchainApi;
+  executionClient: ExecutionClient;
+  consensusClient: ConsensusClient;
+}): Promise<void> {
+  try {
+    const currentEpoch = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
+    await trackValidatorsPerformance({
+      currentEpoch,
+      brainDb,
+      postgresClient,
+      beaconchainApi,
+      executionClient,
+      consensusClient
+    });
+  } catch (e) {
+    logger.error(`Error tracking validator performance: ${e}`);
+  }
+}
+
+/**
+ * Cron task that will track validators performance for the epoch finalized and store it in the Postgres DB.
+ * If any issue is arisen during the process, it will be retried after 30 seconds. If the issue persists until the epoch
+ * finalized changes, the issue will be logged and stored in the DB.
+ *
+ * @param validatorPubkeys - The pubkeys of the validators to track.
+ * @param postgresClient - Postgres client to interact with the DB.
+ * @param beaconchainApi - Beaconchain API client to interact with the Beaconchain API.
+ * @param executionClient - The execution client to interact with.
+ * @param consensusClient - The consensus client to interact with.
+ *
+ * @throws {Error} If there is an error when updating the latestEpoch in the error handling
+ */
+async function trackValidatorsPerformance({
+  currentEpoch,
+  brainDb,
+  postgresClient,
+  beaconchainApi,
+  executionClient,
+  consensusClient
+}: {
+  currentEpoch: number;
+  brainDb: BrainDataBase;
+  postgresClient: PostgresClient;
+  beaconchainApi: BeaconchainApi;
+  executionClient: ExecutionClient;
+  consensusClient: ConsensusClient;
+}): Promise<void> {
+  let latestEpoch = currentEpoch;
+
+  while (currentEpoch === latestEpoch) {
+    try {
+      logger.debug(`${logPrefix}Starting to track performance for epoch: ${currentEpoch}`);
+
+      const activeValidatorsIndexes = await getActiveValidatorsLoadedInBrain({ beaconchainApi, brainDb });
+      if (activeValidatorsIndexes.length === 0) {
+        logger.info(`${logPrefix}No active validators found`);
+        return; // Exit if no active validators are found
+      }
+
+      const { el_offline, is_syncing } = (await beaconchainApi.getSyncingStatus()).data;
+      if (is_syncing) {
+        logger.debug(`${logPrefix}Node is syncing, skipping epoch ${currentEpoch}`);
+        return; // Exit if the node is syncing. Head finalized will change
+      }
+      if (el_offline) throw new Error("EL Node offline"); // throw error and retry
+
+      const validatorsAttestationsTotalRewards = await getAttestationsTotalRewards({
+        beaconchainApi,
+        epoch: currentEpoch.toString(),
+        activeValidatorsIndexes
+      });
+
+      const validatorBlockStatusMap = await getBlockProposalStatusMap({
+        beaconchainApi,
+        epoch: currentEpoch.toString(),
+        activeValidatorsIndexes
+      });
+
+      await insertPerformanceDataNotThrow({
+        postgresClient,
+        activeValidatorsIndexes,
+        currentEpoch,
+        validatorBlockStatusMap,
+        validatorsAttestationsTotalRewards,
+        error: undefined,
+        executionClient,
+        consensusClient
+      });
+
+      logger.debug(`${logPrefix}Finished tracking performance for epoch: ${currentEpoch}`);
+      return; // Success, exit function
+    } catch (e) {
+      logger.error(`${logPrefix}Error tracking validator peformance for epoch ${currentEpoch}: ${e}`);
+      latestEpoch = await beaconchainApi.getEpochHeader({ blockId: "finalized" });
+      if (latestEpoch !== currentEpoch) {
+        logger.info(`${logPrefix}Epoch has changed from ${currentEpoch} to ${latestEpoch}, aborting retry.`);
+        await insertPerformanceDataNotThrow({
+          postgresClient,
+          activeValidatorsIndexes: [],
+          currentEpoch,
+          validatorBlockStatusMap: new Map(),
+          validatorsAttestationsTotalRewards: [],
+          error: e.message, // Store the error in the DB after all retries are exhausted
+          executionClient,
+          consensusClient
+        });
+        return; // Exit after final attempt
+      }
+      await new Promise((resolve) => setTimeout(resolve, 30 * 1000)); // Wait 30 seconds before retrying
+    }
+  }
+}