[Feature] - cncli.sh - integration of full stakepool history load into epochdata table

.github/workflows/mithril-latest.yml

@@ -7,7 +7,7 @@ jobs:
   get-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.REPO_SCOPED_TOKEN }}
           fetch-depth: 0
@@ -18,13 +18,19 @@ jobs:
       - name: Assigns release version
         run: |
           VERSION=$(cat ./files/docker/node/release-versions/mithril-latest.txt)
+      - name: Source mithril.library and check upgrade safety
+        run: |
+          set -e
+          . scripts/cnode-helper-scripts/mithril.library workflow
+          check_mithril_upgrade_safe
+          echo "MITHRIL_UPGRADE_SAFE=$MITHRIL_UPGRADE_SAFE" >> $GITHUB_ENV
       - name: Check for modified files
         id: git-check
         run: echo ::set-output name=modified::$([ -z "`git status --porcelain`" ] && echo "false" || echo "true")
       - name: Commit latest release version
-        if: steps.git-check.outputs.modified == 'true'
+        if: steps.git-check.outputs.modified == 'true' && env.MITHRIL_UPGRADE_SAFE == 'Y'
         run: |
           git config --global user.name ${{ secrets.REPO_SCOPED_USER }}
           git config --global user.email ${{ secrets.REPO_SCOPED_EMAIL }}
           git commit -am "New mithril release version ${VERSION}"
-          git push
+          git push

scripts/cnode-helper-scripts/cncli.sh

@@ -45,10 +45,13 @@ usage() {
 		  force     Manually force leaderlog calculation and overwrite even if already done, exits after leaderlog is calculated
 		validate    Continously monitor and confirm that the blocks made actually was accepted and adopted by chain (deployed as service)
 		  all       One-time re-validation of all blocks in blocklog db
-		  epoch     One-time re-validation of blocks in blocklog db for the specified epoch 
+		  epoch     One-time re-validation of blocks in blocklog db for the specified epoch
+		epochdata   Manually re-calculate leaderlog to load stakepool history into epochdata table of blocklog db. Needs completion of cncli.sh sync and cncli.sh validate processes.
+		  all       One-time re-calculation of all epochs (avg execution duration: 1hr / 50 epochs)
+		  epoch     One-time re-calculation for the specified epoch
 		ptsendtip   Send node tip to PoolTool for network analysis and to show that your node is alive and well with a green badge (deployed as service)
 		ptsendslots Securely sends PoolTool the number of slots you have assigned for an epoch and validates the correctness of your past epochs (deployed as service)
-		  force     Manually force pooltool sendslots submission ignoring configured time window 
+		  force     Manually force pooltool sendslots submission ignoring configured time window
 		init        One-time initialization adding all minted and confirmed blocks to blocklog
 		metrics     Print cncli block metrics in Prometheus format
 		  deploy    Install dependencies and deploy cncli monitoring agent service (available through port specified by CNCLI_PROM_PORT)
@@ -127,11 +130,15 @@ getLedgerData() { # getNodeMetrics expected to have been already run
 }
 
 getConsensus() {
-  getProtocolParams
+  if isNumber "$1"; then
+     getProtocolParamsHist "$(( $1 - 1 ))" || return 1
+  else
+     getProtocolParams || return 1
+  fi
   if versionCheck "9.0" "${PROT_VERSION}"; then
     consensus="cpraos"
     stability_window_factor=3
-  elif versionCheck "8.0" "${PROT_VERSION}"; then
+  elif versionCheck "7.0" "${PROT_VERSION}"; then
     consensus="praos"
     stability_window_factor=2
   else
@@ -795,10 +802,184 @@ cncliPTsendslots() {
   done
 }
 
+#################################
+# epochdata table load process  #
+#################################
+
+getCurrNextEpoch() {
+  getNodeMetrics
+  curr_epoch=${epochnum}
+  next_epoch=$((curr_epoch+1))
+}
+
+runCurrentEpoch() {
+  getKoiosData
+  echo "Processing current epoch: ${1}"
+  stake_param_curr="--active-stake ${active_stake_set} --pool-stake ${pool_stake_set}"
+
+  ${CNCLI} leaderlog ${cncliParams} --consensus "${consensus}" --epoch="${1}" ${stake_param_curr} |
+  jq -r '[.epoch, .epochNonce, .poolId, .sigma, .d, .epochSlotsIdeal, .maxPerformance, .activeStake, .totalActiveStake] | @csv' |
+  sed 's/"//g' >> "$tmpcsv"
+}
+
+runNextEpoch() {
+  getKoiosData
+  echo "Processing next epoch: ${1}"
+  stake_param_next="--active-stake ${active_stake_mark} --pool-stake ${pool_stake_mark}"
+
+  ${CNCLI} leaderlog ${cncliParams} --consensus "${consensus}" --epoch="${1}" ${stake_param_next} |
+  jq -r '[.epoch, .epochNonce, .poolId, .sigma, .d, .epochSlotsIdeal, .maxPerformance, .activeStake, .totalActiveStake] | @csv' |
+  sed 's/"//g' >> "$tmpcsv"
+}
+
+runPreviousEpochs() {
+  [[ -z ${KOIOS_API} ]] && return 1
+  if ! pool_hist=$(curl -sSL -f "${KOIOS_API}/pool_history?_pool_bech32=${POOL_ID_BECH32}&_epoch_no=${1}" 2>&1); then
+    echo "ERROR: Koios pool_stake_snapshot history query failed."
+    return 1
+  fi
+
+  if ! epoch_hist=$(curl -sSL -f "${KOIOS_API}/epoch_info?_epoch_no=${1}" 2>&1); then
+    echo "ERROR: Koios epoch_stake_snapshot history query failed."
+    return 1
+  fi
+
+  pool_stake_hist=$(jq -r '.[].active_stake' <<< "${pool_hist}")
+  active_stake_hist=$(jq -r '.[].active_stake' <<< "${epoch_hist}")
+
+  echo "Processing previous epoch: ${1}"
+  stake_param_prev="--active-stake ${active_stake_hist} --pool-stake ${pool_stake_hist}"
+
+  ${CNCLI} leaderlog ${cncliParams} --consensus "${consensus}" --epoch="${1}" ${stake_param_prev} |
+  jq -r '[.epoch, .epochNonce, .poolId, .sigma, .d, .epochSlotsIdeal, .maxPerformance, .activeStake, .totalActiveStake] | @csv' |
+  sed 's/"//g' >> "$tmpcsv"
+
+  return 0
+}
+
+processAllEpochs() {
+  getCurrNextEpoch
+  IFS=' ' read -r -a epochs_array <<< "$EPOCHS"
+
+  for epoch in "${epochs_array[@]}"; do
+    if ! getConsensus "${epoch}"; then echo "ERROR: Failed to fetch protocol parameters for epoch ${epoch}."; return 1; fi
+    if [[ "$epoch" == "$curr_epoch" ]]; then
+      runCurrentEpoch ${epoch}
+    elif [[ "$epoch" == "$next_epoch" ]]; then
+      runNextEpoch ${epoch}
+    else
+      runPreviousEpochs ${epoch}
+    fi
+  done
+
+  id=1
+  while IFS= read -r row; do
+    echo "$id,$row" >> "$csvfile"
+    ((id++))
+  done < "$tmpcsv"
+
+  sqlite3 "$BLOCKLOG_DB" <<EOF
+DELETE FROM epochdata;
+VACUUM;
+.mode csv
+.import '$csvfile' epochdata
+REINDEX epochdata;
+EOF
+
+  row_count=$(sqlite3 "$BLOCKLOG_DB" "SELECT COUNT(*) FROM epochdata;")
+  echo "$row_count rows have been loaded into epochdata table in blocklog db"
+  echo "~ CNCLI epochdata table load completed ~"
+
+  rm $csvfile $tmpcsv
+}
+
+processSingleEpoch() {
+  getCurrNextEpoch
+  IFS=' ' read -r -a epochs_array <<< "$EPOCHS"
+
+  unset matched
+  for epoch in "${epochs_array[@]}"; do
+    [[ ${epoch} = "$1" ]] && matched=true && break
+  done
+  if [[ -z ${matched} ]]; then
+    echo -e "No slots found in blocklog table for epoch ${1}.\n"; return 1
+  fi
+  if ! getConsensus "${1}"; then echo "ERROR: Failed to fetch protocol parameters for epoch ${1}."; return 1; fi
+  if [[ "$1" == "$curr_epoch" ]]; then
+     runCurrentEpoch ${1}
+  elif [[ "$1" == "$next_epoch" ]]; then
+     runNextEpoch ${1}
+  else
+     runPreviousEpochs ${1}
+  fi
+
+  ID=$(sqlite3 "$BLOCKLOG_DB" "SELECT max(id) + 1 FROM epochdata;")
+  csv_row=$(cat "$tmpcsv")
+  modified_csv_row="${ID},${csv_row}"
+  echo "$modified_csv_row" > "$onerow_csv"
+
+  sqlite3 "$BLOCKLOG_DB" "DELETE FROM epochdata WHERE epoch = ${1};"
+  sqlite3 "$BLOCKLOG_DB" <<EOF
+.mode csv
+.import "$onerow_csv" epochdata
+REINDEX epochdata;
+EOF
+   row_count=$(sqlite3 "$BLOCKLOG_DB" "SELECT COUNT(*) FROM epochdata WHERE epoch = ${1};")
+   echo "$row_count row has been loaded into epochdata table in blocklog db for epoch ${1}"
+   echo "~ CNCLI epochdata table load completed ~"
+   echo
+
+   rm $onerow_csv $tmpcsv
+}
+
+cncliEpochData() {
+  getNodeMetrics
+  cncliParams="--db ${CNCLI_DB} --byron-genesis ${BYRON_GENESIS_JSON} --shelley-genesis ${GENESIS_JSON} --pool-id ${POOL_ID} --pool-vrf-skey ${POOL_VRF_SKEY} --tz UTC"
+  EPOCHS=$(sqlite3 "$BLOCKLOG_DB" "SELECT group_concat(epoch,' ') FROM (SELECT DISTINCT epoch FROM blocklog ORDER BY epoch);")
+  csvdir=/tmp ; tmpcsv="${csvdir}/epochdata_tmp.csv" ; csvfile="${csvdir}/epochdata.csv" ; onerow_csv="${csvdir}/one_epochdata.csv"
+  > "$tmpcsv" ; > "$csvfile" ; > "$onerow_csv"
+
+  proc_msg="~ CNCLI epochdata table load started ~"
+
+  if ! cncliDBinSync; then
+    echo ${proc_msg}
+    echo "CNCLI DB out of sync :( [$(printf "%2.4f %%" ${cncli_sync_prog})] ... check cncli sync service!"
+    exit 1
+  else
+    echo ${proc_msg}
+    read_genesis
+    local epochSlot=${slot_in_epoch}
+    local epochLength=${EPOCH_LENGTH}
+    local remainingSlots=$(( epochLength - epochSlot))
+    local waitFromEpochstart=21600 # if within 6 hours of new epoch don't run epochdata
+
+    if [[ $(( epochLength - remainingSlots)) -le $waitFromEpochstart ]]; then
+      getCurrNextEpoch
+      echo "First six hours of new epoch "$curr_epoch" detected, too early to run this process."
+      echo "Current epochSlot is ${epochSlot}, please try againg after epochSlot 21600"
+      echo
+    else
+
+      if [[ "${subcommand}" == "epochdata" ]]; then
+          if [[ ${subarg} == "all" ]]; then
+             processAllEpochs
+        elif isNumber "${subarg}"; then
+             processSingleEpoch "${subarg}"
+      else
+          echo
+          echo "ERROR: unknown argument passed to validate command, valid options incl the string 'all' or the epoch number to recalculate"
+          echo
+          exit 1
+        fi
+      fi
+    fi
+  fi
+}
+
 #################################
 
 case ${subcommand} in
-  sync ) 
+  sync )
     cncliInit && cncliSync ;;
   leaderlog )
     cncliInit && cncliLeaderlog ;;
@@ -812,5 +993,8 @@ case ${subcommand} in
     cncliInit && cncliInitBlocklogDB ;;
   metrics )
     cncliMetrics ;; # no cncliInit needed
-  * ) usage ;;
+  epochdata )
+    cncliInit && cncliEpochData ;;
+  * )
+    usage ;;
 esac

scripts/cnode-helper-scripts/env

@@ -110,6 +110,7 @@
 #CIP0094_POLL_URL="https://raw.githubusercontent.com/cardano-foundation/CIP-0094-polls/main/networks/polls.json" # URL for polls to vote against
 
 #MITHRIL_DOWNLOAD="N"                                   # (Y|N) Download latest Mithril snapshot
+#MITHRIL_HOME="${CNODE_HOME}/mithril"                   # Override default Mithril path
 #MITHRIL_SIGNER_ENABLED="N"                             # (Y|N) Enable the gLiveView Mithril Signer Section
 
 #STRICT_VERSION_CHECK="Y"                               # (Y|N) Restrict operation to supported major.minor version leaving patch version open. If disabled, any version will be accepted (unsupported)
@@ -844,6 +845,24 @@ getProtocolParams() {
   [[ ${PROT_MAJOR} -eq 0 ]] && return 1 || return 0
 }
 
+getProtocolParamsHist() {
+  if [[ -z ${KOIOS_API} ]]; then
+    return 1
+  fi
+
+  if ! PROT_PARAMS=$(curl -sSL -f -X GET -H "accept: application/json" "${KOIOS_API}/epoch_params?_epoch_no=${1}" 2>&1); then
+    return 2
+  fi
+
+  # Extract historical protocol major and minor versions in TSV format
+  read -r PROT_MAJOR PROT_MINOR <<<"$(jq -r '.[0] | [
+    .protocol_major //0,
+    .protocol_minor //0
+  ] | @tsv' <<<"${PROT_PARAMS}" 2>/dev/null)"
+  PROT_VERSION="${PROT_MAJOR}.${PROT_MINOR}"
+
+}
+
 telegramSend() {
   if [[ -z "${TG_BOT_TOKEN}" ]] || [[ -z "${TG_CHAT_ID}" ]]; then
     echo "Warn: to use the telegramSend function you must first set the bot and chat id in the env file"
@@ -950,6 +969,7 @@ set_default_vars() {
   [[ -z ${ASSET_POLICY_ID_FILENAME} ]] && ASSET_POLICY_ID_FILENAME="policy.id"
   [[ -z ${CIP0094_POLL_URL} ]] && CIP0094_POLL_URL="https://raw.githubusercontent.com/cardano-foundation/CIP-0094-polls/main/networks/polls.json"
   [[ -z ${MITHRIL_DOWNLOAD} ]] && MITHRIL_DOWNLOAD="N"
+  [[ -z ${MITHRIL_HOME} ]] && MITHRIL_HOME="${CNODE_HOME}/mithril"
   [[ -z ${MITHRIL_SIGNER_ENABLED} ]] && MITHRIL_SIGNER_ENABLED="N"
   [[ -z ${STRICT_VERSION_CHECK} ]] && STRICT_VERSION_CHECK="Y"
   FG_BLACK='\e[30m'

scripts/cnode-helper-scripts/gLiveView.sh

@@ -778,8 +778,8 @@ unset cpu_now cpu_last
 
 mithrilSignerVars() {
   # mithril.env sourcing needed to have values in ${METRICS_SERVER_IP} and ${METRICS_SERVER_PORT}
-  . ${CNODE_HOME}/mithril/mithril.env
-  signerMetricsEnabled=$(grep -q "ENABLE_METRICS_SERVER=true" ${CNODE_HOME}/mithril/mithril.env && echo "true" || echo "false")
+  . ${MITHRIL_HOME}/mithril.env
+  signerMetricsEnabled=$(grep -q "ENABLE_METRICS_SERVER=true" ${MITHRIL_HOME}/mithril.env && echo "true" || echo "false")
   if [[ "${signerMetricsEnabled}" == "true" ]] ; then
     mithrilSignerMetrics=$(curl -s "http://${METRICS_SERVER_IP}:${METRICS_SERVER_PORT}/metrics" 2>/dev/null | grep -v -E "HELP|TYPE" | sed 's/mithril_signer_//g')
     SIGNER_METRICS_HTTP_RESPONSE=$(curl --write-out "%{http_code}" --silent --output /dev/null --connect-timeout 2 http://${METRICS_SERVER_IP}:${METRICS_SERVER_PORT}/metrics)

scripts/cnode-helper-scripts/guild-deploy.sh

@@ -112,6 +112,7 @@ set_defaults() {
   [[ -z "${CARDANO_NODE_VERSION}" ]] && CARDANO_NODE_VERSION="$(curl -sfk "https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators/${BRANCH}/files/docker/node/release-versions/cardano-node-latest.txt")"
   CNODE_HOME="${CNODE_PATH}/${CNODE_NAME}"
   CNODE_VNAME=$(echo "$CNODE_NAME" | awk '{print toupper($0)}')
+  [[ -z ${MITHRIL_HOME} ]] && MITHRIL_HOME="${CNODE_HOME}/mithril"
   REPO="https://github.com/${G_ACCOUNT}/guild-operators"
   REPO_RAW="https://raw.githubusercontent.com/${G_ACCOUNT}/guild-operators"
   URL_RAW="${REPO_RAW}/${BRANCH}"
@@ -588,7 +589,7 @@ setup_folder() {
     echo -e "\nexport ${CNODE_VNAME}_HOME=${CNODE_HOME}" >> "${HOME}"/.bashrc
   fi
 
-  $sudo mkdir -p "${CNODE_HOME}"/files "${CNODE_HOME}"/db "${CNODE_HOME}"/guild-db "${CNODE_HOME}"/logs "${CNODE_HOME}"/scripts "${CNODE_HOME}"/scripts/archive "${CNODE_HOME}"/sockets "${CNODE_HOME}"/priv "${CNODE_HOME}"/mithril/data-stores
+  $sudo mkdir -p "${CNODE_HOME}"/files "${CNODE_HOME}"/db "${CNODE_HOME}"/guild-db "${CNODE_HOME}"/logs "${CNODE_HOME}"/scripts "${CNODE_HOME}"/scripts/archive "${CNODE_HOME}"/sockets "${CNODE_HOME}"/priv "${MITHRIL_HOME}"/data-stores
   $sudo chown -R "$U_ID":"$G_ID" "${CNODE_HOME}" 2>/dev/null
 
 }