Add prometheus metrics

[adammilnesmith]

Example from /metrics for easier review. Metrics prefixed with node_indexer are most relevant.

# HELP go_gc_duration_seconds A summary of the pause duration of garbage collection cycles.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 6.1738e-05
go_gc_duration_seconds{quantile="0.25"} 7.1521e-05
go_gc_duration_seconds{quantile="0.5"} 9.9216e-05
go_gc_duration_seconds{quantile="0.75"} 0.000120568
go_gc_duration_seconds{quantile="1"} 0.000171919
go_gc_duration_seconds_sum 0.000932135
go_gc_duration_seconds_count 9
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 100
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.22.5"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 5.6840992e+07
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 7.1836536e+07
# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table.
# TYPE go_memstats_buck_hash_sys_bytes gauge
go_memstats_buck_hash_sys_bytes 7827
# HELP go_memstats_frees_total Total number of frees.
# TYPE go_memstats_frees_total counter
go_memstats_frees_total 86933
# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata.
# TYPE go_memstats_gc_sys_bytes gauge
go_memstats_gc_sys_bytes 3.155264e+06
# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use.
# TYPE go_memstats_heap_alloc_bytes gauge
go_memstats_heap_alloc_bytes 5.6840992e+07
# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used.
# TYPE go_memstats_heap_idle_bytes gauge
go_memstats_heap_idle_bytes 3.23584e+06
# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use.
# TYPE go_memstats_heap_inuse_bytes gauge
go_memstats_heap_inuse_bytes 6.1022208e+07
# HELP go_memstats_heap_objects Number of allocated objects.
# TYPE go_memstats_heap_objects gauge
go_memstats_heap_objects 214892
# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS.
# TYPE go_memstats_heap_released_bytes gauge
go_memstats_heap_released_bytes 3.227648e+06
# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system.
# TYPE go_memstats_heap_sys_bytes gauge
go_memstats_heap_sys_bytes 6.4258048e+07
# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection.
# TYPE go_memstats_last_gc_time_seconds gauge
go_memstats_last_gc_time_seconds 1.721748720169012e+09
# HELP go_memstats_lookups_total Total number of pointer lookups.
# TYPE go_memstats_lookups_total counter
go_memstats_lookups_total 0
# HELP go_memstats_mallocs_total Total number of mallocs.
# TYPE go_memstats_mallocs_total counter
go_memstats_mallocs_total 301825
# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures.
# TYPE go_memstats_mcache_inuse_bytes gauge
go_memstats_mcache_inuse_bytes 38400
# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system.
# TYPE go_memstats_mcache_sys_bytes gauge
go_memstats_mcache_sys_bytes 46800
# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures.
# TYPE go_memstats_mspan_inuse_bytes gauge
go_memstats_mspan_inuse_bytes 554560
# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system.
# TYPE go_memstats_mspan_sys_bytes gauge
go_memstats_mspan_sys_bytes 554880
# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place.
# TYPE go_memstats_next_gc_bytes gauge
go_memstats_next_gc_bytes 9.3412448e+07
# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations.
# TYPE go_memstats_other_sys_bytes gauge
go_memstats_other_sys_bytes 4.581869e+06
# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator.
# TYPE go_memstats_stack_inuse_bytes gauge
go_memstats_stack_inuse_bytes 2.850816e+06
# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator.
# TYPE go_memstats_stack_sys_bytes gauge
go_memstats_stack_sys_bytes 2.850816e+06
# HELP go_memstats_sys_bytes Number of bytes obtained from system.
# TYPE go_memstats_sys_bytes gauge
go_memstats_sys_bytes 7.5455504e+07
# HELP go_threads Number of OS threads created.
# TYPE go_threads gauge
go_threads 33
# HELP node_indexer_dlq_size The number of blocks in the DLQ
# TYPE node_indexer_dlq_size gauge
node_indexer_dlq_size 184630
# HELP node_indexer_dune_client_block_per_batch Number of blocks per batch
# TYPE node_indexer_dune_client_block_per_batch histogram
node_indexer_dune_client_block_per_batch_bucket{status="200",le="1"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="2"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="4"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="8"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="16"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="32"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="64"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="128"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="256"} 33
node_indexer_dune_client_block_per_batch_bucket{status="200",le="+Inf"} 33
node_indexer_dune_client_block_per_batch_sum{status="200"} 0
node_indexer_dune_client_block_per_batch_count{status="200"} 33
# HELP node_indexer_dune_client_send_block_batch_duration_millis Duration of a send block batch request in milliseconds
# TYPE node_indexer_dune_client_send_block_batch_duration_millis histogram
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="10"} 0
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="25"} 0
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="50"} 0
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="100"} 13
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="250"} 33
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="500"} 33
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="1000"} 33
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="2000"} 33
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="4000"} 33
node_indexer_dune_client_send_block_batch_duration_millis_bucket{status="200",le="+Inf"} 33
node_indexer_dune_client_send_block_batch_duration_millis_sum{status="200"} 3326
node_indexer_dune_client_send_block_batch_duration_millis_count{status="200"} 33
# HELP node_indexer_dune_client_send_requests_total Number of send requests
# TYPE node_indexer_dune_client_send_requests_total counter
node_indexer_dune_client_send_requests_total{status="200"} 0
# HELP node_indexer_dune_client_sent_block_total Total number of blocks sent in requests
# TYPE node_indexer_dune_client_sent_block_total counter
node_indexer_dune_client_sent_block_total{status="200"} 33
# HELP node_indexer_ingested_block_number The highest block number ingested so far
# TYPE node_indexer_ingested_block_number gauge
node_indexer_ingested_block_number 638935
# HELP node_indexer_latest_block_number The latest known block number for the chain
# TYPE node_indexer_latest_block_number gauge
node_indexer_latest_block_number 4.443365e+06
# HELP node_indexer_rpc_client_request_duration_millis Duration of RPC node requests in milliseconds
# TYPE node_indexer_rpc_client_request_duration_millis histogram
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="10"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="25"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="50"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="100"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="250"} 152
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="500"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="1000"} 170
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="2000"} 170
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="4000"} 170
node_indexer_rpc_client_request_duration_millis_bucket{method="debug_traceBlockByNumber",status="200",le="+Inf"} 170
node_indexer_rpc_client_request_duration_millis_sum{method="debug_traceBlockByNumber",status="200"} 33528
node_indexer_rpc_client_request_duration_millis_count{method="debug_traceBlockByNumber",status="200"} 170
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="10"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="25"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="50"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="100"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="250"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="500"} 1
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="1000"} 2
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="2000"} 2
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="4000"} 2
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_blockNumber",status="200",le="+Inf"} 2
node_indexer_rpc_client_request_duration_millis_sum{method="eth_blockNumber",status="200"} 862
node_indexer_rpc_client_request_duration_millis_count{method="eth_blockNumber",status="200"} 2
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="10"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="25"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="50"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="100"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="250"} 153
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="500"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="1000"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="2000"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="4000"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockByNumber",status="200",le="+Inf"} 169
node_indexer_rpc_client_request_duration_millis_sum{method="eth_getBlockByNumber",status="200"} 32687
node_indexer_rpc_client_request_duration_millis_count{method="eth_getBlockByNumber",status="200"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="10"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="25"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="50"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="100"} 0
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="250"} 156
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="500"} 168
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="1000"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="2000"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="4000"} 169
node_indexer_rpc_client_request_duration_millis_bucket{method="eth_getBlockReceipts",status="200",le="+Inf"} 169
node_indexer_rpc_client_request_duration_millis_sum{method="eth_getBlockReceipts",status="200"} 32443
node_indexer_rpc_client_request_duration_millis_count{method="eth_getBlockReceipts",status="200"} 169
# HELP node_indexer_rpc_client_request_total Total number of RPC node requests
# TYPE node_indexer_rpc_client_request_total counter
node_indexer_rpc_client_request_total{method="debug_traceBlockByNumber",status="200"} 170
node_indexer_rpc_client_request_total{method="eth_blockNumber",status="200"} 2
node_indexer_rpc_client_request_total{method="eth_getBlockByNumber",status="200"} 169
node_indexer_rpc_client_request_total{method="eth_getBlockReceipts",status="200"} 169
# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
# TYPE process_cpu_seconds_total counter
process_cpu_seconds_total 0.53
# HELP process_max_fds Maximum number of open file descriptors.
# TYPE process_max_fds gauge
process_max_fds 1.048576e+06
# HELP process_open_fds Number of open file descriptors.
# TYPE process_open_fds gauge
process_open_fds 11
# HELP process_resident_memory_bytes Resident memory size in bytes.
# TYPE process_resident_memory_bytes gauge
process_resident_memory_bytes 4.9020928e+07
# HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
# TYPE process_start_time_seconds gauge
process_start_time_seconds 1.72174871716e+09
# HELP process_virtual_memory_bytes Virtual memory size in bytes.
# TYPE process_virtual_memory_bytes gauge
process_virtual_memory_bytes 1.268514816e+09
# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes.
# TYPE process_virtual_memory_max_bytes gauge
process_virtual_memory_max_bytes 1.8446744073709552e+19
# HELP promhttp_metric_handler_requests_in_flight Current number of scrapes being served.
# TYPE promhttp_metric_handler_requests_in_flight gauge
promhttp_metric_handler_requests_in_flight 1
# HELP promhttp_metric_handler_requests_total Total number of scrapes by HTTP status code.
# TYPE promhttp_metric_handler_requests_total counter
promhttp_metric_handler_requests_total{code="200"} 0
promhttp_metric_handler_requests_total{code="500"} 0
promhttp_metric_handler_requests_total{code="503"} 0

[adammilnesmith]

• Add prometheus metrics #62 👈
• main

This stack of pull requests is managed by Graphite. Learn more about stacking.

Join @adammilnesmith and the rest of your teammates on Graphite

[helanto]

LGTM! 🔥