prometheums metrics for promtool

test/heapwatch/metrics_aggs.py

@@ -164,7 +164,7 @@ def main():
                 mmax = max(rw)
                 mmin = min(rw)
                 print(f'{nick}: {metric_name}: count {len(rw)}, max {mmax}, min {mmin}, min-max {mmax - mmin}')
-                metric = Metric(metric_name, 0, MetricType.COUNTER)
+                metric = Metric(metric_name, 0, '', MetricType.COUNTER)
                 if metric.short_name() not in metric_names_nick_max_avg:
                     metric_names_nick_max_avg[metric.short_name()] = []
                 if args.avg_max_min:

test/heapwatch/metrics_gra.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019-2024 Algorand, Inc.
+# This file is part of go-algorand
+#
+# go-algorand is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# go-algorand is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with go-algorand.  If not, see <https://www.gnu.org/licenses/>.
+#
+###
+#
+# Convert metrics collected by heapWatch.py from prometheus format to prometheus + timestamp format.
+# See https://prometheus.io/docs/prometheus/latest/storage/#backfilling-from-openmetrics-format
+#
+# Usage:
+# python3 /data/go-algorand/test/heapwatch/metrics_gra.py -d metrics/500x15/ -o prom-metrics.txt
+#
+# Local Grafana setup:
+# 1. Download standalone and unpack from https://grafana.com/grafana/download
+# 2. Run ./grafana-v11.2.2/bin/grafana server -config ./grafana-v11.2.2/conf/defaults.ini -homepath ./grafana-v11.2.2
+# 3. Open http://localhost:3000/ in web browser
+#
+# Prometheus setup:
+# 1. Download and unpack from https://prometheus.io/download/
+#
+# Apply prom-metrics.txt to prometheus:
+# (cd ./prometheus-2.54.1.linux-amd64 && ./promtool tsdb create-blocks-from openmetrics prom-metrics.txt)
+# Start Prometheus
+# ./prometheus-2.54.1.linux-amd64/prometheus --config.file=./prometheus-2.54.1.linux-amd64/prometheus.yml --storage.tsdb.path=./prometheus-2.54.1.linux-amd64/data --storage.tsdb.retention.time=60d --storage.tsdb.retention.size=500MB
+# This should import the data into ./prometheus-2.54.1.linux-amd64/data and have them available for plotting. Use https://127.0.0.1:9090/ as Prometheus data source location in Grafana.
+# Then create new or import dashboards from internal Grafana.
+###
+
+import argparse
+import glob
+import logging
+import os
+import sys
+
+from metrics_lib import gather_metrics_files_by_nick, parse_metrics
+
+logger = logging.getLogger(__name__)
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('-d', '--dir', type=str, default=None, help='dir path to find /*.metrics in')
+    ap.add_argument('--nick-re', action='append', default=[], help='regexp to filter node names, may be repeated')
+    ap.add_argument('--nick-lre', action='append', default=[], help='label:regexp to filter node names, may be repeated')
+    ap.add_argument('-o', '--output', type=str, default=None, help='output file to write to')
+    ap.add_argument('--verbose', default=False, action='store_true')
+    args = ap.parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    if not args.dir:
+        logging.error('need at least one dir set with -d/--dir')
+        return 1
+
+    metrics_files = sorted(glob.glob(os.path.join(args.dir, '*.metrics')))
+    metrics_files.extend(glob.glob(os.path.join(args.dir, 'terraform-inventory.host')))
+    filesByNick = gather_metrics_files_by_nick(metrics_files, args.nick_re, args.nick_lre)
+
+    outf = sys.stdout
+    if args.output:
+        outf = open(args.output, 'wt')
+
+    for nick, files_by_ts in filesByNick.items():
+        for ts, metrics_file in files_by_ts.items():
+            with open(metrics_file, 'rt') as fin:
+                metrics = parse_metrics(fin, nick)
+                for metric_seq in metrics.values():
+                    for metric in metric_seq:
+                        print('# TYPE', metric.short_name(), metric.type, file=outf)
+                        print('# HELP', metric.short_name(), metric.desc, file=outf)
+                        print(metric.string(with_role=True, quote=True), metric.value, int(ts.timestamp()), file=outf)
+
+    print('# EOF', file=outf)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

test/heapwatch/metrics_lib.py

@@ -166,13 +166,17 @@ class MetricType(Enum):
     GAUGE = 0
     COUNTER = 1
 
+    def __str__(self):
+        return self.name.lower()
+
 class Metric:
     """Metric with tags"""
-    def __init__(self, metric_name: str, type: MetricType, value: Union[int, float]):
+    def __init__(self, metric_name: str, type: MetricType, desc: str, value: Union[int, float]):
         full_name = metric_name.strip()
         self.name = full_name
         self.value = value
         self.type = type
+        self.desc = desc
         self.tags: Dict[str, str] = {}
         self.tag_keys: set = set()
 
@@ -200,31 +204,20 @@ def short_name(self):
     def __str__(self):
         return self.string()
 
-    def string(self, tags: Optional[set[str]]=None):
+    def string(self, tags: Optional[set[str]]=None, with_role=False, quote=False) -> str:
         result = self.name
-        if self.tags:
-            if not tags:
-                tags = self.tags
-            result += '{' + ','.join([f'{k}={v}' for k, v in sorted(self.tags.items()) if k in tags]) + '}'
-        return result
 
-    def graphite_string(self, with_role=False):
-        restricted_chars = ('"', '$', '(', ')', '*', '+', ',', '?', '[', ']', '\\', '^', '`', '{', '}', '|', ' ')
-        translate_table = str.maketrans({c: '_' for c in restricted_chars})
-        result = self.name
         if with_role:
             node = self.tags.get('n')
             if node:
                 role = 'relay' if node.startswith('r') else 'npn' if node.startswith('npn') else 'node'
                 self.add_tag('role', role)
 
-        if self.tags:
-            tags = []
-            for k, v in sorted(self.tags.items()):
-                v = v.translate(translate_table)
-                tags.append(f'{k}={v}')
-            result += ';' + ';'.join(tags)
-            # result += ';' + ';'.join([f'{k}={v}' for k, v in sorted(self.tags.items())])
+        if self.tags or tags:
+            if not tags:
+                tags = self.tags
+            esc = '"' if quote else ''
+            result += '{' + ','.join([f'{k}={esc}{v}{esc}' for k, v in sorted(self.tags.items()) if k in tags]) + '}'
         return result
 
     def add_tag(self, key: str, value: str):
@@ -252,6 +245,7 @@ def parse_metrics(
     out = {}
     try:
         last_type = None
+        last_desc = None
         for line in fin:
             if not line:
                 continue
@@ -265,6 +259,8 @@ def parse_metrics(
                         last_type = MetricType.GAUGE
                     elif tpe == 'counter':
                         last_type = MetricType.COUNTER
+                elif line.startswith('# HELP'):
+                    last_desc = line.split(None, 3)[-1]  # skip first 3 words (#, HELP, metric name)
                 continue
             m = metric_line_re.match(line)
             if m:
@@ -275,7 +271,7 @@ def parse_metrics(
                 name = ab[0]
                 value = num(ab[1])
 
-            metric = Metric(name, last_type, value)
+            metric = Metric(name, last_type, last_desc, value)
             metric.add_tag('n', nick)
             if not metrics_names or metric.name in metrics_names:
                 if metric.name not in out:
@@ -288,7 +284,7 @@ def parse_metrics(
     if diff and metrics_names and len(metrics_names) == 2 and len(out) == 2:
         m = list(out.keys())
         name = f'{m[0]}_-_{m[1]}'
-        metric = Metric(name, MetricType.GAUGE, out[m[0]][0].value - out[m[1]][0].value)
+        metric = Metric(name, MetricType.GAUGE, f'Diff of {m[0]} and {m[1]}', out[m[0]][0].value - out[m[1]][0].value)
         out = {name: [metric]}
 
     return out