From 178b23944b6e5af270a8bc80c324f22a39fef260 Mon Sep 17 00:00:00 2001 From: derrandz Date: Wed, 15 Feb 2023 13:02:39 +0000 Subject: [PATCH] chore: renames and changes + historical sync 256 and fix dashboard --- build/docker-compose.yml | 15 +- .../grafana/dashboards/live-network-new.json | 2813 +++++++++++++++++ ...ive-network.json => live-network-old.json} | 1381 ++++++-- build/grafana/dashboards/main.yml | 12 + build/grafana/datasources/config.yml | 6 + build/otel-collector/config.yml | 8 +- build/prometheus/prometheus.yml | 11 +- .../1-8-200-default-getters.toml | 88 - .../64-square-size/1-8-200-shrex-only.toml | 88 - .../1-4-32-default-getters.toml | 42 +- .../128-square-size/1-4-32-ipld-only.toml} | 37 +- .../1-4-32-default-getters.toml | 111 + .../256-square-size}/1-4-32-ipld-only.toml | 35 +- .../1-4-32-default-getters.toml | 111 + .../64-square-size/1-4-32-ipld-only.toml | 42 +- .../1-3-32-default-getters.toml} | 12 +- .../128-square-size/1-3-32-ipld-only.toml} | 15 +- .../1-2-32-default-getters.toml} | 14 +- .../256-square-size/1-2-32-ipld-only.toml} | 15 +- .../1-4-32-default-getters.toml} | 23 +- .../64-square-size}/1-4-32-ipld-only.toml | 4 +- .../1-4-32-default-getters.toml | 90 + .../128-square-size/1-4-32-ipld-only.toml} | 24 +- .../1-4-32-default-getters.toml | 90 + .../256-square-size/1-4-32-ipld-only.toml} | 23 +- .../1-4-32-default-getters.toml} | 20 +- .../64-square-size}/1-4-32-ipld-only.toml | 18 +- go.mod | 5 +- go.sum | 10 +- main.go | 4 +- testkit/sync.go | 15 +- .../historical}/run_bridge.go | 45 +- .../helpers/blocksync/historical/run_full.go | 146 + .../historical/run_historical_full.go} | 38 +- .../historical}/run_validator.go | 72 +- .../latest-net-parition}/run_bridge.go | 2 +- .../latest-net-parition}/run_full.go | 16 +- .../latest-net-parition}/run_validator.go | 2 +- .../latest}/run_bridge.go | 2 +- .../latest}/run_full.go | 18 +- .../latest}/run_validator.go | 2 +- tests/helpers/common/bridge.go | 2 +- tests/plans/block-sync/block-sync.go | 49 +- 43 files changed, 4860 insertions(+), 716 deletions(-) create mode 100644 build/grafana/dashboards/live-network-new.json rename build/grafana/dashboards/{live-network.json => live-network-old.json} (64%) create mode 100644 build/grafana/dashboards/main.yml create mode 100644 build/grafana/datasources/config.yml delete mode 100644 compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-default-getters.toml delete mode 100644 compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-shrex-only.toml rename compositions/cluster-k8s/{blocksync-historical/64-square-size => blocksync/historical/128-square-size}/1-4-32-default-getters.toml (68%) rename compositions/cluster-k8s/{blocksync-historical/128-square-size/1-4-32-default-getters.toml => blocksync/historical/128-square-size/1-4-32-ipld-only.toml} (71%) create mode 100644 compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-default-getters.toml rename compositions/cluster-k8s/{blocksync-historical/128-square-size => blocksync/historical/256-square-size}/1-4-32-ipld-only.toml (72%) create mode 100644 compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-default-getters.toml rename compositions/cluster-k8s/{blocksync-historical => blocksync/historical}/64-square-size/1-4-32-ipld-only.toml (68%) rename compositions/cluster-k8s/{blocksync-latest-net-partition/128-square-size/1-4-32-default-getters.toml => blocksync/latest-net-partition/128-square-size/1-3-32-default-getters.toml} (93%) rename compositions/cluster-k8s/{blocksync-latest-net-partition/128-square-size/1-4-32-shrex-only.toml => blocksync/latest-net-partition/128-square-size/1-3-32-ipld-only.toml} (93%) rename compositions/cluster-k8s/{blocksync-latest-net-partition/64-square-size/1-4-32-default-getters.toml => blocksync/latest-net-partition/256-square-size/1-2-32-default-getters.toml} (92%) rename compositions/cluster-k8s/{blocksync-latest-net-partition/64-square-size/1-4-32-shrex-only.toml => blocksync/latest-net-partition/256-square-size/1-2-32-ipld-only.toml} (93%) rename compositions/cluster-k8s/{blocksync-latest-net-partition/64-square-size/1-4-32-ipld-only.toml => blocksync/latest-net-partition/64-square-size/1-4-32-default-getters.toml} (86%) rename compositions/cluster-k8s/{blocksync-latest-net-partition/128-square-size => blocksync/latest-net-partition/64-square-size}/1-4-32-ipld-only.toml (97%) create mode 100644 compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-default-getters.toml rename compositions/cluster-k8s/{blocksync-latest/128-square-size/1-4-32-default-getters.toml => blocksync/latest/128-square-size/1-4-32-ipld-only.toml} (88%) create mode 100644 compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-default-getters.toml rename compositions/cluster-k8s/{blocksync-latest/64-square-size/1-8-200-ipld-only.toml => blocksync/latest/256-square-size/1-4-32-ipld-only.toml} (88%) rename compositions/cluster-k8s/{blocksync-latest/128-square-size/1-4-32-shrex-only.toml => blocksync/latest/64-square-size/1-4-32-default-getters.toml} (90%) rename compositions/cluster-k8s/{blocksync-latest/128-square-size => blocksync/latest/64-square-size}/1-4-32-ipld-only.toml (90%) rename tests/helpers/{blocksyncbench-historical => blocksync/historical}/run_bridge.go (76%) create mode 100644 tests/helpers/blocksync/historical/run_full.go rename tests/helpers/{blocksyncbench-historical/run_full.go => blocksync/historical/run_historical_full.go} (85%) rename tests/helpers/{blocksyncbench-historical => blocksync/historical}/run_validator.go (69%) rename tests/helpers/{blocksyncbench-latest-net-parition => blocksync/latest-net-parition}/run_bridge.go (98%) rename tests/helpers/{blocksyncbench-latest-net-parition => blocksync/latest-net-parition}/run_full.go (94%) rename tests/helpers/{blocksyncbench-latest-net-parition => blocksync/latest-net-parition}/run_validator.go (98%) rename tests/helpers/{blocksyncbench-latest => blocksync/latest}/run_bridge.go (98%) rename tests/helpers/{blocksyncbench-latest => blocksync/latest}/run_full.go (92%) rename tests/helpers/{blocksyncbench-latest => blocksync/latest}/run_validator.go (99%) diff --git a/build/docker-compose.yml b/build/docker-compose.yml index a644a704..fc217a38 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -47,6 +47,19 @@ services: - "4317:4317" # OTLP gRPC receiver - "4318:4318" # OTLP http receiver + grafana: + container_name: grafana + image: bitnami/grafana:9.2.4 + ports: + - 3001:3000 + restart: unless-stopped + volumes: + - ${PWD}/grafana/datasources:/etc/grafana/provisioning/datasources + - ${PWD}/grafana/dashboards/:/etc/grafana/provisioning/dashboards/ + - ${PWD}/grafana/dashboards/:/var/lib/grafana/dashboards/ + - grafana-data:/var/lib/grafana + volumes: prometheus-data: - loki-data: \ No newline at end of file + loki-data: + grafana-data: \ No newline at end of file diff --git a/build/grafana/dashboards/live-network-new.json b/build/grafana/dashboards/live-network-new.json new file mode 100644 index 00000000..1b9c4ddf --- /dev/null +++ b/build/grafana/dashboards/live-network-new.json @@ -0,0 +1,2813 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 0 + }, + "id": 123156, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "count(count by (exported_instance) (testground_head))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "# Node Alive", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 2, + "y": 0 + }, + "id": 123167, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_total_sampled_headers{exported_instance=\"$node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Sampled Heights", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 4, + "y": 0 + }, + "id": 123165, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Height$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count_values(\"testground_das_sampled_chain_head\", testground_das_sampled_chain_head{exported_instance=\"$node_id\"})", + "format": "table", + "instant": true, + "interval": "5m", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Current Sample Head", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "testground_das_sampled_chain_head": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 0, + "testground_das_sampled_chain_head": 1 + }, + "renameByName": { + "Value": "Count", + "testground_das_sampled_chain_head": "Height" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Count": { + "aggregations": [], + "operation": "groupby" + }, + "Height": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Height" + } + ] + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 6, + "y": 0 + }, + "id": 123163, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(testground_head)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Network Head", + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Time" + } + ] + } + }, + { + "id": "reduce", + "options": { + "reducers": [ + "max" + ] + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 18, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 8, + "y": 0 + }, + "id": 123145, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "vertical", + "showValue": "always", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count_values(\"testground_das_sampled_chain_head\", testground_das_sampled_chain_head)", + "format": "table", + "instant": true, + "interval": "5m", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "# nodes / height", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "testground_das_sampled_chain_head": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 0, + "testground_das_sampled_chain_head": 1 + }, + "renameByName": { + "Value": "Count", + "testground_das_sampled_chain_head": "Height" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Count": { + "aggregations": [], + "operation": "groupby" + }, + "Height": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Height" + } + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 18, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 17, + "y": 0 + }, + "id": 123173, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "orientation": "vertical", + "showValue": "always", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count_values(\"testground_das_network_head\", testground_das_network_head)", + "format": "table", + "instant": true, + "interval": "5m", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "# nodes / das_network_head", + "transformations": [ + { + "disabled": true, + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "testground_das_sampled_chain_head": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "disabled": true, + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 0, + "testground_das_sampled_chain_head": 1 + }, + "renameByName": { + "Value": "Count", + "testground_das_sampled_chain_head": "Height" + } + } + }, + { + "disabled": true, + "id": "groupBy", + "options": { + "fields": { + "Count": { + "aggregations": [], + "operation": "groupby" + }, + "Height": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "disabled": true, + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Height" + } + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1000 + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 123187, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "rate (testground_das_sample_time_hist_sum{failed=\"false\", header_width=~\"$square_size\"}[5m])/rate(testground_das_sample_time_hist_count{failed=\"false\", header_width=~\"$square_size\"}[5m])", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_sample_time_hist_sum{failed=\"false\", header_width=\"$square_size\"}/testground_das_sample_time_hist_count{failed=\"false\", header_width=~\"$square_size\"}", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "(Network) Avg DASing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 17 + }, + "id": 123190, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(sum by (height) (testground_das_sample_time_hist_bucket{header_width=~\"$square_size\", le=\"0.005\"}))/(sum by (height) (testground_das_sample_time_hist_count{header_width=~\"$square_size\"}))", + "format": "heatmap", + "hide": true, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "sum(rate(testground_das_sample_time_hist_bucket{le=\"1\"}[5m])) by (height)\n/\nsum(rate(testground_das_sample_time_hist_count[5m])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "% of DAS times < 5s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 7, + "x": 5, + "y": 17 + }, + "id": 123189, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(sum(rate(testground_das_sample_time_hist_bucket{le=\"+Inf\", header_width=~\"$square_size\"}[5m])) by (height) - sum(rate(testground_das_sample_time_hist_bucket{le=\"10\", header_width=~\"$square_size\"}[5m])) by (height))\n/ sum(rate(testground_das_sample_time_hist_count{header_width=\"$square_size\"}[5m])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "% of DAS times > 10s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 17 + }, + "id": 123184, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(sum by (height) (testground_das_sample_time_hist_bucket{header_width=~\"$square_size\", le=\"0.005\"}))/(sum by (height) (testground_das_sample_time_hist_count{header_width=~\"$square_size\"}))", + "format": "heatmap", + "hide": true, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "sum(rate(testground_das_sample_time_hist_bucket{le=\"5\"}[5m])) by (height)\n/\nsum(rate(testground_das_sample_time_hist_count[5m])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "% of DAS times < 5s", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 3, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 17 + }, + "id": 123188, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(sum(rate(testground_das_sample_time_hist_bucket{le=\"10\", header_width=~\"$square_size\"}[5m])) by (height) - sum(rate(testground_das_sample_time_hist_bucket{le=\"5\", header_width=~\"$square_size\"}[5m])) by (height))\n/ sum(rate(testground_das_sample_time_hist_count[5m])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "% of DAS times in [5s, 10s]", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 123137, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate (testground_das_sample_time_hist_sum{exported_instance=\"$node_id\", header_width=~\"$square_size\"}[5m])/rate(testground_das_sample_time_hist_count{exported_instance=\"$node_id\", header_width=~\"$square_size\"}[5m])", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_sample_time_hist_sum{exported_instance=\"$node_id\", failed=\"false\", header_width=\"$square_size\"}/testground_das_sample_time_hist_count{exported_instance=\"$node_id\", failed=\"false\", header_width=\"$square_size\"}", + "format": "table", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "(Node) Avg DASing time", + "transformations": [ + { + "disabled": true, + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "header": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 31 + }, + "id": 123175, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_peer_count{exported_instance=\"$light_node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "# of Peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "series", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "right", + "axisSoftMax": 30, + "axisSoftMin": 0.002, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 31 + }, + "id": 123142, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "colorByField": "exported_instance", + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "orientation": "auto", + "showValue": "always", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "header", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by (header, exported_instance) (\n (rate (testground_das_sample_time_hist_sum{failed=\"false\", header_width=\"$square_size\"}[5m]) != 0)\n /\n rate(testground_das_sample_time_hist_count{failed=\"false\", header_width=\"$square_size\"}[5m])\n)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "(Network) DAS durations (s)", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "exported_instance": { + "aggregations": [], + "operation": "groupby" + }, + "header": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "header" + } + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 38 + }, + "id": 123152, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(testground_node_share_blackbox_eds_size_sum)/2", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Square Size", + "transformations": [ + { + "id": "groupBy", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 3, + "y": 38 + }, + "id": 123153, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "testground_node_share_blackbox_eds_size_sum{}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "EDS Size", + "transformations": [ + { + "id": "groupBy", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 18, + "x": 6, + "y": 38 + }, + "id": 123169, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(testground_das_total_sampled_headers{exported_instance=\"$node_id\"}/testground_das_network_head{exported_instance=\"$node_id\"})", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_node_runtime_counter_in_seconds{exported_instance=\"$node_id\"}/(time() - testground_node_start_ts{exported_instance=\"$node_id\"})", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Uptime", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 123171, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Value #das_sampled_chain_head" + } + ] + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "testground_das_sampled_chain_head", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "das_sampled_chain_head" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_network_head", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "das_network_head" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_total_synced_headers", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "total_synced_headers" + } + ], + "title": "Peer ID / height", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "exported_instance", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "exported_job", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "job", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "instance", + "mode": "outer" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Time": { + "aggregations": [] + }, + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "exported_instance": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Total Synced Headers", + "exported_instance": "PeerID" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 52 + }, + "id": 123133, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(rate(testground_node_header_blackbox_request_duration_sum{exported_instance=\"$node_id\"}[$__range])/rate(testground_node_header_blackbox_request_duration_count{exported_instance=\"$node_id\"}[$__range]))/1000", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Avg Header Transfer Rate (bytes/s)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 52 + }, + "id": 123131, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(rate(testground_node_share_blackbox_request_duration_sum{exported_instance=\"$node_id\"}[5m])/rate(testground_node_share_blackbox_request_duration_count{exported_instance=\"$node_id\"}[5m]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Avg Share Transfer Rate (bytes/s)", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 52 + }, + "id": 123148, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_bandwidth_rate_outbound_sum{exported_instance=\"$node_id\"}/testground_p2p_bandwidth_rate_outbound_count{exported_instance=\"$node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Avg OUT BW Rate (bytes/s)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 52 + }, + "id": 123150, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_bandwidth_rate_inbound_sum{exported_instance=\"$node_id\"}/testground_p2p_bandwidth_rate_inbound_count{exported_instance=\"$node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Avg IN BW Rate (bytes/s)", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 123149, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_bandwidth_total_outbound_sum{exported_instance=\"$node_id\"}/testground_p2p_bandwidth_total_outbound_count{exported_instance=\"$node_id\"}", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total OUT BW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 123159, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_bandwidth_total_inbound_sum{exported_instance=\"$node_id\"}/testground_p2p_bandwidth_total_inbound_count{exported_instance=\"$node_id\"}", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total IN BW", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 123147, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_total_outbound_by_peer_sum{exported_instance=\"$node_id\"}/testground_p2p_total_outbound_by_peer_count{exported_instance=\"$node_id\"}", + "format": "time_series", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total OUT BW by Peer", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 123160, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_total_inbound_by_peer_sum{exported_instance=\"$node_id\"}/testground_p2p_total_inbound_by_peer_count{exported_instance=\"$node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total IN BW by Peer", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 123127, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "avg by (height) (\n rate (testground_node_header_blackbox_block_time_sum{exported_instance=\"$node_id\"}[5m])\n /\n rate(testground_node_header_blackbox_block_time_count{exported_instance=\"$node_id\"}[5m])\n)", + "format": "table", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Block time per Height", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "height": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "barchart" + } + ], + "refresh": "10s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "12D3KooWEPHV53PoGN38vx44mu1pUCGavtyCxcji9euZLTzM6H4o", + "value": "12D3KooWEPHV53PoGN38vx44mu1pUCGavtyCxcji9euZLTzM6H4o" + }, + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "definition": "label_values(testground_das_network_head, exported_instance)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "node_id", + "options": [], + "query": { + "query": "label_values(testground_das_network_head, exported_instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "ce2m15p6tspcfpfpg2m0", + "value": "ce2m15p6tspcfpfpg2m0" + }, + "datasource": { + "type": "influxdb", + "uid": "CIHiCGKVk" + }, + "definition": "show tag values with key = \"run\"", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "run", + "options": [], + "query": "show tag values with key = \"run\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "definition": "label_values(testground_das_sample_time_hist_bucket, header_width)", + "hide": 0, + "includeAll": true, + "label": "square_size", + "multi": true, + "name": "square_size", + "options": [], + "query": { + "query": "label_values(testground_das_sample_time_hist_bucket, header_width)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "prom_ds", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Live Network Light Nodes Metrics -", + "uid": "Fccs8xxTacVj", + "version": 78, + "weekStart": "" +} \ No newline at end of file diff --git a/build/grafana/dashboards/live-network.json b/build/grafana/dashboards/live-network-old.json similarity index 64% rename from build/grafana/dashboards/live-network.json rename to build/grafana/dashboards/live-network-old.json index 2f97d993..75f81609 100644 --- a/build/grafana/dashboards/live-network.json +++ b/build/grafana/dashboards/live-network-old.json @@ -26,7 +26,7 @@ "graphTooltip": 0, "id": 7, "links": [], - "liveNow": false, + "liveNow": true, "panels": [ { "datasource": { @@ -80,13 +80,13 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "expr": "count(count by (exported_instance) (testground_das_network_head))", + "expr": "count(count by (exported_instance) (testground_head))", "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "# Light Node Alive", + "title": "# Node Alive", "type": "stat" }, { @@ -122,6 +122,199 @@ "x": 2, "y": 0 }, + "id": 123167, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_total_sampled_headers{exported_instance=\"$node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Sampled Heights", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 4, + "y": 0 + }, + "id": 123165, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Height$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count_values(\"testground_das_sampled_chain_head\", testground_das_sampled_chain_head{exported_instance=\"$node_id\"})", + "format": "table", + "instant": true, + "interval": "5m", + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Current Sample Head", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "testground_das_sampled_chain_head": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value": 0, + "testground_das_sampled_chain_head": 1 + }, + "renameByName": { + "Value": "Count", + "testground_das_sampled_chain_head": "Height" + } + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Count": { + "aggregations": [], + "operation": "groupby" + }, + "Height": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "Height" + } + ] + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 6, + "y": 0 + }, "id": 123163, "options": { "colorMode": "value", @@ -146,7 +339,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "testground_head", + "expr": "max(testground_head)", "format": "table", "instant": true, "legendFormat": "__auto", @@ -225,8 +418,8 @@ }, "gridPos": { "h": 7, - "w": 11, - "x": 4, + "w": 9, + "x": 8, "y": 0 }, "id": 123145, @@ -344,8 +537,8 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", + "fillOpacity": 18, + "gradientMode": "opacity", "hideFrom": { "legend": false, "tooltip": false, @@ -370,17 +563,17 @@ } ] }, - "unit": "short" + "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 9, - "x": 15, + "w": 7, + "x": 17, "y": 0 }, - "id": 123144, + "id": 123173, "options": { "barRadius": 0, "barWidth": 0.97, @@ -391,8 +584,8 @@ "placement": "right", "showLegend": true }, - "orientation": "horizontal", - "showValue": "never", + "orientation": "vertical", + "showValue": "always", "stacking": "normal", "tooltip": { "mode": "single", @@ -410,7 +603,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count_values(\"testground_das_sampled_chain_head\", testground_das_sampled_chain_head)", + "expr": "count_values(\"testground_das_network_head\", testground_das_network_head)", "format": "table", "instant": true, "interval": "5m", @@ -419,9 +612,10 @@ "refId": "A" } ], - "title": "# nodes per height", + "title": "# nodes / das_network_head", "transformations": [ { + "disabled": true, "id": "groupBy", "options": { "fields": { @@ -437,10 +631,14 @@ } }, { + "disabled": true, "id": "organize", "options": { "excludeByName": {}, - "indexByName": {}, + "indexByName": { + "Value": 0, + "testground_das_sampled_chain_head": 1 + }, "renameByName": { "Value": "Count", "testground_das_sampled_chain_head": "Height" @@ -448,6 +646,7 @@ } }, { + "disabled": true, "id": "groupBy", "options": { "fields": { @@ -463,29 +662,14 @@ } }, { - "id": "calculateField", - "options": { - "alias": "Percentage", - "binary": { - "left": "Count", - "operator": "/", - "reducer": "sum", - "right": "500" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - } - } - }, - { + "disabled": true, "id": "sortBy", "options": { "fields": {}, "sort": [ { - "desc": true, - "field": "Count" + "desc": false, + "field": "Height" } ] } @@ -500,6 +684,9 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", @@ -507,10 +694,14 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, @@ -520,10 +711,10 @@ "x": 0, "y": 7 }, - "id": 123153, + "id": 123137, "options": { - "colorMode": "background", - "graphMode": "none", + "colorMode": "value", + "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { @@ -544,19 +735,44 @@ }, "editorMode": "code", "exemplar": false, - "expr": "testground_node_share_blackbox_eds_size_sum", + "expr": "(rate (testground_das_sample_time_hist_sum{exported_instance=\"$node_id\", failed=\"false\", header_width=~\"$square_size\"}[$__rate_interval]))/rate(testground_das_sample_time_hist_count{exported_instance=\"$node_id\", failed=\"false\", header_width=~\"$square_size\"}[$__rate_interval])", "format": "table", + "hide": false, "instant": true, "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_sample_time_hist_sum{exported_instance=\"$node_id\", failed=\"false\", header_width=\"$square_size\"}/testground_das_sample_time_hist_count{exported_instance=\"$node_id\", failed=\"false\", header_width=\"$square_size\"}", + "format": "table", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "title": "EDS Size", + "title": "(Node) Avg DASing time", "transformations": [ { "id": "groupBy", - "options": {} + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "header": { + "aggregations": [], + "operation": "groupby" + } + } + } } ], "type": "stat" @@ -568,6 +784,9 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, "mappings": [], "thresholds": { "mode": "absolute", @@ -575,10 +794,18 @@ { "color": "green", "value": null + }, + { + "color": "orange", + "value": 1000 + }, + { + "color": "red", + "value": 2000 } ] }, - "unit": "short" + "unit": "s" }, "overrides": [] }, @@ -588,11 +815,8 @@ "x": 3, "y": 7 }, - "id": 123152, + "id": 123183, "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -601,7 +825,8 @@ "fields": "", "values": false }, - "textMode": "auto" + "showThresholdLabels": false, + "showThresholdMarkers": true }, "pluginVersion": "9.2.4", "targets": [ @@ -611,23 +836,171 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "exemplar": false, - "expr": "(testground_node_share_blackbox_eds_size_sum)/2", - "format": "table", - "instant": true, + "expr": "max(rate (testground_das_sample_time_hist_sum{failed=\"false\", header_width=~\"$square_size\"}[$__rate_interval])/rate(testground_das_sample_time_hist_count{failed=\"false\", header_width=~\"$square_size\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "interval": "", "legendFormat": "__auto", - "range": false, + "range": true, "refId": "A" } ], - "title": "Square Size", - "transformations": [ + "title": "(Network) Max Avg DASing time", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1000 + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 6, + "y": 7 + }, + "id": 123187, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ { - "id": "groupBy", - "options": {} + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "avg by (header_width) ((rate (testground_das_sample_time_hist_sum{failed=\"false\", header_width=~\"$square_size\"}[$__rate_interval]) != 0)/rate(testground_das_sample_time_hist_count{failed=\"false\", header_width=~\"$square_size\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_sample_time_hist_sum{failed=\"false\", header_width=\"$square_size\"}/testground_das_sample_time_hist_count{failed=\"false\", header_width=~\"$square_size\"}", + "hide": true, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "type": "stat" + "title": "(Network) Avg DASing time", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 1000 + }, + { + "color": "red", + "value": 2000 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 7 + }, + "id": 123186, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "min(rate (testground_das_sample_time_hist_sum{failed=\"false\", header_width=\"$square_size\"}[$__rate_interval])/rate(testground_das_sample_time_hist_count{failed=\"false\", header_width=\"$square_size\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "interval": "", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "(Network) Min Avg DASing time", + "type": "gauge" }, { "datasource": { @@ -639,7 +1012,6 @@ "color": { "mode": "thresholds" }, - "displayName": "% of DASing Requests =< 500 (ms)", "mappings": [], "thresholds": { "mode": "absolute", @@ -661,10 +1033,10 @@ "gridPos": { "h": 7, "w": 3, - "x": 6, + "x": 12, "y": 7 }, - "id": 123129, + "id": 123185, "options": { "colorMode": "value", "graphMode": "area", @@ -687,15 +1059,28 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "expr": "(sum by (height) (testground_das_sample_time_hist_bucket{exported_instance=\"$light_node_id\", le=\"0.005\"}))/(sum by (height) (testground_das_sample_time_hist_count{exported_instance=\"$light_node_id\"}))", + "expr": "(sum by (height) (testground_das_sample_time_hist_bucket{exported_instance=\"node_id\", le=\"0.005\"}))/(sum by (height) (testground_das_sample_time_hist_count{exported_instance=\"node_id\", header_width=\"$square_size\"}))", "format": "heatmap", + "hide": true, "interval": "", "legendFormat": "{{le}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "sum(rate(testground_das_sample_time_hist_bucket{le=\"1\"}[$__rate_interval])) by (height)\n/\nsum(rate(testground_das_sample_time_hist_count[$__rate_interval])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "title": "% of DAS Requests =< 500 (ms)", + "title": "% of DAS times < 1 s", "type": "stat" }, { @@ -708,7 +1093,6 @@ "color": { "mode": "thresholds" }, - "displayName": "% of DASing Requests > 1s", "mappings": [], "thresholds": { "mode": "absolute", @@ -730,10 +1114,10 @@ "gridPos": { "h": 7, "w": 3, - "x": 9, + "x": 15, "y": 7 }, - "id": 123134, + "id": 123189, "options": { "colorMode": "value", "graphMode": "area", @@ -756,15 +1140,14 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "expr": "(sum by (height) (testground_das_sample_time_hist_count{exported_instance=\"$light_node_id\"})) - (sum by (height) (testground_das_sample_time_hist_bucket{exported_instance=\"$light_node_id\", le=\"1\"}))", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", + "expr": "(sum(rate(testground_das_sample_time_hist_bucket{le=\"+inf\"}[$__rate_interval])) by (height) - sum(rate(testground_das_sample_time_hist_bucket{le=\"10\"}[$__rate_interval])) by (height))\n/ sum(rate(testground_das_sample_time_hist_count[$__rate_interval])) by (height)", + "hide": false, + "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "% of DAS Requests > 1s", + "title": "% of DAS times > 10s", "type": "stat" }, { @@ -785,28 +1168,27 @@ "color": "green", "value": null }, - { - "color": "orange", - "value": 1000 - }, { "color": "red", - "value": 2000 + "value": 80 } ] }, - "unit": "ms" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 3, - "x": 12, + "x": 18, "y": 7 }, - "id": 123161, + "id": 123184, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -815,8 +1197,7 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "textMode": "auto" }, "pluginVersion": "9.2.4", "targets": [ @@ -826,16 +1207,29 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "expr": "avg by (height) (testground_das_sample_time_hist_sum{exported_instance=\"$light_node_id\"}/testground_das_sample_time_hist_count{exported_instance=\"$light_node_id\"})", + "expr": "(sum by (height) (testground_das_sample_time_hist_bucket{exported_instance=\"node_id\", le=\"0.005\"}))/(sum by (height) (testground_das_sample_time_hist_count{exported_instance=\"node_id\", header_width=\"$square_size\"}))", "format": "heatmap", + "hide": true, "interval": "", - "legendFormat": "__auto", + "legendFormat": "{{le}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "sum(rate(testground_das_sample_time_hist_bucket{le=\"5\"}[$__rate_interval])) by (height)\n/\nsum(rate(testground_das_sample_time_hist_count[$__rate_interval])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "title": "Avg DASing time (ms)", - "type": "gauge" + "title": "% of DAS times < 5s", + "type": "stat" }, { "datasource": { @@ -845,24 +1239,7 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1, - "scaleDistribution": { - "type": "linear" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -878,37 +1255,432 @@ } ] }, - "unit": "short" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, - "w": 9, - "x": 15, + "w": 3, + "x": 21, "y": 7 }, - "id": 123137, + "id": 123188, "options": { - "barRadius": 0, - "barWidth": 0.97, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "orientation": "auto", - "showValue": "always", - "stacking": "none", + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "(sum(rate(testground_das_sample_time_hist_bucket{le=\"10\"}[$__rate_interval])) by (height) - sum(rate(testground_das_sample_time_hist_bucket{le=\"5\"}[5m])) by (height))\n/ sum(rate(testground_das_sample_time_hist_count[$__rate_interval])) by (height)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "% of DAS times in [5s, 10s]", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 14 + }, + "id": 123175, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_p2p_peer_count{exported_instance=\"$light_node_id\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "# of Peers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 3, + "y": 14 + }, + "id": 123153, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "testground_node_share_blackbox_eds_size_sum{}", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "EDS Size", + "transformations": [ + { + "id": "groupBy", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 6, + "y": 14 + }, + "id": 123152, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(testground_node_share_blackbox_eds_size_sum)/2", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Square Size", + "transformations": [ + { + "id": "groupBy", + "options": {} + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "series", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "right", + "axisSoftMax": 30, + "axisSoftMin": 0.002, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 15, + "x": 9, + "y": 14 + }, + "id": 123142, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "colorByField": "exported_instance", + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "orientation": "auto", + "showValue": "always", + "stacking": "normal", "tooltip": { "mode": "single", "sort": "none" }, + "xField": "header", "xTickLabelRotation": 0, - "xTickLabelSpacing": 0 + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by (header, exported_instance) (\n (rate (testground_das_sample_time_hist_sum{failed=\"false\", header_width=\"$square_size\"}[5m]) != 0)\n /\n rate(testground_das_sample_time_hist_count{failed=\"false\", header_width=\"$square_size\"}[5m])\n)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "(Network) DAS durations (s)", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": [], + "operation": "groupby" + }, + "exported_instance": { + "aggregations": [], + "operation": "groupby" + }, + "header": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "desc": false, + "field": "header" + } + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 123171, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "Value #das_sampled_chain_head" + } + ] }, + "pluginVersion": "9.2.4", "targets": [ { "datasource": { @@ -917,33 +1689,110 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg by (header) (testground_das_sample_time_hist_sum{exported_instance=\"$light_node_id\"})", + "expr": "testground_das_sampled_chain_head", "format": "table", + "hide": false, "instant": true, + "interval": "", + "legendFormat": "__auto", + "range": false, + "refId": "das_sampled_chain_head" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_das_network_head", + "format": "table", + "hide": false, "legendFormat": "__auto", "range": true, - "refId": "A" + "refId": "das_network_head" + }, + { + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_total_synced_headers", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "total_synced_headers" } ], - "title": "DASing time per Height (current instance)", + "title": "# nodes / height", "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "exported_instance", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "exported_job", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "job", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "joinByField", + "options": { + "byField": "instance", + "mode": "outer" + } + }, { "id": "groupBy", "options": { "fields": { + "Time": { + "aggregations": [] + }, "Value": { "aggregations": [], "operation": "groupby" }, - "header": { + "exported_instance": { "aggregations": [], "operation": "groupby" } } } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "Value": "Total Synced Headers", + "exported_instance": "PeerID" + } + } } ], - "type": "barchart" + "type": "table" }, { "datasource": { @@ -960,16 +1809,29 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "fillOpacity": 80, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, "mappings": [], @@ -977,47 +1839,35 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - }, - "unit": "none" + } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 10, "w": 24, "x": 0, - "y": 14 + "y": 28 }, - "id": 123142, + "id": 123169, "options": { - "barRadius": 0, - "barWidth": 0.97, - "colorByField": "exported_instance", - "groupWidth": 0.7, "legend": { "calcs": [], "displayMode": "list", - "placement": "right", + "placement": "bottom", "showLegend": true }, - "orientation": "auto", - "showValue": "always", - "stacking": "normal", "tooltip": { "mode": "single", "sort": "none" - }, - "xField": "header", - "xTickLabelRotation": 0, - "xTickLabelSpacing": 100 + } }, "targets": [ { @@ -1026,50 +1876,27 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "exemplar": false, - "expr": "avg by (header, exported_instance) (testground_das_sample_time_hist_sum)", - "format": "table", - "instant": true, + "expr": "(testground_das_total_sampled_headers{exported_instance=\"$light_node_id\"}/testground_das_network_head{exported_instance=\"$light_node_id\"})", + "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" - } - ], - "title": "DASing time overview (all instances)", - "transformations": [ - { - "id": "groupBy", - "options": { - "fields": { - "Value": { - "aggregations": [], - "operation": "groupby" - }, - "exported_instance": { - "aggregations": [], - "operation": "groupby" - }, - "header": { - "aggregations": [], - "operation": "groupby" - } - } - } }, { - "id": "sortBy", - "options": { - "fields": {}, - "sort": [ - { - "desc": false, - "field": "header" - } - ] - } + "datasource": { + "type": "prometheus", + "uid": "lW0MKED4k" + }, + "editorMode": "code", + "expr": "testground_node_runtime_counter_in_seconds{exported_instance=\"$light_node_id\"}/(time() - testground_node_start_ts{exported_instance=\"$light_node_id\"})", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" } ], - "type": "barchart" + "title": "Uptime", + "type": "timeseries" }, { "datasource": { @@ -1087,8 +1914,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1104,7 +1930,7 @@ "h": 6, "w": 6, "x": 0, - "y": 22 + "y": 38 }, "id": 123133, "options": { @@ -1151,8 +1977,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1168,7 +1993,7 @@ "h": 6, "w": 6, "x": 6, - "y": 22 + "y": 38 }, "id": 123131, "options": { @@ -1215,8 +2040,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1228,7 +2052,7 @@ "h": 6, "w": 6, "x": 12, - "y": 22 + "y": 38 }, "id": 123148, "options": { @@ -1277,8 +2101,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -1290,7 +2113,7 @@ "h": 6, "w": 6, "x": 18, - "y": 22 + "y": 38 }, "id": 123150, "options": { @@ -1369,8 +2192,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1386,7 +2208,7 @@ "h": 9, "w": 12, "x": 0, - "y": 28 + "y": 44 }, "id": 123149, "options": { @@ -1463,8 +2285,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1480,7 +2301,7 @@ "h": 9, "w": 12, "x": 12, - "y": 28 + "y": 44 }, "id": 123159, "options": { @@ -1512,67 +2333,6 @@ "title": "Total IN BW", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 3, - "x": 0, - "y": 37 - }, - "id": 123158, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "9.2.4", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "lW0MKED4k" - }, - "editorMode": "code", - "expr": "count(count by (peer_id) (testground_p2p_total_inbound_by_peer_sum{exported_instance=\"$light_node_id\"}))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "# Peers Of Current Instance", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -1618,8 +2378,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1633,9 +2392,9 @@ }, "gridPos": { "h": 8, - "w": 11, - "x": 3, - "y": 37 + "w": 12, + "x": 0, + "y": 53 }, "id": 123147, "options": { @@ -1714,8 +2473,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1729,9 +2487,9 @@ }, "gridPos": { "h": 8, - "w": 10, - "x": 14, - "y": 37 + "w": 12, + "x": 12, + "y": 53 }, "id": 123160, "options": { @@ -1794,8 +2552,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1810,7 +2567,7 @@ "h": 9, "w": 24, "x": 0, - "y": 45 + "y": 61 }, "id": 123127, "options": { @@ -1840,7 +2597,7 @@ "uid": "lW0MKED4k" }, "editorMode": "code", - "expr": "(avg by (height) (testground_node_header_blackbox_block_time_sum{exported_instance=\"$light_node_id\"}))/1000", + "expr": "avg by (height) (\n rate (testground_node_header_blackbox_block_time_sum{exported_instance=\"$node_id\"}[5m])\n /\n rate(testground_node_header_blackbox_block_time_count{exported_instance=\"$node_id\"}[5m])\n)", "format": "table", "legendFormat": "__auto", "range": true, @@ -1868,6 +2625,7 @@ "type": "barchart" } ], + "refresh": "10s", "schemaVersion": 37, "style": "dark", "tags": [], @@ -1875,22 +2633,23 @@ "list": [ { "current": { + "isNone": true, "selected": false, - "text": "12D3KooWF9PnoopchmXBC7RyRFDQyJXRbP218Fidoa3qR8RWqdhR", - "value": "12D3KooWF9PnoopchmXBC7RyRFDQyJXRbP218Fidoa3qR8RWqdhR" + "text": "None", + "value": "" }, "datasource": { "type": "prometheus", "uid": "lW0MKED4k" }, - "definition": "label_values(testground_das_sample_time_hist_sum, exported_instance)", + "definition": "label_values(testground_das_network_head, exported_instance)", "hide": 0, "includeAll": false, "multi": false, - "name": "light_node_id", + "name": "node_id", "options": [], "query": { - "query": "label_values(testground_das_sample_time_hist_sum, exported_instance)", + "query": "label_values(testground_das_network_head, exported_instance)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1900,7 +2659,11 @@ "type": "query" }, { - "current": {}, + "current": { + "selected": false, + "text": "ce2m15p6tspcfpfpg2m0", + "value": "ce2m15p6tspcfpfpg2m0" + }, "datasource": { "type": "influxdb", "uid": "CIHiCGKVk" @@ -1917,6 +2680,52 @@ "skipUrlSync": false, "sort": 0, "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(testground_das_sample_time_hist_sum, header_width)", + "hide": 0, + "includeAll": true, + "label": "square_size", + "multi": true, + "name": "square_size", + "options": [], + "query": { + "query": "label_values(testground_das_sample_time_hist_sum, header_width)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "", + "multi": false, + "name": "prom_ds", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" } ] }, @@ -1925,7 +2734,7 @@ "to": "now" }, "timepicker": { - "hidden": true, + "hidden": false, "refresh_intervals": [ "5s", "10s", @@ -1952,8 +2761,8 @@ "type": "timepicker" }, "timezone": "browser", - "title": "Live Network Light Nodes Metrics -", + "title": "(OLD) Live Network Light Nodes Metrics -", "uid": "Fccs8xxTacVj", - "version": 17, + "version": 68, "weekStart": "" } \ No newline at end of file diff --git a/build/grafana/dashboards/main.yml b/build/grafana/dashboards/main.yml new file mode 100644 index 00000000..57d37973 --- /dev/null +++ b/build/grafana/dashboards/main.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "Dashboard provider" + orgId: 1 + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true \ No newline at end of file diff --git a/build/grafana/datasources/config.yml b/build/grafana/datasources/config.yml new file mode 100644 index 00000000..30b84c37 --- /dev/null +++ b/build/grafana/datasources/config.yml @@ -0,0 +1,6 @@ +datasources: + - name: Prometheus + access: proxy + type: prometheus + url: http://0.0.0.0:9000 + isDefault: true \ No newline at end of file diff --git a/build/otel-collector/config.yml b/build/otel-collector/config.yml index 9eca6b7d..83be5331 100644 --- a/build/otel-collector/config.yml +++ b/build/otel-collector/config.yml @@ -10,21 +10,15 @@ receivers: # endpoint: "0.0.0.0:4318" exporters: - logging: - logLevel: debug prometheus: endpoint: "otel-collector:8889" namespace: "testground" send_timestamps: true - metric_expiration: 30m - -processors: - batch: + metric_expiration: 1800m service: extensions: [health_check] pipelines: metrics: receivers: [otlp] - processors: [batch] exporters: [prometheus] \ No newline at end of file diff --git a/build/prometheus/prometheus.yml b/build/prometheus/prometheus.yml index 1a140653..79c62711 100644 --- a/build/prometheus/prometheus.yml +++ b/build/prometheus/prometheus.yml @@ -1,10 +1,15 @@ global: - scrape_interval: 5s - scrape_timeout: 3s + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s scrape_configs: - job_name: 'collector' metrics_path: /metrics + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + scheme: http static_configs: - targets: - - 'otel-collector:8889' \ No newline at end of file + - 'otel-collector:8889' diff --git a/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-default-getters.toml b/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-default-getters.toml deleted file mode 100644 index c9e50190..00000000 --- a/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-default-getters.toml +++ /dev/null @@ -1,88 +0,0 @@ -[metadata] - name = "blocksyncbench-latest" - author = "derrandz" - -[global] - plan = "celestia" - case = "blocksyncbench-latest" - total_instances = 209 - builder = "docker:generic" - runner = "cluster:k8s" - disable_metrics = false - -# we define 1 validator that produces 8mb blocks -# the msg-size does not result in exactly 8mb -# because we are accounting for padding -[global.run.test_params] - execution-time = "20" - persistent-peers = "1" - submit-times = "10" - msg-size = "600000" - validator = "1" - bridge = "8" - full = "200" - block-height = "30" - otel-collector-address = "178.128.163.171:4318" - -[[groups]] - id = "validators" - builder = "docker:generic" - [groups.resources] - memory = "4Gi" - cpu = "1" - [groups.instances] - count = 1 - percentage = 0.0 - [groups.build_config] - build_base_image = "golang:1.19.1" - enable_go_build_cache = true - enabled = true - go_version = "1.19" - [groups.build] - [groups.run] - [groups.run.test_params] - latency = "50" - bandwidth = "256Mib" - role = "validator" - -[[groups]] - id = "bridges" - builder = "docker:generic" - [groups.resources] - memory = "8Gi" - cpu = "1" - [groups.instances] - count = 8 - percentage = 0.0 - [groups.build_config] - build_base_image = "golang:1.19.1" - enable_go_build_cache = true - enabled = true - go_version = "1.19" - [groups.build] - [groups.run] - [groups.run.test_params] - latency = "50" - bandwidth = "256Mib" - role = "bridge" - -[[groups]] - id = "fulls" - builder = "docker:generic" - [groups.resources] - memory = "8Gi" - cpu = "1" - [groups.instances] - count = 200 - percentage = 0.0 - [groups.build_config] - build_base_image = "golang:1.19.1" - enable_go_build_cache = true - enabled = true - go_version = "1.19" - [groups.build] - [groups.run] - [groups.run.test_params] - latency = "50" - bandwidth = "256Mib" - role = "full" diff --git a/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-shrex-only.toml b/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-shrex-only.toml deleted file mode 100644 index c9e50190..00000000 --- a/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-shrex-only.toml +++ /dev/null @@ -1,88 +0,0 @@ -[metadata] - name = "blocksyncbench-latest" - author = "derrandz" - -[global] - plan = "celestia" - case = "blocksyncbench-latest" - total_instances = 209 - builder = "docker:generic" - runner = "cluster:k8s" - disable_metrics = false - -# we define 1 validator that produces 8mb blocks -# the msg-size does not result in exactly 8mb -# because we are accounting for padding -[global.run.test_params] - execution-time = "20" - persistent-peers = "1" - submit-times = "10" - msg-size = "600000" - validator = "1" - bridge = "8" - full = "200" - block-height = "30" - otel-collector-address = "178.128.163.171:4318" - -[[groups]] - id = "validators" - builder = "docker:generic" - [groups.resources] - memory = "4Gi" - cpu = "1" - [groups.instances] - count = 1 - percentage = 0.0 - [groups.build_config] - build_base_image = "golang:1.19.1" - enable_go_build_cache = true - enabled = true - go_version = "1.19" - [groups.build] - [groups.run] - [groups.run.test_params] - latency = "50" - bandwidth = "256Mib" - role = "validator" - -[[groups]] - id = "bridges" - builder = "docker:generic" - [groups.resources] - memory = "8Gi" - cpu = "1" - [groups.instances] - count = 8 - percentage = 0.0 - [groups.build_config] - build_base_image = "golang:1.19.1" - enable_go_build_cache = true - enabled = true - go_version = "1.19" - [groups.build] - [groups.run] - [groups.run.test_params] - latency = "50" - bandwidth = "256Mib" - role = "bridge" - -[[groups]] - id = "fulls" - builder = "docker:generic" - [groups.resources] - memory = "8Gi" - cpu = "1" - [groups.instances] - count = 200 - percentage = 0.0 - [groups.build_config] - build_base_image = "golang:1.19.1" - enable_go_build_cache = true - enabled = true - go_version = "1.19" - [groups.build] - [groups.run] - [groups.run.test_params] - latency = "50" - bandwidth = "256Mib" - role = "full" diff --git a/compositions/cluster-k8s/blocksync-historical/64-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/historical/128-square-size/1-4-32-default-getters.toml similarity index 68% rename from compositions/cluster-k8s/blocksync-historical/64-square-size/1-4-32-default-getters.toml rename to compositions/cluster-k8s/blocksync/historical/128-square-size/1-4-32-default-getters.toml index 0c786dc7..b6ad8c4a 100644 --- a/compositions/cluster-k8s/blocksync-historical/64-square-size/1-4-32-default-getters.toml +++ b/compositions/cluster-k8s/blocksync/historical/128-square-size/1-4-32-default-getters.toml @@ -12,20 +12,25 @@ [global.run.test_params] execution-time = "20" - msg-size = "600000" + msg-size = "800000" validator = "1" bridge = "3" full = "32" - block-height = 100 - target-height = 70 + block-height = "30" + target-height = "15" # the height at which historicals kick-in otel-collector-address = "178.128.163.171:4318" - getter = "default" + getter = "shrex" + +[global.run.networking_params] + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" + memory = "8Gi" cpu = "1" [groups.instances] count = 1 @@ -47,7 +52,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -68,9 +73,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 32 + count = 16 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -83,3 +88,24 @@ latency = "50" bandwidth = "256Mib" role = "full" + +[[groups]] + id = "historical-fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "historical-full" \ No newline at end of file diff --git a/compositions/cluster-k8s/blocksync-historical/128-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/historical/128-square-size/1-4-32-ipld-only.toml similarity index 71% rename from compositions/cluster-k8s/blocksync-historical/128-square-size/1-4-32-default-getters.toml rename to compositions/cluster-k8s/blocksync/historical/128-square-size/1-4-32-ipld-only.toml index ca69f990..d2f9b92f 100644 --- a/compositions/cluster-k8s/blocksync-historical/128-square-size/1-4-32-default-getters.toml +++ b/compositions/cluster-k8s/blocksync/historical/128-square-size/1-4-32-ipld-only.toml @@ -17,16 +17,20 @@ bridge = "3" full = "32" block-height = "30" - target-height = "15" + target-height = "15" # the height at which historicals kick-in otel-collector-address = "178.128.163.171:4318" - getter = "default" + getter = "ipld" + +[global.run.networking_params] peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" + memory = "8Gi" cpu = "1" [groups.instances] count = 1 @@ -48,7 +52,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -69,9 +73,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 32 + count = 16 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -84,3 +88,24 @@ latency = "50" bandwidth = "256Mib" role = "full" + +[[groups]] + id = "historical-fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "historical-full" \ No newline at end of file diff --git a/compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-default-getters.toml new file mode 100644 index 00000000..b33989a8 --- /dev/null +++ b/compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-default-getters.toml @@ -0,0 +1,111 @@ +[metadata] + name = "blocksyncbench-historical" + author = "derrandz" + +[global] + plan = "celestia" + case = "blocksyncbench-historical" + total_instances = 36 + builder = "docker:generic" + runner = "cluster:k8s" + disable_metrics = false + +[global.run.test_params] + execution-time = "20" + msg-size = "1200000" + validator = "1" + bridge = "3" + full = "32" + block-height = "30" + target-height = "15" # the height at which historicals kick-in + otel-collector-address = "178.128.163.171:4318" + getter = "shrex" + +[global.run.networking_params] + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" + +[[groups]] + id = "validators" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "1" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "validator" + +[[groups]] + id = "bridges" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "bridge" + +[[groups]] + id = "fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "full" + +[[groups]] + id = "historical-fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "historical-full" \ No newline at end of file diff --git a/compositions/cluster-k8s/blocksync-historical/128-square-size/1-4-32-ipld-only.toml b/compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-ipld-only.toml similarity index 72% rename from compositions/cluster-k8s/blocksync-historical/128-square-size/1-4-32-ipld-only.toml rename to compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-ipld-only.toml index df3e8458..aae42a58 100644 --- a/compositions/cluster-k8s/blocksync-historical/128-square-size/1-4-32-ipld-only.toml +++ b/compositions/cluster-k8s/blocksync/historical/256-square-size/1-4-32-ipld-only.toml @@ -12,21 +12,23 @@ [global.run.test_params] execution-time = "20" - msg-size = "800000" + msg-size = "1200000" validator = "1" bridge = "3" full = "32" block-height = "30" - target-height = "15" + target-height = "15" # the height at which historicals kick-in otel-collector-address = "178.128.163.171:4318" getter = "ipld" peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" + memory = "8Gi" cpu = "1" [groups.instances] count = 1 @@ -48,7 +50,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -69,9 +71,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 32 + count = 16 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -84,3 +86,24 @@ latency = "50" bandwidth = "256Mib" role = "full" + +[[groups]] + id = "historical-fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "historical-full" \ No newline at end of file diff --git a/compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-default-getters.toml new file mode 100644 index 00000000..0871a980 --- /dev/null +++ b/compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-default-getters.toml @@ -0,0 +1,111 @@ +[metadata] + name = "blocksyncbench-historical" + author = "derrandz" + +[global] + plan = "celestia" + case = "blocksyncbench-historical" + total_instances = 36 + builder = "docker:generic" + runner = "cluster:k8s" + disable_metrics = false + +[global.run.test_params] + execution-time = "20" + msg-size = "500000" + validator = "1" + bridge = "3" + full = "32" + block-height = "30" + target-height = "15" # the height at which historicals kick-in + otel-collector-address = "178.128.163.171:4318" + getter = "shrex" + +[global.run.networking_params] + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" + +[[groups]] + id = "validators" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "1" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "validator" + +[[groups]] + id = "bridges" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "bridge" + +[[groups]] + id = "fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "full" + +[[groups]] + id = "historical-fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "historical-full" \ No newline at end of file diff --git a/compositions/cluster-k8s/blocksync-historical/64-square-size/1-4-32-ipld-only.toml b/compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-ipld-only.toml similarity index 68% rename from compositions/cluster-k8s/blocksync-historical/64-square-size/1-4-32-ipld-only.toml rename to compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-ipld-only.toml index fc172fd0..0871a980 100644 --- a/compositions/cluster-k8s/blocksync-historical/64-square-size/1-4-32-ipld-only.toml +++ b/compositions/cluster-k8s/blocksync/historical/64-square-size/1-4-32-ipld-only.toml @@ -12,20 +12,25 @@ [global.run.test_params] execution-time = "20" - msg-size = "600000" + msg-size = "500000" validator = "1" bridge = "3" full = "32" - block-height = 100 - target-height = 70 + block-height = "30" + target-height = "15" # the height at which historicals kick-in otel-collector-address = "178.128.163.171:4318" - getter = "ipld" + getter = "shrex" + +[global.run.networking_params] + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" + memory = "8Gi" cpu = "1" [groups.instances] count = 1 @@ -47,7 +52,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -68,9 +73,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 32 + count = 16 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -83,3 +88,24 @@ latency = "50" bandwidth = "256Mib" role = "full" + +[[groups]] + id = "historical-fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 16 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "historical-full" \ No newline at end of file diff --git a/compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/latest-net-partition/128-square-size/1-3-32-default-getters.toml similarity index 93% rename from compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-default-getters.toml rename to compositions/cluster-k8s/blocksync/latest-net-partition/128-square-size/1-3-32-default-getters.toml index d3acccfe..2bd16c7d 100644 --- a/compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-default-getters.toml +++ b/compositions/cluster-k8s/blocksync/latest-net-partition/128-square-size/1-3-32-default-getters.toml @@ -22,20 +22,20 @@ full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" - getter = "" # must be left empty for default getters + getter = "shrex" network-partition-height = "10" fullnode-entrypoints-list = "3" # provide string list of fullnode id as follows: 10-1-44-31 (this equals [10, 1, 44, 31] bridgenode-entrypoints-list = "1" # provide a string list of bridge node ids as follows: 1-2-5-19-3 (this equals [1,2,5,19,3] - use-ipld-fallback = "1" peers-limit = "6" interconnect-bridges = "0" + bootstrapper = "1" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "1" + memory = "8Gi" + cpu = "2" [groups.instances] count = 1 percentage = 0.0 @@ -56,7 +56,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -77,7 +77,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-shrex-only.toml b/compositions/cluster-k8s/blocksync/latest-net-partition/128-square-size/1-3-32-ipld-only.toml similarity index 93% rename from compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-shrex-only.toml rename to compositions/cluster-k8s/blocksync/latest-net-partition/128-square-size/1-3-32-ipld-only.toml index 3f139f2e..c88d0f1d 100644 --- a/compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-shrex-only.toml +++ b/compositions/cluster-k8s/blocksync/latest-net-partition/128-square-size/1-3-32-ipld-only.toml @@ -22,19 +22,20 @@ full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" - getter = "shrex" - network-parition-height = "10" + getter = "ipld" + network-partition-height = "10" fullnode-entrypoints-list = "3" # provide string list of fullnode id as follows: 10-1-44-31 (this equals [10, 1, 44, 31] bridgenode-entrypoints-list = "1" # provide a string list of bridge node ids as follows: 1-2-5-19-3 (this equals [1,2,5,19,3] peers-limit = "6" interconnect-bridges = "0" + bootstrapper = "1" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "1" + memory = "8Gi" + cpu = "2" [groups.instances] count = 1 percentage = 0.0 @@ -55,9 +56,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "3" + cpu = "4" [groups.instances] - count = 3 + count = 4 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -76,7 +77,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "3500m" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/latest-net-partition/256-square-size/1-2-32-default-getters.toml similarity index 92% rename from compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-default-getters.toml rename to compositions/cluster-k8s/blocksync/latest-net-partition/256-square-size/1-2-32-default-getters.toml index 461827e9..d7e81bec 100644 --- a/compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-default-getters.toml +++ b/compositions/cluster-k8s/blocksync/latest-net-partition/256-square-size/1-2-32-default-getters.toml @@ -16,26 +16,26 @@ [global.run.test_params] execution-time = "20" persistent-peers = "1" - msg-size = "600000" + msg-size = "1200000" validator = "1" bridge = "3" full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" - getter = "" # must be left empty for default getters + getter = "shrex" network-partition-height = "10" fullnode-entrypoints-list = "3" # provide string list of fullnode id as follows: 10-1-44-31 (this equals [10, 1, 44, 31] bridgenode-entrypoints-list = "1" # provide a string list of bridge node ids as follows: 1-2-5-19-3 (this equals [1,2,5,19,3] - use-ipld-fallback = "1" peers-limit = "6" interconnect-bridges = "0" + bootstrapper = "1" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "1" + memory = "8Gi" + cpu = "2" [groups.instances] count = 1 percentage = 0.0 @@ -56,7 +56,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -77,7 +77,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-shrex-only.toml b/compositions/cluster-k8s/blocksync/latest-net-partition/256-square-size/1-2-32-ipld-only.toml similarity index 93% rename from compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-shrex-only.toml rename to compositions/cluster-k8s/blocksync/latest-net-partition/256-square-size/1-2-32-ipld-only.toml index 73ef606e..942d4803 100644 --- a/compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-shrex-only.toml +++ b/compositions/cluster-k8s/blocksync/latest-net-partition/256-square-size/1-2-32-ipld-only.toml @@ -16,25 +16,26 @@ [global.run.test_params] execution-time = "20" persistent-peers = "1" - msg-size = "600000" + msg-size = "1200000" validator = "1" bridge = "3" full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" - getter = "shrex" - network-parition-height = "10" + getter = "ipld" + network-partition-height = "10" fullnode-entrypoints-list = "3" # provide string list of fullnode id as follows: 10-1-44-31 (this equals [10, 1, 44, 31] bridgenode-entrypoints-list = "1" # provide a string list of bridge node ids as follows: 1-2-5-19-3 (this equals [1,2,5,19,3] peers-limit = "6" interconnect-bridges = "0" + bootstrapper = "1" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "1" + memory = "8" + cpu = "2" [groups.instances] count = 1 percentage = 0.0 @@ -55,7 +56,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "3" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -76,7 +77,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "3500m" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-ipld-only.toml b/compositions/cluster-k8s/blocksync/latest-net-partition/64-square-size/1-4-32-default-getters.toml similarity index 86% rename from compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-ipld-only.toml rename to compositions/cluster-k8s/blocksync/latest-net-partition/64-square-size/1-4-32-default-getters.toml index a7a0aeb0..aeed93e0 100644 --- a/compositions/cluster-k8s/blocksync-latest-net-partition/64-square-size/1-4-32-ipld-only.toml +++ b/compositions/cluster-k8s/blocksync/latest-net-partition/64-square-size/1-4-32-default-getters.toml @@ -1,11 +1,11 @@ [metadata] - name = "blocksyncbench-latest-net-parition" + name = "blocksyncbench-latest-net-partition" author = "derrandz" [global] plan = "celestia" - case = "blocksyncbench-latest-net-parition" - total_instances = 37 + case = "blocksyncbench-latest-net-partition" + total_instances = 71 builder = "docker:generic" runner = "cluster:k8s" disable_metrics = false @@ -16,18 +16,19 @@ [global.run.test_params] execution-time = "20" persistent-peers = "1" - msg-size = "600000" + msg-size = "500000" validator = "1" - bridge = "4" - full = "32" + bridge = "6" + full = "64" block-height = "30" otel-collector-address = "178.128.163.171:4318" - getter = "ipld" - network-parition-height = "10" + getter = "shrex" + network-partition-height = "10" fullnode-entrypoints-list = "3" # provide string list of fullnode id as follows: 10-1-44-31 (this equals [10, 1, 44, 31] bridgenode-entrypoints-list = "1" # provide a string list of bridge node ids as follows: 1-2-5-19-3 (this equals [1,2,5,19,3] + interconnect-bridges = "1" peers-limit = "6" - interconnect-bridges = "0" + bootstrapper = "1" [[groups]] id = "validators" @@ -57,7 +58,7 @@ memory = "8Gi" cpu = "1" [groups.instances] - count = 4 + count = 6 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -78,7 +79,7 @@ memory = "8Gi" cpu = "1" [groups.instances] - count = 32 + count = 64 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" diff --git a/compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-ipld-only.toml b/compositions/cluster-k8s/blocksync/latest-net-partition/64-square-size/1-4-32-ipld-only.toml similarity index 97% rename from compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-ipld-only.toml rename to compositions/cluster-k8s/blocksync/latest-net-partition/64-square-size/1-4-32-ipld-only.toml index 7ef12b6d..e010db2a 100644 --- a/compositions/cluster-k8s/blocksync-latest-net-partition/128-square-size/1-4-32-ipld-only.toml +++ b/compositions/cluster-k8s/blocksync/latest-net-partition/64-square-size/1-4-32-ipld-only.toml @@ -16,14 +16,14 @@ [global.run.test_params] execution-time = "20" persistent-peers = "1" - msg-size = "800000" + msg-size = "500000" validator = "1" bridge = "4" full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" getter = "ipld" - network-parition-height = "10" + network-partition-height = "10" fullnode-entrypoints-list = "3" # provide string list of fullnode id as follows: 10-1-44-31 (this equals [10, 1, 44, 31] bridgenode-entrypoints-list = "1" # provide a string list of bridge node ids as follows: 1-2-5-19-3 (this equals [1,2,5,19,3] peers-limit = "6" diff --git a/compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-default-getters.toml new file mode 100644 index 00000000..53ff3c62 --- /dev/null +++ b/compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-default-getters.toml @@ -0,0 +1,90 @@ +[metadata] + name = "blocksyncbench-latest" + author = "derrandz" + +[global] + plan = "celestia" + case = "blocksyncbench-latest" + total_instances = 36 + builder = "docker:generic" + runner = "cluster:k8s" + disable_metrics = false + +# we define 1 validator that produces 8mb blocks +# the msg-size does not result in exactly 8mb +# because we are accounting for padding +[global.run.test_params] + execution-time = "20" + msg-size = "800000" + validator = "1" + bridge = "3" + full = "32" + block-height = "100" + otel-collector-address = "178.128.163.171:4318" + getter = "shrex" + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" + +[[groups]] + id = "validators" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "2" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "validator" + +[[groups]] + id = "bridges" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "bridge" + +[[groups]] + id = "fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 32 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "full" diff --git a/compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-ipld-only.toml similarity index 88% rename from compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-default-getters.toml rename to compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-ipld-only.toml index 161bfefc..56e34579 100644 --- a/compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-default-getters.toml +++ b/compositions/cluster-k8s/blocksync/latest/128-square-size/1-4-32-ipld-only.toml @@ -5,7 +5,7 @@ [global] plan = "celestia" case = "blocksyncbench-latest" - total_instances = 37 + total_instances = 36 builder = "docker:generic" runner = "cluster:k8s" disable_metrics = false @@ -15,23 +15,23 @@ # because we are accounting for padding [global.run.test_params] execution-time = "20" - persistent-peers = "1" - submit-times = "10" msg-size = "800000" validator = "1" - bridge = "4" - full = "32" + bridge = "6" + full = "64" block-height = "30" otel-collector-address = "178.128.163.171:4318" - getter = "" - use-ipld-fallback = "1" + getter = "ipld" + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "1" + memory = "8Gi" + cpu = "2" [groups.instances] count = 1 percentage = 0.0 @@ -52,9 +52,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 4 + count = 3 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -73,7 +73,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-default-getters.toml b/compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-default-getters.toml new file mode 100644 index 00000000..86b208fb --- /dev/null +++ b/compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-default-getters.toml @@ -0,0 +1,90 @@ +[metadata] + name = "blocksyncbench-latest" + author = "derrandz" + +[global] + plan = "celestia" + case = "blocksyncbench-latest" + total_instances = 36 + builder = "docker:generic" + runner = "cluster:k8s" + disable_metrics = false + +# we define 1 validator that produces 8mb blocks +# the msg-size does not result in exactly 8mb +# because we are accounting for padding +[global.run.test_params] + execution-time = "20" + msg-size = "1200000" + validator = "1" + bridge = "3" + full = "32" + block-height = "100" + otel-collector-address = "178.128.163.171:4318" + getter = "shrex" + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" + +[[groups]] + id = "validators" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "2" + [groups.instances] + count = 1 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "validator" + +[[groups]] + id = "bridges" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 3 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "bridge" + +[[groups]] + id = "fulls" + builder = "docker:generic" + [groups.resources] + memory = "8Gi" + cpu = "4" + [groups.instances] + count = 32 + percentage = 0.0 + [groups.build_config] + build_base_image = "golang:1.19.1" + enable_go_build_cache = true + enabled = true + go_version = "1.19" + [groups.build] + [groups.run] + [groups.run.test_params] + latency = "50" + bandwidth = "256Mib" + role = "full" diff --git a/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-ipld-only.toml b/compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-ipld-only.toml similarity index 88% rename from compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-ipld-only.toml rename to compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-ipld-only.toml index 4d83dfc4..80a51c43 100644 --- a/compositions/cluster-k8s/blocksync-latest/64-square-size/1-8-200-ipld-only.toml +++ b/compositions/cluster-k8s/blocksync/latest/256-square-size/1-4-32-ipld-only.toml @@ -5,7 +5,7 @@ [global] plan = "celestia" case = "blocksyncbench-latest" - total_instances = 209 + total_instances = 36 builder = "docker:generic" runner = "cluster:k8s" disable_metrics = false @@ -15,22 +15,23 @@ # because we are accounting for padding [global.run.test_params] execution-time = "20" - persistent-peers = "1" - submit-times = "10" - msg-size = "600000" + msg-size = "1200000" validator = "1" - bridge = "8" - full = "200" + bridge = "3" + full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" getter = "ipld" + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" - cpu = "1" + memory = "8Gi" + cpu = "2" [groups.instances] count = 1 percentage = 0.0 @@ -51,7 +52,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 3 percentage = 0.0 @@ -72,9 +73,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "2" [groups.instances] - count = 12 + count = 32 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" diff --git a/compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-shrex-only.toml b/compositions/cluster-k8s/blocksync/latest/64-square-size/1-4-32-default-getters.toml similarity index 90% rename from compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-shrex-only.toml rename to compositions/cluster-k8s/blocksync/latest/64-square-size/1-4-32-default-getters.toml index db5e2a75..13dc0fef 100644 --- a/compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-shrex-only.toml +++ b/compositions/cluster-k8s/blocksync/latest/64-square-size/1-4-32-default-getters.toml @@ -5,7 +5,7 @@ [global] plan = "celestia" case = "blocksyncbench-latest" - total_instances = 37 + total_instances = 36 builder = "docker:generic" runner = "cluster:k8s" disable_metrics = false @@ -15,22 +15,22 @@ # because we are accounting for padding [global.run.test_params] execution-time = "20" - persistent-peers = "1" - submit-times = "10" - msg-size = "800000" + msg-size = "500000" validator = "1" - bridge = "4" + bridge = "3" full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" getter = "shrex" - use-ipld-fallback = "0" + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" + memory = "8Gi" cpu = "1" [groups.instances] count = 1 @@ -52,9 +52,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 4 + count = 3 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -73,7 +73,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-ipld-only.toml b/compositions/cluster-k8s/blocksync/latest/64-square-size/1-4-32-ipld-only.toml similarity index 90% rename from compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-ipld-only.toml rename to compositions/cluster-k8s/blocksync/latest/64-square-size/1-4-32-ipld-only.toml index 5228594d..17062e71 100644 --- a/compositions/cluster-k8s/blocksync-latest/128-square-size/1-4-32-ipld-only.toml +++ b/compositions/cluster-k8s/blocksync/latest/64-square-size/1-4-32-ipld-only.toml @@ -5,7 +5,7 @@ [global] plan = "celestia" case = "blocksyncbench-latest" - total_instances = 37 + total_instances = 36 builder = "docker:generic" runner = "cluster:k8s" disable_metrics = false @@ -17,20 +17,22 @@ execution-time = "20" persistent-peers = "1" submit-times = "10" - msg-size = "800000" + msg-size = "500000" validator = "1" - bridge = "4" + bridge = "3" full = "32" block-height = "30" otel-collector-address = "178.128.163.171:4318" getter = "ipld" - use-ipld-fallback = "0" + peers-limit = "3" + bootstrapper = "1" + interconnect-bridges = "0" [[groups]] id = "validators" builder = "docker:generic" [groups.resources] - memory = "4Gi" + memory = "8Gi" cpu = "1" [groups.instances] count = 1 @@ -52,9 +54,9 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] - count = 4 + count = 3 percentage = 0.0 [groups.build_config] build_base_image = "golang:1.19.1" @@ -73,7 +75,7 @@ builder = "docker:generic" [groups.resources] memory = "8Gi" - cpu = "1" + cpu = "4" [groups.instances] count = 32 percentage = 0.0 diff --git a/go.mod b/go.mod index eb1f246c..4d19e12e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/celestiaorg/test-infra go 1.19 replace ( - github.com/celestiaorg/celestia-node => github.com/derrandz/celestia-node v0.7.0-rc1-blackbox-metrics.0.20230213113250-6919f87f25b2 + github.com/celestiaorg/celestia-node => github.com/derrandz/celestia-node v0.7.0-rc1-blackbox-metrics.0.20230216131011-2f5d8c204d93 github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.8.0-sdk-v0.46.7 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20221014072825-395797efb659 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 @@ -14,7 +14,7 @@ replace ( require ( cosmossdk.io/errors v1.0.0-beta.7 // indirect - github.com/celestiaorg/celestia-app v0.12.0-rc6 + github.com/celestiaorg/celestia-app v0.12.0-rc7 github.com/cosmos/cosmos-sdk v0.46.7 github.com/ipfs/go-log/v2 v2.5.1 github.com/spf13/cobra v1.6.1 @@ -197,7 +197,6 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect - github.com/jonboulle/clockwork v0.3.1-0.20230117163003-a89700cec744 // indirect github.com/klauspost/compress v1.15.12 // indirect github.com/klauspost/cpuid/v2 v2.2.1 // indirect github.com/klauspost/reedsolomon v1.11.1 // indirect diff --git a/go.sum b/go.sum index 964c5df2..86162f8d 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v0.12.0-rc6 h1:MQ1Nb+awcnqK+EHpsH+NlBLT6F5Mph6Z3VnzgXUlxAU= -github.com/celestiaorg/celestia-app v0.12.0-rc6/go.mod h1:4qMJfFq0Yr9At8dpP271seswj1wav20vqNfrnBlFxmo= +github.com/celestiaorg/celestia-app v0.12.0-rc7 h1:83Ig3eTc0rAapUZyc4RbygJFLVfZ4z3TGgloN0fIXJw= +github.com/celestiaorg/celestia-app v0.12.0-rc7/go.mod h1:60QSrI+bbGzL/m80ocWrYpdg/ECLxmVvVets6vPuG8k= github.com/celestiaorg/celestia-core v1.14.0-tm-v0.34.23 h1:8zE523TUe5W33/nheJ9umHF2d1q6iHQlqJfMXMTPe3k= github.com/celestiaorg/celestia-core v1.14.0-tm-v0.34.23/go.mod h1:fGDSg7aw2OH/Uze1zymop0x0y1kAPEO9OII2A2cb99Q= github.com/celestiaorg/cosmos-sdk v1.8.0-sdk-v0.46.7 h1:EADZy33ufskVIy6Rj6jbi3SOVCeYYo26zUi7iYx+QR0= @@ -364,8 +364,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2U github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= -github.com/derrandz/celestia-node v0.7.0-rc1-blackbox-metrics.0.20230213113250-6919f87f25b2 h1:hZgRudfsckM+U7x9yuNt4oFaVCH+LaOsMeHfSP6Wa78= -github.com/derrandz/celestia-node v0.7.0-rc1-blackbox-metrics.0.20230213113250-6919f87f25b2/go.mod h1:dPKHAWfxf334I9TAfhuPHdcvv8kvMUfabG8Zw/11hW8= +github.com/derrandz/celestia-node v0.7.0-rc1-blackbox-metrics.0.20230216131011-2f5d8c204d93 h1:VxzMKKkH/tMugGp1LhRuigV5NXS9XHFU0OhQgp8SBTs= +github.com/derrandz/celestia-node v0.7.0-rc1-blackbox-metrics.0.20230216131011-2f5d8c204d93/go.mod h1:hstRSWRQPReb1Gl8FFSPEN7VKbxaaeHRY+1oFK86IGg= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= @@ -952,8 +952,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.3.1-0.20230117163003-a89700cec744 h1:fJ+REXDOpsMqA2spt3wAq3HGJJvWnNitGK2KVZTos+8= -github.com/jonboulle/clockwork v0.3.1-0.20230117163003-a89700cec744/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= diff --git a/main.go b/main.go index 507e0355..fd1ea674 100644 --- a/main.go +++ b/main.go @@ -28,11 +28,11 @@ var testcases = map[string]interface{}{ // BlockSync Benchmarks - Syncing Latest "blocksyncbench-latest": blocksync.BlockSyncLatest, // BlockSync Benchmarks - Syncing Latest With Network Hiccups - "blocksyncbench-latest-net-hiccup": blocksync.BlockSyncLatestWithNetworkPartitions, + "blocksyncbench-latest-net-partition": blocksync.BlockSyncLatestWithNetworkPartitions, // BlockSync Benchmarks - Syncing Historical "blocksyncbench-historical": blocksync.BlockSyncHistorical, // BlockSync Benchmarks - Syncing Historical With Network Hiccups - "blocksyncbench-historical-with-hiccups": blocksync.BlockSyncHistoricalWithHiccups, + "blocksyncbench-historical-net-partition": blocksync.BlockSyncHistoricalWithHiccups, } func main() { diff --git a/testkit/sync.go b/testkit/sync.go index a560297f..93d5cd44 100644 --- a/testkit/sync.go +++ b/testkit/sync.go @@ -51,11 +51,12 @@ var ( // FinishState should be signaled by those, againts which we are testing var ( - AppStartedState = sync.State("app-started") - BridgeStartedState = sync.State("bridge-started") - PastBlocksGeneratedState = sync.State("past-blocks-generated") - AccountsFundedState = sync.State("accounts-funded") - FinishState = sync.State("test-finished") - LightNodesStartedState = sync.State("light-nodes-started") - ValidatorReadyTopic = sync.State("validator-ready") + AppStartedState = sync.State("app-started") + BridgeStartedState = sync.State("bridge-started") + PastBlocksGeneratedState = sync.State("past-blocks-generated") + AccountsFundedState = sync.State("accounts-funded") + FinishState = sync.State("test-finished") + LightNodesStartedState = sync.State("light-nodes-started") + ValidatorReadyTopic = sync.State("validator-ready") + FullsFinishedSyncingState = sync.State("fulls-finished-syncing") ) diff --git a/tests/helpers/blocksyncbench-historical/run_bridge.go b/tests/helpers/blocksync/historical/run_bridge.go similarity index 76% rename from tests/helpers/blocksyncbench-historical/run_bridge.go rename to tests/helpers/blocksync/historical/run_bridge.go index 5c076fa5..11f31170 100644 --- a/tests/helpers/blocksyncbench-historical/run_bridge.go +++ b/tests/helpers/blocksync/historical/run_bridge.go @@ -1,4 +1,4 @@ -package blocksyncbenchhistorical +package blocksynchistorical import ( "context" @@ -8,13 +8,13 @@ import ( "github.com/celestiaorg/test-infra/testkit" "github.com/celestiaorg/test-infra/testkit/nodekit" "github.com/celestiaorg/test-infra/tests/helpers/common" + "github.com/libp2p/go-libp2p/core/host" "github.com/testground/sdk-go/network" "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" - "go.opentelemetry.io/otel/metric" ) -func RunBridgeNode(runenv *runtime.RunEnv, initCtx *run.InitContext, _ metric.Meter) error { +func RunBridgeNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { ctx, cancel := context.WithTimeout( context.Background(), time.Minute*time.Duration(runenv.IntParam("execution-time")), @@ -82,6 +82,17 @@ func RunBridgeNode(runenv *runtime.RunEnv, initCtx *run.InitContext, _ metric.Me return lerr } + runenv.RecordMessage("Connecting to other bridge nodes") + if runenv.IntParam("interconnect-bridges") == 1 { + bridgeNodes, _ := common.GetBridgeNodes(ctx, syncclient, runenv.IntParam("bridge")) + for _, bridge := range bridgeNodes { + if bridge.AddrInfo.ID != host.InfoFromHost(nd.Host).ID { + nd.Host.Connect(ctx, bridge.AddrInfo) + runenv.RecordMessage("Connected to Bridge:", bridge.AddrInfo.Addrs) + } + } + } + for i := 0; i < runenv.IntParam("block-height"); i++ { // After reaching a dedicated block-height, we can signal other node types // to start syncing the past @@ -95,18 +106,32 @@ func RunBridgeNode(runenv *runtime.RunEnv, initCtx *run.InitContext, _ metric.Me runenv.IntParam("block-height"), eh.Commit.BlockID.Hash.String(), ) - if int64(runenv.IntParam("target-height")) == eh.RawHeader.Height { - _, err := syncclient.SignalEntry(ctx, testkit.PastBlocksGeneratedState) - if err != nil { - runenv.RecordFailure(err) - } + } - // runenv.D().Counter("bridge_nodes_reached_target_height").Inc(1) - } + _, err = syncclient.SignalEntry(ctx, testkit.PastBlocksGeneratedState) + if err != nil { + runenv.RecordFailure(err) + return err } if nd.HeaderServ.IsSyncing(ctx) { runenv.RecordFailure(fmt.Errorf("Bridge node is still syncing the past")) + return err + } + + l, err = syncclient.Barrier( + ctx, + testkit.FullsFinishedSyncingState, + runenv.IntParam("historical-syncers-max-id"), // 1 + ) + if err != nil { + runenv.RecordFailure(err) + return err + } + lerr = <-l.C + if lerr != nil { + runenv.RecordFailure(lerr) + return err } err = nd.Stop(ctx) diff --git a/tests/helpers/blocksync/historical/run_full.go b/tests/helpers/blocksync/historical/run_full.go new file mode 100644 index 00000000..bc180555 --- /dev/null +++ b/tests/helpers/blocksync/historical/run_full.go @@ -0,0 +1,146 @@ +package blocksynchistorical + +import ( + "context" + "fmt" + "time" + + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/test-infra/testkit" + "github.com/celestiaorg/test-infra/testkit/nodekit" + "github.com/celestiaorg/test-infra/tests/helpers/common" + "github.com/testground/sdk-go/network" + "github.com/testground/sdk-go/run" + "github.com/testground/sdk-go/runtime" +) + +func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { + ctx, cancel := context.WithTimeout( + context.Background(), + time.Minute*time.Duration(runenv.IntParam("execution-time")), + ) + defer cancel() + + err := nodekit.SetLoggersLevel("DEBUG") + if err != nil { + return err + } + + syncclient := initCtx.SyncClient + netclient := network.NewClient(syncclient, runenv) + + netclient.MustWaitNetworkInitialized(ctx) + + config := network.Config{ + Network: "default", + Enable: true, + Default: network.LinkShape{ + Latency: time.Duration(runenv.IntParam("latency")), + Bandwidth: common.GetBandwidthValue(runenv.StringParam("bandwidth")), + }, + CallbackState: "network-configured", + RoutingPolicy: network.AllowAll, + } + + config.IPv4 = runenv.TestSubnet + + // using the assigned `GlobalSequencer` id per each of instance + // to fill in the last 2 octets of the new IP address for the instance + ipC := byte((initCtx.GlobalSeq >> 8) + 1) + ipD := byte(initCtx.GlobalSeq) + config.IPv4.IP = append(config.IPv4.IP[0:2:2], ipC, ipD) + + err = netclient.ConfigureNetwork(ctx, &config) + if err != nil { + return err + } + + bridgeNodes, err := common.GetBridgeNodes(ctx, syncclient, runenv.IntParam("bridge")) + if err != nil { + return err + } + + var bridgeNode *testkit.BridgeNodeInfo + for _, bridge := range bridgeNodes { + if (int(initCtx.GroupSeq) % runenv.IntParam("bridge")) == (bridge.ID % runenv.IntParam("bridge")) { + bridgeNode = bridge + } + } + trustedPeers := []string{} + for _, bridge := range bridgeNodes { + trustedPeers = append(trustedPeers, bridge.Maddr) + } + + ndhome := fmt.Sprintf("/.celestia-full-%d", initCtx.GlobalSeq) + runenv.RecordMessage(ndhome) + + ip, err := initCtx.NetClient.GetDataNetworkIP() + if err != nil { + return err + } + + cfg := nodekit.NewConfig(node.Full, ip, trustedPeers, bridgeNode.TrustedHash) + + switch runenv.StringParam("getter") { + case "shrex": + cfg.Share.UseShareExchange = true + case "ipld": + fallthrough + default: + cfg.Share.UseShareExchange = false + } + + + nd, err := nodekit.NewNode( + ndhome, + node.Full, + cfg, + ) + if err != nil { + return err + } + + runenv.RecordMessage("Waiting for historical blocks to be generated...") + + runenv.RecordMessage("Starting full node") + err = nd.Start(ctx) + if err != nil { + return err + } + + runenv.RecordMessage("Full node is syncing") + + eh, err := nd.HeaderServ.GetByHeight(ctx, uint64(runenv.IntParam("block-height"))) + if err != nil { + return err + } + runenv.RecordMessage("Reached Block#%d contains Hash: %s", + runenv.IntParam("block-height"), + eh.Commit.BlockID.Hash.String()) + + if nd.HeaderServ.IsSyncing(ctx) { + runenv.RecordFailure(fmt.Errorf("full node is still syncing the past")) + } + + l, err := syncclient.Barrier(ctx, testkit.FullsFinishedSyncingState, runenv.IntParam("historical-syncers-max-id")) + if err != nil { + runenv.RecordFailure(err) + return err + } + lerr := <-l.C + if lerr != nil { + runenv.RecordFailure(lerr) + return err + } + err = nd.Stop(ctx) + if err != nil { + return err + } + + _, err = syncclient.SignalEntry(ctx, testkit.FinishState) + if err != nil { + return err + } + + return err +} diff --git a/tests/helpers/blocksyncbench-historical/run_full.go b/tests/helpers/blocksync/historical/run_historical_full.go similarity index 85% rename from tests/helpers/blocksyncbench-historical/run_full.go rename to tests/helpers/blocksync/historical/run_historical_full.go index ae7c9642..cf2fcc8c 100644 --- a/tests/helpers/blocksyncbench-historical/run_full.go +++ b/tests/helpers/blocksync/historical/run_historical_full.go @@ -1,4 +1,4 @@ -package blocksyncbenchhistorical +package blocksynchistorical import ( "context" @@ -12,24 +12,19 @@ import ( "github.com/celestiaorg/test-infra/tests/helpers/common" "github.com/testground/sdk-go/network" "github.com/testground/sdk-go/run" - "github.com/testground/sdk-go/runtime" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" - "go.opentelemetry.io/otel/metric" + "github.com/testground/sdk-go/runtime" ) -func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric.Meter) error { +func RunHistoricalFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { ctx, cancel := context.WithTimeout( context.Background(), time.Minute*time.Duration(runenv.IntParam("execution-time")), ) defer cancel() - syncingFullNodesMetric, err := meter.SyncInt64().Counter("syncing_full_nodes") - if err != nil { - runenv.RecordFailure(err) - } - - err = nodekit.SetLoggersLevel("INFO") + err := nodekit.SetLoggersLevel("DEBUG") if err != nil { return err } @@ -74,6 +69,10 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric. bridgeNode = bridge } } + trustedPeers := []string{} + for _, bridge := range bridgeNodes { + trustedPeers = append(trustedPeers, bridge.Maddr) + } ndhome := fmt.Sprintf("/.celestia-full-%d", initCtx.GlobalSeq) runenv.RecordMessage(ndhome) @@ -83,13 +82,15 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric. return err } - trustedPeers := []string{bridgeNode.Maddr} cfg := nodekit.NewConfig(node.Full, ip, trustedPeers, bridgeNode.TrustedHash) switch runenv.StringParam("getter") { + case "shrex": + cfg.Share.UseShareExchange = true case "ipld": - cfg.Share.NoCascade = true - cfg.Share.DefaultGetter = "ipld" + fallthrough + default: + cfg.Share.UseShareExchange = false } optlOpts := []otlpmetrichttp.Option{ @@ -102,7 +103,7 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric. cfg, nodebuilder.WithMetrics( optlOpts, - node.Light, + node.Full, ), ) if err != nil { @@ -128,8 +129,6 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric. return err } - syncingFullNodesMetric.Add(ctx, 1) - runenv.RecordMessage("Full node is syncing") eh, err := nd.HeaderServ.GetByHeight(ctx, uint64(runenv.IntParam("block-height"))) @@ -143,7 +142,13 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric. if nd.HeaderServ.IsSyncing(ctx) { runenv.RecordFailure(fmt.Errorf("full node is still syncing the past")) } + <-time.After(time.Minute * 2) + if err = nd.DASer.WaitCatchUp(ctx); err != nil { + runenv.RecordFailure(err) + return err + } + _, err = syncclient.SignalEntry(ctx, testkit.FullsFinishedSyncingState) err = nd.Stop(ctx) if err != nil { return err @@ -154,6 +159,5 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext, meter metric. return err } - syncingFullNodesMetric.Add(ctx, -1) return err } diff --git a/tests/helpers/blocksyncbench-historical/run_validator.go b/tests/helpers/blocksync/historical/run_validator.go similarity index 69% rename from tests/helpers/blocksyncbench-historical/run_validator.go rename to tests/helpers/blocksync/historical/run_validator.go index b760956a..0176a1b0 100644 --- a/tests/helpers/blocksyncbench-historical/run_validator.go +++ b/tests/helpers/blocksync/historical/run_validator.go @@ -1,4 +1,4 @@ -package blocksyncbenchhistorical +package blocksynchistorical import ( "context" @@ -7,7 +7,6 @@ import ( "github.com/celestiaorg/test-infra/testkit/appkit" "github.com/celestiaorg/test-infra/tests/helpers/common" - "go.opentelemetry.io/otel/metric" "github.com/celestiaorg/test-infra/testkit" "github.com/testground/sdk-go/network" @@ -15,7 +14,7 @@ import ( "github.com/testground/sdk-go/runtime" ) -func RunValidator(runenv *runtime.RunEnv, initCtx *run.InitContext, _ metric.Meter) error { +func RunValidator(runenv *runtime.RunEnv, initCtx *run.InitContext) error { ctx, cancel := context.WithTimeout( context.Background(), time.Minute*time.Duration(runenv.IntParam("execution-time")), @@ -104,35 +103,56 @@ func RunValidator(runenv *runtime.RunEnv, initCtx *run.InitContext, _ metric.Met return err } - for j := 0; j < runenv.IntParam("block-height"); j++ { - runenv.RecordMessage("Submitting PFD with %d bytes random data", runenv.IntParam("msg-size")) - err = appcmd.PayForBlob( - appcmd.AccountAddress, - runenv.IntParam("msg-size"), - "test", - appcmd.GetHomePath(), - ) - if err != nil { - runenv.RecordFailure(err) - return err - } - - _, _, err := appkit.GetLatestBlockSizeAndHeight(net.ParseIP("127.0.0.1")) - if err != nil { - runenv.RecordMessage("err in last size call, %s", err.Error()) - } - } - l, err = syncclient.Barrier(ctx, testkit.FinishState, runenv.IntParam("full")+runenv.IntParam("bridge")) if err != nil { runenv.RecordFailure(err) return err } - lerr = <-l.C - if lerr != nil { - runenv.RecordFailure(lerr) - return err + for { + select { + case lerr := <-l.C: + if err != nil { + runenv.RecordFailure(lerr) + return err + } + default: + runenv.RecordMessage("Submitting PFD with %d bytes random data", runenv.IntParam("msg-size")) + err = appcmd.PayForBlob( + appcmd.AccountAddress, + runenv.IntParam("msg-size"), + "test", + appcmd.GetHomePath(), + ) + if err != nil { + runenv.RecordFailure(err) + return err + } + + _, _, err := appkit.GetLatestBlockSizeAndHeight(net.ParseIP("127.0.0.1")) + if err != nil { + runenv.RecordMessage("err in last size call, %s", err.Error()) + } + } } + // for j := 0; j < runenv.IntParam("block-height"); j++ { + // runenv.RecordMessage("Submitting PFD with %d bytes random data", runenv.IntParam("msg-size")) + // err = appcmd.PayForBlob( + // appcmd.AccountAddress, + // runenv.IntParam("msg-size"), + // "test", + // appcmd.GetHomePath(), + // ) + // if err != nil { + // runenv.RecordFailure(err) + // return err + // } + + // _, _, err := appkit.GetLatestBlockSizeAndHeight(net.ParseIP("127.0.0.1")) + // if err != nil { + // runenv.RecordMessage("err in last size call, %s", err.Error()) + // } + // } + runenv.RecordSuccess() return nil diff --git a/tests/helpers/blocksyncbench-latest-net-parition/run_bridge.go b/tests/helpers/blocksync/latest-net-parition/run_bridge.go similarity index 98% rename from tests/helpers/blocksyncbench-latest-net-parition/run_bridge.go rename to tests/helpers/blocksync/latest-net-parition/run_bridge.go index 853f9b99..dd0b5ff7 100644 --- a/tests/helpers/blocksyncbench-latest-net-parition/run_bridge.go +++ b/tests/helpers/blocksync/latest-net-parition/run_bridge.go @@ -1,4 +1,4 @@ -package blocksyncbenchlatestnetpartition +package blocksynclatestnetpartition import ( "context" diff --git a/tests/helpers/blocksyncbench-latest-net-parition/run_full.go b/tests/helpers/blocksync/latest-net-parition/run_full.go similarity index 94% rename from tests/helpers/blocksyncbench-latest-net-parition/run_full.go rename to tests/helpers/blocksync/latest-net-parition/run_full.go index 9c7fcff1..a5589508 100644 --- a/tests/helpers/blocksyncbench-latest-net-parition/run_full.go +++ b/tests/helpers/blocksync/latest-net-parition/run_full.go @@ -1,4 +1,4 @@ -package blocksyncbenchlatestnetpartition +package blocksynclatestnetpartition import ( "context" @@ -91,18 +91,12 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { cfg.Share.PeersLimit = uint(runenv.IntParam("peers-limit")) switch runenv.StringParam("getter") { - case "ipld": - cfg.Share.NoCascade = true - cfg.Share.DefaultGetter = "ipld" - case "shrex": - cfg.Share.NoCascade = true - cfg.Share.DefaultGetter = "shrex" - + cfg.Share.UseShareExchange = true + case "ipld": + fallthrough default: - if runenv.IntParam("use-ipld-fallback") == 0 { - cfg.Share.UseIPLDFallback = false - } + cfg.Share.UseShareExchange = false } optlOpts := []otlpmetrichttp.Option{ diff --git a/tests/helpers/blocksyncbench-latest-net-parition/run_validator.go b/tests/helpers/blocksync/latest-net-parition/run_validator.go similarity index 98% rename from tests/helpers/blocksyncbench-latest-net-parition/run_validator.go rename to tests/helpers/blocksync/latest-net-parition/run_validator.go index 2ca96f3a..1bafb121 100644 --- a/tests/helpers/blocksyncbench-latest-net-parition/run_validator.go +++ b/tests/helpers/blocksync/latest-net-parition/run_validator.go @@ -1,4 +1,4 @@ -package blocksyncbenchlatestnetpartition +package blocksynclatestnetpartition import ( "context" diff --git a/tests/helpers/blocksyncbench-latest/run_bridge.go b/tests/helpers/blocksync/latest/run_bridge.go similarity index 98% rename from tests/helpers/blocksyncbench-latest/run_bridge.go rename to tests/helpers/blocksync/latest/run_bridge.go index d0908081..c0ba9192 100644 --- a/tests/helpers/blocksyncbench-latest/run_bridge.go +++ b/tests/helpers/blocksync/latest/run_bridge.go @@ -1,4 +1,4 @@ -package blocksyncbenchlatest +package blocksynclatest import ( "context" diff --git a/tests/helpers/blocksyncbench-latest/run_full.go b/tests/helpers/blocksync/latest/run_full.go similarity index 92% rename from tests/helpers/blocksyncbench-latest/run_full.go rename to tests/helpers/blocksync/latest/run_full.go index 7c8176a0..92ac53c1 100644 --- a/tests/helpers/blocksyncbench-latest/run_full.go +++ b/tests/helpers/blocksync/latest/run_full.go @@ -1,4 +1,4 @@ -package blocksyncbenchlatest +package blocksynclatest import ( "context" @@ -74,18 +74,12 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { cfg := nodekit.NewConfig(node.Full, ip, trustedPeers, bridgeNode.TrustedHash) switch runenv.StringParam("getter") { - case "ipld": - cfg.Share.NoCascade = true - cfg.Share.DefaultGetter = "ipld" - case "shrex": - cfg.Share.NoCascade = true - cfg.Share.DefaultGetter = "shrex" - + cfg.Share.UseShareExchange = true + case "ipld": + fallthrough default: - if runenv.IntParam("use-ipld-fallback") == 0 { - cfg.Share.UseIPLDFallback = false - } + cfg.Share.UseShareExchange = false } optlOpts := []otlpmetrichttp.Option{ @@ -98,7 +92,7 @@ func RunFullNode(runenv *runtime.RunEnv, initCtx *run.InitContext) error { cfg, nodebuilder.WithMetrics( optlOpts, - node.Light, + node.Full, ), ) if err != nil { diff --git a/tests/helpers/blocksyncbench-latest/run_validator.go b/tests/helpers/blocksync/latest/run_validator.go similarity index 99% rename from tests/helpers/blocksyncbench-latest/run_validator.go rename to tests/helpers/blocksync/latest/run_validator.go index d3ea1e0e..03cedd24 100644 --- a/tests/helpers/blocksyncbench-latest/run_validator.go +++ b/tests/helpers/blocksync/latest/run_validator.go @@ -1,4 +1,4 @@ -package blocksyncbenchlatest +package blocksynclatest import ( "context" diff --git a/tests/helpers/common/bridge.go b/tests/helpers/common/bridge.go index 014daa03..04a34ede 100644 --- a/tests/helpers/common/bridge.go +++ b/tests/helpers/common/bridge.go @@ -50,7 +50,7 @@ func BuildBridge(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitC cfg.Core.GRPCPort = "9090" cfg.Gateway.Enabled = true cfg.Gateway.Port = "26659" - cfg.P2P.Bootstrapper = true + cfg.P2P.Bootstrapper = bool(runenv.IntParam("bootstrapper") == 1) cfg.Share.PeersLimit = uint(runenv.IntParam("peers-limit")) optlOpts := []otlpmetrichttp.Option{ diff --git a/tests/plans/block-sync/block-sync.go b/tests/plans/block-sync/block-sync.go index 1f9e70d3..b1c079e7 100644 --- a/tests/plans/block-sync/block-sync.go +++ b/tests/plans/block-sync/block-sync.go @@ -4,13 +4,11 @@ import ( "context" "github.com/celestiaorg/test-infra/testkit" - "github.com/celestiaorg/test-infra/testkit/optlkit" - blocksyncbenchhistorical "github.com/celestiaorg/test-infra/tests/helpers/blocksyncbench-historical" - blocksyncbenchlatest "github.com/celestiaorg/test-infra/tests/helpers/blocksyncbench-latest" - blocksyncbenchlatestnetpartition "github.com/celestiaorg/test-infra/tests/helpers/blocksyncbench-latest-net-parition" + blocksynchistorical "github.com/celestiaorg/test-infra/tests/helpers/blocksync/historical" + blocksynclatest "github.com/celestiaorg/test-infra/tests/helpers/blocksync/latest" + blocksynclatestnetpartition "github.com/celestiaorg/test-infra/tests/helpers/blocksync/latest-net-parition" "github.com/testground/sdk-go/run" "github.com/testground/sdk-go/runtime" - "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" ) // BlockSyncLatest represents a testcase of W validator, X bridges, Y full nodes that @@ -20,11 +18,11 @@ import ( func BlockSyncLatest(runenv *runtime.RunEnv, initCtx *run.InitContext) (err error) { switch runenv.StringParam("role") { case "validator": - err = blocksyncbenchlatest.RunValidator(runenv, initCtx) + err = blocksynclatest.RunValidator(runenv, initCtx) case "bridge": - err = blocksyncbenchlatest.RunBridgeNode(runenv, initCtx) + err = blocksynclatest.RunBridgeNode(runenv, initCtx) case "full": - err = blocksyncbenchlatest.RunFullNode(runenv, initCtx) + err = blocksynclatest.RunFullNode(runenv, initCtx) } if err != nil { @@ -46,11 +44,11 @@ func BlockSyncLatest(runenv *runtime.RunEnv, initCtx *run.InitContext) (err erro func BlockSyncLatestWithNetworkPartitions(runenv *runtime.RunEnv, initCtx *run.InitContext) (err error) { switch runenv.StringParam("role") { case "validator": - err = blocksyncbenchlatestnetpartition.RunValidator(runenv, initCtx) + err = blocksynclatestnetpartition.RunValidator(runenv, initCtx) case "bridge": - err = blocksyncbenchlatestnetpartition.RunBridgeNode(runenv, initCtx) + err = blocksynclatestnetpartition.RunBridgeNode(runenv, initCtx) case "full": - err = blocksyncbenchlatestnetpartition.RunFullNode(runenv, initCtx) + err = blocksynclatestnetpartition.RunFullNode(runenv, initCtx) } if err != nil { @@ -68,28 +66,15 @@ func BlockSyncLatestWithNetworkPartitions(runenv *runtime.RunEnv, initCtx *run.I // using either IPLDGetter only or the default CascadeGetter (_see compositions/cluster-k8s/blocksync-historical/*/*-{getter}.toml) // More information under docs/test-plans/005-Block-Sync func BlockSyncHistorical(runenv *runtime.RunEnv, initCtx *run.InitContext) (err error) { - optlOpts := []otlpmetrichttp.Option{ - otlpmetrichttp.WithEndpoint(runenv.StringParam("otel-collector-address")), - otlpmetrichttp.WithInsecure(), - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - meterProvider, stopFn, err := optlkit.SetupMeter(ctx, initCtx, "BlockSyncHistorical", runenv.StringParam("role"), optlOpts) - if err != nil { - runenv.RecordFailure(err) - return err - } - - blockSyncMeter := (*meterProvider).Meter("blocksync-historical") - switch runenv.StringParam("role") { case "validator": - err = blocksyncbenchhistorical.RunValidator(runenv, initCtx, blockSyncMeter) + err = blocksynchistorical.RunValidator(runenv, initCtx) case "bridge": - err = blocksyncbenchhistorical.RunBridgeNode(runenv, initCtx, blockSyncMeter) + err = blocksynchistorical.RunBridgeNode(runenv, initCtx) case "full": - err = blocksyncbenchhistorical.RunFullNode(runenv, initCtx, blockSyncMeter) + err = blocksynchistorical.RunFullNode(runenv, initCtx) + case "historical-full": + err = blocksynchistorical.RunHistoricalFullNode(runenv, initCtx) } if err != nil { @@ -98,12 +83,6 @@ func BlockSyncHistorical(runenv *runtime.RunEnv, initCtx *run.InitContext) (err return err } - err = stopFn(ctx) - if err != nil { - runenv.RecordFailure(err) - return err - } - runenv.RecordSuccess() return }